update uint256 for evm

blocks/antelope/Makefile

@@ -11,7 +11,7 @@ build:
 
 .PHONY: gui
 gui:
-	substreams gui -e eos.substreams.pinax.network:443 map_events -s 386946801 -t 386946802
+	substreams gui . -e eos.substreams.pinax.network:443 map_events -s 386946801 -t 386946802 --network eos
 
 .PHONY: protogen
 protogen:

blocks/evm/Makefile

@@ -11,19 +11,19 @@ build:
 
 .PHONY: gui
 gui:
-	substreams gui -e eth.substreams.pinax.network:443 map_events -s 20500000 -t 20500001
+	substreams gui . -e eth.substreams.pinax.network:443 map_events -s 20500000 -t 20500001 --network mainnet
 
 .PHONY: protogen
 protogen:
 	substreams protogen
 
 .PHONY: parquet
 parquet:
-	substreams-sink-files run eth.substreams.pinax.network:443 substreams.yaml map_events ./out 20500000:20500001 --encoder parquet --file-block-count 1 --development-mode --parquet-default-column-compression snappy
+	rm -rf out state.yaml && substreams-sink-files run eth.substreams.pinax.network:443 substreams.yaml map_events ./out 20500000:20500001 --encoder parquet --file-block-count 1 --development-mode --parquet-default-column-compression snappy
 
 .PHONY: s3
 s3:
-	substreams-sink-files run eth.substreams.pinax.network:443 substreams.yaml map_events s3://pinax/eth/7605ec58a864d4016ed2245b066d8a8c09b891d3?region=us-east-1 21252718: --encoder parquet --file-block-count 1 --development-mode --parquet-default-column-compression snappy
+	substreams-sink-files run eth.substreams.pinax.network:443 substreams.yaml map_events s3://pinax/eth/7605ec58a864d4016ed2245b066d8a8c09b891d3?region=us-east-1 21252718: --encoder parquet --file-block-count 1 --development-mode --parquet-default-column-compression uncompressed
 
 .PHONY: hash
 hash:

blocks/evm/src/balance_changes.rs

@@ -1,6 +1,5 @@
 use common::structs::BlockTimestamp;
-use common::utils::optional_bigint_to_string;
-use common::utils::{bytes_to_hex, optional_bigint_to_decimal};
+use common::utils::bytes_to_hex;
 use substreams_ethereum::pb::eth::v2::{BalanceChange, Block, TransactionTrace};
 
 use crate::pb::pinax::evm::v1::BalanceChange as BalanceChangeEvent;
@@ -58,7 +57,7 @@ pub fn collect_balance_changes(block: &Block, timestamp: &BlockTimestamp) -> Vec
 }
 
 pub fn parse_balance_change(balance_change: &BalanceChange, transaction: &TransactionTrace, timestamp: &BlockTimestamp) -> BalanceChangeEvent {
-    let amount = optional_bigint_to_decimal(balance_change.new_value.clone()) - optional_bigint_to_decimal(balance_change.old_value.clone());
+    // let amount = optional_bigint_to_decimal(balance_change.new_value.clone()) - optional_bigint_to_decimal(balance_change.old_value.clone());
     BalanceChangeEvent {
         // block
         block_time: Some(timestamp.time),
@@ -71,9 +70,9 @@ pub fn parse_balance_change(balance_change: &BalanceChange, transaction: &Transa
 
         // balance changes
         address: bytes_to_hex(&balance_change.address),
-        new_balance: optional_bigint_to_string(&balance_change.new_value, "0"),
-        old_balance: optional_bigint_to_string(&balance_change.old_value, "0"),
-        amount: amount.to_string(),
+        old_balance_bytes: balance_change.old_value.clone().unwrap_or_default().bytes,
+        new_balance_bytes: balance_change.new_value.clone().unwrap_or_default().bytes,
+
         ordinal: balance_change.ordinal,
         reason: balance_change_reason_to_string(balance_change.reason),
         reason_code: balance_change.reason as u32,

blocks/evm/src/blocks.rs

@@ -44,7 +44,7 @@ pub fn collect_block(block: &Block, timestamp: &BlockTimestamp) -> BlockHeader {
         parent_beacon_root: bytes_to_hex(&header.parent_beacon_root),
         miner: bytes_to_hex(&header.coinbase),
         difficulty: optional_bigint_to_u64(&header.difficulty),
-        total_difficulty: optional_bigint_to_string(&header.total_difficulty, "0"),
+        total_difficulty_bytes: header.total_difficulty.clone().unwrap_or_default().bytes,
         mix_hash: bytes_to_hex(&header.mix_hash),
         extra_data: bytes_to_hex(&header.extra_data),
         extra_data_utf8: String::from_utf8(header.extra_data.clone()).unwrap_or_default(),

blocks/evm/src/pb/mod.rs

@@ -1,9 +1,4 @@
 // @generated
-// @@protoc_insertion_point(attribute:parquet)
-pub mod parquet {
-    include!("parquet.rs");
-    // @@protoc_insertion_point(parquet)
-}
 pub mod pinax {
     pub mod evm {
         // @@protoc_insertion_point(attribute:pinax.evm.v1)

blocks/evm/src/pb/pinax.evm.v1.rs

@@ -63,8 +63,9 @@ pub struct Block {
     pub miner: ::prost::alloc::string::String,
     #[prost(uint64, tag="15")]
     pub difficulty: u64,
-    #[prost(string, tag="16")]
-    pub total_difficulty: ::prost::alloc::string::String,
+    /// UInt256
+    #[prost(bytes="vec", tag="16")]
+    pub total_difficulty_bytes: ::prost::alloc::vec::Vec<u8>,
     #[prost(string, tag="17")]
     pub mix_hash: ::prost::alloc::string::String,
     #[prost(string, tag="18")]
@@ -109,8 +110,7 @@ pub struct Block {
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct Transaction {
     /// -- block --
-    #[prost(message, optional, tag="1")]
-    pub block_time: ::core::option::Option<::prost_types::Timestamp>,
+    /// google.protobuf.Timestamp block_time = 1;
     #[prost(uint64, tag="2")]
     pub block_number: u64,
     #[prost(string, tag="3")]
@@ -139,12 +139,14 @@ pub struct Transaction {
     pub status_code: u32,
     #[prost(bool, tag="14")]
     pub success: bool,
-    #[prost(string, tag="15")]
-    pub gas_price: ::prost::alloc::string::String,
+    /// UInt256
+    #[prost(bytes="vec", tag="15")]
+    pub gas_price_bytes: ::prost::alloc::vec::Vec<u8>,
     #[prost(uint64, tag="16")]
     pub gas_limit: u64,
-    #[prost(string, tag="17")]
-    pub value: ::prost::alloc::string::String,
+    /// UInt256
+    #[prost(bytes="vec", tag="17")]
+    pub value_bytes: ::prost::alloc::vec::Vec<u8>,
     #[prost(string, tag="18")]
     pub data: ::prost::alloc::string::String,
     #[prost(string, tag="19")]
@@ -161,17 +163,21 @@ pub struct Transaction {
     /// EIP-1559
     #[prost(uint32, tag="24")]
     pub type_code: u32,
-    #[prost(string, tag="25")]
-    pub max_fee_per_gas: ::prost::alloc::string::String,
-    #[prost(string, tag="26")]
-    pub max_priority_fee_per_gas: ::prost::alloc::string::String,
+    /// UInt256
+    #[prost(bytes="vec", tag="25")]
+    pub max_fee_per_gas_bytes: ::prost::alloc::vec::Vec<u8>,
+    /// UInt256
+    #[prost(bytes="vec", tag="26")]
+    pub max_priority_fee_per_gas_bytes: ::prost::alloc::vec::Vec<u8>,
     #[prost(uint64, tag="27")]
     pub begin_ordinal: u64,
     #[prost(uint64, tag="28")]
     pub end_ordinal: u64,
     /// -- transaction receipt --
-    #[prost(string, tag="29")]
-    pub blob_gas_price: ::prost::alloc::string::String,
+    ///
+    /// UInt256
+    #[prost(bytes="vec", tag="29")]
+    pub blob_gas_price_bytes: ::prost::alloc::vec::Vec<u8>,
     #[prost(uint64, tag="30")]
     pub blob_gas_used: u64,
     #[prost(uint64, tag="31")]
@@ -268,32 +274,36 @@ pub struct Trace {
     pub call_type_code: u32,
     #[prost(string, tag="18")]
     pub address: ::prost::alloc::string::String,
-    #[prost(string, tag="19")]
-    pub value: ::prost::alloc::string::String,
-    #[prost(uint64, tag="20")]
-    pub gas_limit: u64,
+    /// UInt256
+    #[prost(bytes="vec", tag="19")]
+    pub value_bytes: ::prost::alloc::vec::Vec<u8>,
+    /// UInt256 - FOR TESTING PURPOSES
+    #[prost(string, tag="20")]
+    pub value_hex: ::prost::alloc::string::String,
     #[prost(uint64, tag="21")]
+    pub gas_limit: u64,
+    #[prost(uint64, tag="22")]
     pub gas_consumed: u64,
     /// Return data is set by contract calls using RETURN or REVERT.
-    #[prost(string, tag="22")]
-    pub return_data: ::prost::alloc::string::String,
     #[prost(string, tag="23")]
+    pub return_data: ::prost::alloc::string::String,
+    #[prost(string, tag="24")]
     pub input: ::prost::alloc::string::String,
-    #[prost(bool, tag="24")]
+    #[prost(bool, tag="25")]
     pub suicide: bool,
-    #[prost(string, tag="25")]
+    #[prost(string, tag="26")]
     pub failure_reason: ::prost::alloc::string::String,
-    #[prost(bool, tag="26")]
-    pub state_reverted: bool,
     #[prost(bool, tag="27")]
-    pub status_reverted: bool,
+    pub state_reverted: bool,
     #[prost(bool, tag="28")]
-    pub status_failed: bool,
+    pub status_reverted: bool,
     #[prost(bool, tag="29")]
+    pub status_failed: bool,
+    #[prost(bool, tag="30")]
     pub executed_code: bool,
-    #[prost(uint64, tag="30")]
-    pub begin_ordinal: u64,
     #[prost(uint64, tag="31")]
+    pub begin_ordinal: u64,
+    #[prost(uint64, tag="32")]
     pub end_ordinal: u64,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
@@ -314,12 +324,12 @@ pub struct BalanceChange {
     /// -- balance change --
     #[prost(string, tag="6")]
     pub address: ::prost::alloc::string::String,
-    #[prost(string, tag="7")]
-    pub new_balance: ::prost::alloc::string::String,
-    #[prost(string, tag="8")]
-    pub old_balance: ::prost::alloc::string::String,
-    #[prost(string, tag="9")]
-    pub amount: ::prost::alloc::string::String,
+    /// UInt256
+    #[prost(bytes="vec", tag="7")]
+    pub new_balance_bytes: ::prost::alloc::vec::Vec<u8>,
+    /// UInt256
+    #[prost(bytes="vec", tag="8")]
+    pub old_balance_bytes: ::prost::alloc::vec::Vec<u8>,
     #[prost(uint64, tag="10")]
     pub ordinal: u64,
     #[prost(string, tag="11")]

blocks/evm/src/traces.rs

@@ -1,7 +1,4 @@
-use common::{
-    structs::BlockTimestamp,
-    utils::{bytes_to_hex, optional_bigint_to_string},
-};
+use common::{structs::BlockTimestamp, utils::bytes_to_hex};
 use substreams_ethereum::pb::eth::v2::{Block, Call, TransactionTrace};
 
 use crate::{
@@ -67,7 +64,8 @@ pub fn parse_traces(call: &Call, transaction: &TransactionTrace, timestamp: &Blo
         call_type: call_types_to_string(call.call_type),
         call_type_code: call.call_type as u32,
         address: bytes_to_hex(&call.address),
-        value: optional_bigint_to_string(&call.value, "0"),
+        value_bytes: call.value.clone().unwrap_or_default().bytes,
+        value_hex: bytes_to_hex(&call.value.clone().unwrap_or_default().bytes),
         gas_limit: call.gas_limit,
         gas_consumed: call.gas_consumed,
         input: bytes_to_hex(&call.input),

blocks/evm/src/transactions.rs

@@ -1,9 +1,8 @@
 use common::structs::BlockTimestamp;
 use common::utils::bytes_to_hex;
-use common::utils::optional_bigint_to_string;
-use substreams_ethereum::pb::eth::v2::Block;
 
 use crate::pb::pinax::evm::v1::Transaction;
+use substreams_ethereum::pb::eth::v2::Block;
 
 pub fn transaction_type_to_string(r#type: i32) -> String {
     match r#type {
@@ -47,7 +46,7 @@ pub fn collect_transactions(block: &Block, timestamp: &BlockTimestamp) -> Vec<Tr
             let receipt = transaction.receipt.clone().unwrap();
             Transaction {
                 // block
-                block_time: Some(timestamp.time),
+                // block_time: Some(timestamp.time),
                 block_number: timestamp.number,
                 block_hash: timestamp.hash.clone(),
                 block_date: timestamp.date.clone(),
@@ -65,21 +64,21 @@ pub fn collect_transactions(block: &Block, timestamp: &BlockTimestamp) -> Vec<Tr
                 status: transaction_status_to_string(transaction.status),
                 status_code: transaction.status as u32,
                 success: is_transaction_success(transaction.status),
-                gas_price: optional_bigint_to_string(&transaction.gas_price, "0"),
+                gas_price_bytes: transaction.gas_price.clone().unwrap_or_default().bytes,
                 gas_limit: transaction.gas_limit,
-                value: optional_bigint_to_string(&transaction.value, "0"),
+                value_bytes: transaction.value.clone().unwrap_or_default().bytes,
                 data: bytes_to_hex(&transaction.input),
                 v: bytes_to_hex(&transaction.v),
                 r: bytes_to_hex(&transaction.r),
                 s: bytes_to_hex(&transaction.s),
                 gas_used: transaction.gas_used,
                 r#type: transaction_type_to_string(transaction.r#type),
                 type_code: transaction.r#type as u32,
-                max_fee_per_gas: optional_bigint_to_string(&transaction.max_fee_per_gas, "0"),
-                max_priority_fee_per_gas: optional_bigint_to_string(&transaction.max_priority_fee_per_gas, "0"),
+                max_fee_per_gas_bytes: transaction.max_fee_per_gas.clone().unwrap_or_default().bytes,
+                max_priority_fee_per_gas_bytes: transaction.max_priority_fee_per_gas.clone().unwrap_or_default().bytes,
                 begin_ordinal: transaction.begin_ordinal,
                 end_ordinal: transaction.end_ordinal,
-                blob_gas_price: optional_bigint_to_string(&receipt.blob_gas_price, "0"),
+                blob_gas_price_bytes: receipt.blob_gas_price.clone().unwrap_or_default().bytes,
                 blob_gas_used: receipt.blob_gas_used(),
                 cumulative_gas_used: receipt.cumulative_gas_used,
                 logs_bloom: bytes_to_hex(&receipt.logs_bloom),

blocks/evm/substreams.yaml

@@ -13,7 +13,6 @@ binaries:
 protobuf:
   files:
     - evm.proto
-    - parquet/options.proto
   importPaths:
     - ../../proto
   excludePaths:

blocks/evm/test.py

@@ -0,0 +1,13 @@
+import chdb
+
+result = chdb.sql("""
+SELECT
+    value_hex,
+    hex(value_bytes),
+    reinterpretAsUInt256(reverse(value_bytes)),
+    reinterpretAsUInt256(reverse(unhex(substring(value_hex,3))))
+FROM './out/traces/0020500000-0020500001.parquet'
+WHERE value_bytes != ''
+LIMIT 10;
+""")
+print(result)

common/src/utils.rs

@@ -29,6 +29,35 @@ pub fn add_prefix_to_hex(hex: &str) -> String {
     }
 }
 
+// pub fn optional_bigint_to_string(value: &Option<BigInt>, default: &str) -> String {
+//     match value {
+//         Some(bigint) => {
+//             let mut bytes = bigint.bytes.clone();
+
+//             // Determine if the number is negative by checking the sign bit
+//             let is_negative = bytes.first().map_or(false, |b| b & 0x80 != 0);
+
+//             // Pad or truncate to 32 bytes
+//             if bytes.len() < 32 {
+//                 let mut padding = if is_negative {
+//                     vec![0xFF; 32 - bytes.len()] // Pad with 0xFF for negative numbers
+//                 } else {
+//                     vec![0x00; 32 - bytes.len()] // Pad with 0x00 for positive numbers
+//                 };
+//                 padding.extend_from_slice(&bytes);
+//                 bytes = padding;
+//             } else if bytes.len() > 32 {
+//                 // Truncate higher-order bytes (keep the last 32 bytes)
+//                 bytes = bytes[bytes.len() - 32..].to_vec();
+//             }
+
+//             // Convert bytes to hex string
+//             bytes.iter().map(|b| format!("{:02x}", b)).collect()
+//         }
+//         None => default.to_string(),
+//     }
+// }
+
 // pub fn bytes_to_hex_no_prefix(bytes: &Vec<u8>) -> String {
 //     if bytes.is_empty() {
 //         return "".to_string();
@@ -79,10 +108,10 @@ pub fn optional_bigint_to_decimal(value: Option<BigInt>) -> BigDecimal {
     }
 }
 
-pub fn optional_bigint_to_hex(value: Option<BigInt>) -> String {
+pub fn optional_bigint_to_hex(value: &Option<BigInt>, default: &str) -> String {
     match value {
         Some(bigint) => bytes_to_hex(&bigint.bytes),
-        None => "".to_string(),
+        None => default.to_string(),
     }
 }