zktrie part1: change storage proof from per step to per block (#102)

* change storage proof from step-wise to block-wise. Step-wise proof will be reconstructed in prover side

* minor

* minor

* Update worker.go

* purge hexInt

* Update l2trace.go

* Refactor l2witness/opt-storage-proof (#112)

* rename GetStateData to GetLiveStateObject

* revert EvmTxTraces type

* rename GetLiveStateObject to GetLiveStateAccount

* fix typo

* some renamings

* format codes

* fix typo

* fix typos

* format codes

some reverts

some renamings

some renamings

format codes

* update comments

update comments

* update comments

update comments

update comments

* update comments

update comments

update comments

* rename

* rename

* update

* update comments

Co-authored-by: maskpp <maskpp266@gmail.com>
Co-authored-by: Ho Vei <noelwei@gmail.com>
Co-authored-by: HAOYUatHZ <37070449+HAOYUatHZ@users.noreply.github.com>

Author: 4 people

.gitattributes

@@ -1,3 +1,4 @@
 # Auto detect text files and perform LF normalization
 * text=auto
 *.sol linguist-language=Solidity
+*.go text eol=lf

core/blockchain.go

@@ -1193,17 +1193,17 @@ func (bc *BlockChain) writeKnownBlock(block *types.Block) error {
 }
 
 // WriteBlockWithState writes the block and all associated state to the database.
-func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.Receipt, logs []*types.Log, evmTraces []*types.ExecutionResult, state *state.StateDB, emitHeadEvent bool) (status WriteStatus, err error) {
+func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.Receipt, logs []*types.Log, evmTraces []*types.ExecutionResult, storageTrace *types.StorageTrace, state *state.StateDB, emitHeadEvent bool) (status WriteStatus, err error) {
 	if !bc.chainmu.TryLock() {
 		return NonStatTy, errInsertionInterrupted
 	}
 	defer bc.chainmu.Unlock()
-	return bc.writeBlockWithState(block, receipts, logs, evmTraces, state, emitHeadEvent)
+	return bc.writeBlockWithState(block, receipts, logs, evmTraces, storageTrace, state, emitHeadEvent)
 }
 
 // writeBlockWithState writes the block and all associated state to the database,
 // but is expects the chain mutex to be held.
-func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.Receipt, logs []*types.Log, evmTraces []*types.ExecutionResult, state *state.StateDB, emitHeadEvent bool) (status WriteStatus, err error) {
+func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.Receipt, logs []*types.Log, evmTraces []*types.ExecutionResult, storageTrace *types.StorageTrace, state *state.StateDB, emitHeadEvent bool) (status WriteStatus, err error) {
 	if bc.insertStopped() {
 		return NonStatTy, errInsertionInterrupted
 	}
@@ -1327,7 +1327,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 	// Fill blockResult content
 	var blockResult *types.BlockResult
 	if evmTraces != nil {
-		blockResult = bc.writeBlockResult(state, block, evmTraces)
+		blockResult = bc.writeBlockResult(state, block, evmTraces, storageTrace)
 		bc.blockResultCache.Add(block.Hash(), blockResult)
 	}
 
@@ -1351,57 +1351,22 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 }
 
 // Fill blockResult content
-func (bc *BlockChain) writeBlockResult(state *state.StateDB, block *types.Block, evmTraces []*types.ExecutionResult) *types.BlockResult {
+func (bc *BlockChain) writeBlockResult(state *state.StateDB, block *types.Block, evmTraces []*types.ExecutionResult, storageTrace *types.StorageTrace) *types.BlockResult {
 	blockResult := &types.BlockResult{
 		ExecutionResults: evmTraces,
+		StorageTrace:     storageTrace,
 	}
-	coinbase := types.AccountProofWrapper{
+	coinbase := types.AccountWrapper{
 		Address:  block.Coinbase(),
 		Nonce:    state.GetNonce(block.Coinbase()),
 		Balance:  (*hexutil.Big)(state.GetBalance(block.Coinbase())),
 		CodeHash: state.GetCodeHash(block.Coinbase()),
 	}
-	// Get coinbase address's account proof.
-	proof, err := state.GetProof(block.Coinbase())
-	if err != nil {
-		log.Error("Failed to get proof", "blockNumber", block.NumberU64(), "address", block.Coinbase().String(), "err", err)
-	} else {
-		coinbase.Proof = make([]string, len(proof))
-		for i := range proof {
-			coinbase.Proof[i] = hexutil.Encode(proof[i])
-		}
-	}
 
 	blockResult.BlockTrace = types.NewTraceBlock(bc.chainConfig, block, &coinbase)
+	blockResult.StorageTrace.RootAfter = state.GetRootHash()
 	for i, tx := range block.Transactions() {
 		evmTrace := blockResult.ExecutionResults[i]
-
-		from := evmTrace.From.Address
-		// Get proof
-		proof, err := state.GetProof(from)
-		if err != nil {
-			log.Error("Failed to get proof", "blockNumber", block.NumberU64(), "address", from.String(), "err", err)
-		} else {
-			evmTrace.From.Proof = make([]string, len(proof))
-			for i := range proof {
-				evmTrace.From.Proof[i] = hexutil.Encode(proof[i])
-			}
-		}
-
-		if evmTrace.To != nil {
-			to := evmTrace.To.Address
-			// Get proof
-			proof, err = state.GetProof(to)
-			if err != nil {
-				log.Error("Failed to get proof", "blockNumber", block.NumberU64(), "address", to.String(), "err", err)
-			} else {
-				evmTrace.To.Proof = make([]string, len(proof))
-				for i := range proof {
-					evmTrace.To.Proof[i] = hexutil.Encode(proof[i])
-				}
-			}
-		}
-
 		// Contract is called
 		if len(tx.Data()) != 0 && tx.To() != nil {
 			evmTrace.ByteCode = hexutil.Encode(state.GetCode(*tx.To()))
@@ -1727,8 +1692,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er
 
 		// Write the block to the chain and get the status.
 		substart = time.Now()
-		// EvmTraces is nil is safe because l2geth's p2p server is stoped and the code will not execute there.
-		status, err := bc.writeBlockWithState(block, receipts, logs, nil, statedb, false)
+		// EvmTraces & StorageTrace being nil is safe because l2geth's p2p server is stoped and the code will not execute there.
+		status, err := bc.writeBlockWithState(block, receipts, logs, nil, nil, statedb, false)
 		atomic.StoreUint32(&followupInterrupt, 1)
 		if err != nil {
 			return it.index, err

core/state/statedb.go

@@ -325,6 +325,41 @@ func (s *StateDB) GetProofByHash(addrHash common.Hash) ([][]byte, error) {
 	return proof, err
 }
 
+func (s *StateDB) GetLiveStateAccount(addr common.Address) *types.StateAccount {
+	obj, ok := s.stateObjects[addr]
+	if !ok {
+		return nil
+	}
+	return &obj.data
+}
+
+func (s *StateDB) GetRootHash() common.Hash {
+	return s.trie.Hash()
+}
+
+// StorageTrieProof is not in Db interface and used explictily for reading proof in storage trie (not the dirty value)
+func (s *StateDB) GetStorageTrieProof(a common.Address, key common.Hash) ([][]byte, error) {
+	// try the trie in stateObject first, else we would create one
+	stateObject := s.getStateObject(a)
+	if stateObject == nil {
+		return nil, errors.New("storage trie for requested address does not exist")
+	}
+
+	trie := stateObject.trie
+	var err error
+	if trie == nil {
+		// use a new, temporary trie
+		trie, err = s.db.OpenStorageTrie(stateObject.addrHash, stateObject.data.Root)
+		if err != nil {
+			return nil, fmt.Errorf("can't create storage trie on root %s: %v ", stateObject.data.Root, err)
+		}
+	}
+
+	var proof proofList
+	err = trie.Prove(crypto.Keccak256(key.Bytes()), 0, &proof)
+	return proof, err
+}
+
 // GetStorageProof returns the Merkle proof for given storage slot.
 func (s *StateDB) GetStorageProof(a common.Address, key common.Hash) ([][]byte, error) {
 	var proof proofList

core/types/l2trace.go

@@ -23,21 +23,44 @@ var (
 // BlockResult contains block execution traces and results required for rollers.
 type BlockResult struct {
 	BlockTrace       *BlockTrace        `json:"blockTrace"`
+	StorageTrace     *StorageTrace      `json:"storageTrace"`
 	ExecutionResults []*ExecutionResult `json:"executionResults"`
 }
 
+// StorageTrace stores proofs of storage needed by storage circuit
+type StorageTrace struct {
+	// Root hash before block execution:
+	RootBefore common.Hash `json:"rootBefore,omitempty"`
+	// Root hash after block execution, is nil if execution has failed
+	RootAfter common.Hash `json:"rootAfter,omitempty"`
+
+	// All proofs BEFORE execution, for accounts which would be used in tracing
+	Proofs map[string][]hexutil.Bytes `json:"proofs"`
+
+	// All storage proofs BEFORE execution
+	StorageProofs map[string]map[string][]hexutil.Bytes `json:"storageProofs,omitempty"`
+}
+
 // ExecutionResult groups all structured logs emitted by the EVM
 // while replaying a transaction in debug mode as well as transaction
 // execution status, the amount of gas used and the return value
 type ExecutionResult struct {
 	Gas         uint64 `json:"gas"`
 	Failed      bool   `json:"failed"`
 	ReturnValue string `json:"returnValue,omitempty"`
-	// Sender's account proof.
-	From *AccountProofWrapper `json:"from,omitempty"`
-	// Receiver's account proof.
-	To *AccountProofWrapper `json:"to,omitempty"`
-	// It's exist only when tx is a contract call.
+	// Sender's account state (before Tx)
+	From *AccountWrapper `json:"from,omitempty"`
+	// Receiver's account state (before Tx)
+	To *AccountWrapper `json:"to,omitempty"`
+	// AccountCreated record the account if the tx is "create"
+	// (for creating inside a contract, we just handle CREATE op)
+	AccountCreated *AccountWrapper `json:"accountCreated,omitempty"`
+
+	// Record all accounts' state which would be affected AFTER tx executed
+	// currently they are just `from` and `to` account
+	AccountsAfter []*AccountWrapper `json:"accountAfter"`
+
+	// `CodeHash` only exists when tx is a contract call.
 	CodeHash *common.Hash `json:"codeHash,omitempty"`
 	// If it is a contract call, the contract code is returned.
 	ByteCode   string          `json:"byteCode,omitempty"`
@@ -77,30 +100,35 @@ func NewStructLogResBasic(pc uint64, op string, gas, gasCost uint64, depth int,
 }
 
 type ExtraData struct {
+	// Indicate the call succeeds or not for CALL/CREATE op
+	CallFailed bool `json:"callFailed,omitempty"`
 	// CALL | CALLCODE | DELEGATECALL | STATICCALL: [tx.to address’s code, stack.nth_last(1) address’s code]
 	CodeList [][]byte `json:"codeList,omitempty"`
 	// SSTORE | SLOAD: [storageProof]
-	// SELFDESTRUCT: [contract address’s accountProof, stack.nth_last(0) address’s accountProof]
-	// SELFBALANCE: [contract address’s accountProof]
-	// BALANCE | EXTCODEHASH: [stack.nth_last(0) address’s accountProof]
-	// CREATE | CREATE2: [sender's accountProof, created contract address’s accountProof]
-	// CALL | CALLCODE: [caller contract address’s accountProof, stack.nth_last(1) address’s accountProof]
-	ProofList []*AccountProofWrapper `json:"proofList,omitempty"`
+	// SELFDESTRUCT: [contract address’s account, stack.nth_last(0) address’s account]
+	// SELFBALANCE: [contract address’s account]
+	// BALANCE | EXTCODEHASH: [stack.nth_last(0) address’s account]
+	// CREATE | CREATE2: [created contract address’s account (before constructed),
+	// 					  created contract address's account (after constructed)]
+	// CALL | CALLCODE: [caller contract address’s account,
+	// 					stack.nth_last(1) (i.e. callee) address’s account,
+	//					callee contract address's account (value updated, before called)]
+	// STATICCALL: [stack.nth_last(1) (i.e. callee) address’s account,
+	//					  callee contract address's account (before called)]
+	StateList []*AccountWrapper `json:"proofList,omitempty"`
 }
 
-type AccountProofWrapper struct {
-	Address  common.Address       `json:"address"`
-	Nonce    uint64               `json:"nonce"`
-	Balance  *hexutil.Big         `json:"balance"`
-	CodeHash common.Hash          `json:"codeHash,omitempty"`
-	Proof    []string             `json:"proof,omitempty"`
-	Storage  *StorageProofWrapper `json:"storage,omitempty"` // StorageProofWrapper can be empty if irrelated to storage operation
+type AccountWrapper struct {
+	Address  common.Address  `json:"address"`
+	Nonce    uint64          `json:"nonce"`
+	Balance  *hexutil.Big    `json:"balance"`
+	CodeHash common.Hash     `json:"codeHash,omitempty"`
+	Storage  *StorageWrapper `json:"storage,omitempty"` // StorageWrapper can be empty if irrelated to storage operation
 }
 
 // while key & value can also be retrieved from StructLogRes.Storage,
 // we still stored in here for roller's processing convenience.
-type StorageProofWrapper struct {
-	Key   string   `json:"key,omitempty"`
-	Value string   `json:"value,omitempty"`
-	Proof []string `json:"proof,omitempty"`
+type StorageWrapper struct {
+	Key   string `json:"key,omitempty"`
+	Value string `json:"value,omitempty"`
 }

core/types/l2trace_block.go

@@ -9,14 +9,14 @@ import (
 )
 
 type BlockTrace struct {
-	Number       *hexutil.Big         `json:"number"`
-	Hash         common.Hash          `json:"hash"`
-	GasLimit     uint64               `json:"gasLimit"`
-	Difficulty   *hexutil.Big         `json:"difficulty"`
-	BaseFee      *hexutil.Big         `json:"baseFee"`
-	Coinbase     *AccountProofWrapper `json:"coinbase"`
-	Time         uint64               `json:"time"`
-	Transactions []*TransactionTrace  `json:"transactions"`
+	Number       *hexutil.Big        `json:"number"`
+	Hash         common.Hash         `json:"hash"`
+	GasLimit     uint64              `json:"gasLimit"`
+	Difficulty   *hexutil.Big        `json:"difficulty"`
+	BaseFee      *hexutil.Big        `json:"baseFee"`
+	Coinbase     *AccountWrapper     `json:"coinbase"`
+	Time         uint64              `json:"time"`
+	Transactions []*TransactionTrace `json:"transactions"`
 }
 
 type TransactionTrace struct {
@@ -36,7 +36,7 @@ type TransactionTrace struct {
 }
 
 // NewTraceBlock supports necessary fields for roller.
-func NewTraceBlock(config *params.ChainConfig, block *Block, coinbase *AccountProofWrapper) *BlockTrace {
+func NewTraceBlock(config *params.ChainConfig, block *Block, coinbase *AccountWrapper) *BlockTrace {
 	txs := make([]*TransactionTrace, block.Transactions().Len())
 	for i, tx := range block.Transactions() {
 		txs[i] = newTraceTransaction(tx, block.NumberU64(), config)

core/vm/interface.go

@@ -47,6 +47,8 @@ type StateDB interface {
 	GetState(common.Address, common.Hash) common.Hash
 	SetState(common.Address, common.Hash, common.Hash)
 
+	GetRootHash() common.Hash
+	GetLiveStateAccount(addr common.Address) *types.StateAccount
 	GetProof(addr common.Address) ([][]byte, error)
 	GetProofByHash(addrHash common.Hash) ([][]byte, error)
 	GetStorageProof(a common.Address, key common.Hash) ([][]byte, error)