feat(proposer): introduce zlib for transactions list bytes compression (#649)

Author: davidtaikocha

driver/chain_syncer/calldata/syncer.go

@@ -22,6 +22,7 @@ import (
 	"github.com/taikoxyz/taiko-client/driver/state"
 	txlistfetcher "github.com/taikoxyz/taiko-client/driver/txlist_fetcher"
 	"github.com/taikoxyz/taiko-client/internal/metrics"
+	"github.com/taikoxyz/taiko-client/internal/utils"
 	eventIterator "github.com/taikoxyz/taiko-client/pkg/chain_iterator/event_iterator"
 	"github.com/taikoxyz/taiko-client/pkg/rpc"
 	txListValidator "github.com/taikoxyz/taiko-client/pkg/txlist_validator"
@@ -247,6 +248,10 @@ func (s *Syncer) onBlockProposed(
 		}
 	}
 
+	if txListBytes, err = utils.Decompress(txListBytes); err != nil {
+		return fmt.Errorf("failed to decompress tx list bytes: %w", err)
+	}
+
 	// If the transactions list is invalid, we simply insert an empty L2 block.
 	if !s.txListValidator.ValidateTxList(event.BlockId, txListBytes, event.Meta.BlobUsed) {
 		log.Info("Invalid transactions list, insert an empty L2 block instead", "blockID", event.BlockId)

internal/utils/util_test.go

@@ -0,0 +1,22 @@
+package utils_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/taikoxyz/taiko-client/internal/testutils"
+	"github.com/taikoxyz/taiko-client/internal/utils"
+)
+
+func TestEncodeDecodeBytes(t *testing.T) {
+	b := testutils.RandomBytes(1024)
+
+	compressed, err := utils.Compress(b)
+	require.Nil(t, err)
+	require.NotEmpty(t, compressed)
+
+	decompressed, err := utils.Decompress(compressed)
+	require.Nil(t, err)
+
+	require.Equal(t, b, decompressed)
+}

internal/utils/utils.go

@@ -1,8 +1,12 @@
 package utils
 
 import (
+	"bytes"
+	"compress/zlib"
 	"crypto/rand"
+	"errors"
 	"fmt"
+	"io"
 	"math/big"
 	"os"
 	"strings"
@@ -14,22 +18,22 @@ import (
 	"golang.org/x/exp/constraints"
 )
 
+// LoadEnv loads all the test environment variables.
 func LoadEnv() {
-	// load test environment variables.
 	currentPath, err := os.Getwd()
 	if err != nil {
-		log.Debug("get current path failed", "err", err)
+		log.Debug("Failed to get current path", "err", err)
 	}
 	path := strings.Split(currentPath, "/taiko-client")
 	if len(path) == 0 {
-		log.Debug("not a taiko-client repo")
+		log.Debug("Not a taiko-client repo")
 	}
-	err = godotenv.Load(fmt.Sprintf("%s/taiko-client/integration_test/.env", path[0]))
-	if err != nil {
-		log.Debug("failed to load test env", "current path", currentPath, "err", err)
+	if godotenv.Load(fmt.Sprintf("%s/taiko-client/integration_test/.env", path[0])) != nil {
+		log.Debug("Failed to load test env", "current path", currentPath, "err", err)
 	}
 }
 
+// RandUint64 returns a random uint64 number.
 func RandUint64(max *big.Int) uint64 {
 	if max == nil {
 		max = new(big.Int)
@@ -40,6 +44,7 @@ func RandUint64(max *big.Int) uint64 {
 	return num.Uint64()
 }
 
+// RandUint32 returns a random uint32 number.
 func RandUint32(max *big.Int) uint32 {
 	if max == nil {
 		max = new(big.Int)
@@ -69,3 +74,38 @@ func Max[T constraints.Integer](a, b T) T {
 	}
 	return b
 }
+
+// Compress compresses the given txList bytes using zlib.
+func Compress(txList []byte) ([]byte, error) {
+	var b bytes.Buffer
+	w := zlib.NewWriter(&b)
+	defer w.Close()
+
+	if _, err := w.Write(txList); err != nil {
+		return nil, err
+	}
+
+	if err := w.Flush(); err != nil {
+		return nil, err
+	}
+
+	return b.Bytes(), nil
+}
+
+// Decompress decompresses the given txList bytes using zlib.
+func Decompress(compressedTxList []byte) ([]byte, error) {
+	r, err := zlib.NewReader(bytes.NewBuffer(compressedTxList))
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+
+	b, err := io.ReadAll(r)
+	if err != nil {
+		if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
+			return nil, err
+		}
+	}
+
+	return b, nil
+}

proposer/proposer.go

@@ -19,6 +19,7 @@ import (
 	"github.com/taikoxyz/taiko-client/bindings"
 	"github.com/taikoxyz/taiko-client/bindings/encoding"
 	"github.com/taikoxyz/taiko-client/internal/metrics"
+	"github.com/taikoxyz/taiko-client/internal/utils"
 	"github.com/taikoxyz/taiko-client/pkg/rpc"
 	"github.com/taikoxyz/taiko-client/pkg/sender"
 	selector "github.com/taikoxyz/taiko-client/proposer/prover_selector"
@@ -319,12 +320,17 @@ func (p *Proposer) ProposeTxList(
 	txListBytes []byte,
 	txNum uint,
 ) error {
+	compressedTxListBytes, err := utils.Compress(txListBytes)
+	if err != nil {
+		return err
+	}
+
 	tx, err := p.txBuilder.Build(
 		ctx,
 		p.tierFees,
 		p.sender.GetOpts(p.ctx),
 		p.IncludeParentMetaHash,
-		txListBytes,
+		compressedTxListBytes,
 	)
 	if err != nil {
 		log.Warn("Failed to build TaikoL1.proposeBlock transaction", "error", encoding.TryParsingCustomError(err))