Merge pull request #10 from MariusVanDerWijden/strats

all: split out strategies, add config files

README.md

@@ -2,9 +2,14 @@
 
 A framework to fuzz Ethereum Virtual Machine implementations.
 
+FuzzyVM uses two different processes. 
+One process generates test cases and the other one executes them on different EVM implementations.
+The traces of the EVM's are collected and compared against each other.
+If a crasher is found, 
+
 ## Environment
-You need to have golang < 1.15 installed as go version 1.15 includes some 
-changes that break go-fuzz.
+You might need to have golang < 1.15 installed as go version 1.15 includes some 
+changes that might break go-fuzz.
 
 ## Install instructions
 
@@ -21,3 +26,14 @@ go build
 ./FuzzyVM
 ```
 
+If the fuzzer breaks (or you shut it down) it might leave test cases laying around.
+You can execute the test cases with `./FuzzyVM --exec`.
+
+It makes sense to create an initial corpus in order to improve the efficiency of the fuzzer.
+You can generate corpus elements with `./FuzzyVM --corpus N`.
+
+## Config
+
+FuzzyVM has to be configured to know which EVMs it should use.
+You can specify the paths to the EVMs in a config file.
+An example for the config file can be found in `config.toml`.

benchmark/linear.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/MariusVanDerWijden/FuzzyVM/executor"
 	"github.com/MariusVanDerWijden/FuzzyVM/generator"
+	"github.com/holiman/goevmlab/evms"
 )
 
 // testGeneration generates N programs.
@@ -32,13 +33,14 @@ func verify(N int) (time.Duration, error) {
 		return time.Nanosecond, err
 	}
 	name = fmt.Sprintf("%v/%v.json", outDir, name)
-	out, err := executor.ExecuteTest(name)
+	exc := newExecutor()
+	out, err := exc.ExecuteTest(name)
 	if err != nil {
 		return time.Nanosecond, err
 	}
 	start := time.Now()
 	for i := 0; i < N; i++ {
-		if !executor.Verify(name, out) {
+		if !exc.Verify(name, out) {
 			return time.Nanosecond, fmt.Errorf("Verification failed: %v", name)
 		}
 	}
@@ -56,10 +58,10 @@ func execution(N int) (time.Duration, error) {
 		return time.Nanosecond, err
 	}
 	name = fmt.Sprintf("%v.json", name)
-	executor.PrintTrace = false
+	exc := newExecutor()
 	start := time.Now()
 	for i := 0; i < N; i++ {
-		executor.ExecuteFullTest(outDir, crashers, name, false)
+		exc.ExecuteFullTest(outDir, crashers, name, false)
 	}
 	return time.Since(start), nil
 }
@@ -87,10 +89,15 @@ func execMultiple(N int, threadlimit int) (time.Duration, error) {
 		}
 		names = append(names, fmt.Sprintf("%v.json", name))
 	}
-	executor.PrintTrace = false
+	exc := newExecutor()
 	start := time.Now()
-	if err := executor.Execute(outDir, crashers, threadlimit); err != nil {
+	if err := exc.Execute(outDir, crashers, threadlimit); err != nil {
 		return time.Nanosecond, err
 	}
 	return time.Since(start), nil
 }
+
+func newExecutor() *executor.Executor {
+	// TODO add meaningful vms for benchmarks
+	return executor.NewExecutor([]evms.Evm{}, false)
+}

config.go

@@ -0,0 +1,94 @@
+// Copyright 2020 Marius van der Wijden
+// This file is part of the fuzzy-vm library.
+//
+// The fuzzy-vm library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The fuzzy-vm library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the fuzzy-vm library. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"os"
+	"reflect"
+	"unicode"
+
+	"github.com/holiman/goevmlab/evms"
+	"github.com/naoina/toml"
+)
+
+func getVMsFromConfig(file string) ([]evms.Evm, error) {
+	conf, err := loadConfig(file)
+	if err != nil {
+		return []evms.Evm{}, err
+	}
+	var vms []evms.Evm
+	for _, s := range conf.Geth {
+		vms = append(vms, evms.NewGethEVM(s))
+	}
+	for _, s := range conf.Nethermind {
+		vms = append(vms, evms.NewNethermindVM(s))
+	}
+	for _, s := range conf.Besu {
+		vms = append(vms, evms.NewBesuVM(s))
+	}
+	for _, s := range conf.OpenEthereum {
+		vms = append(vms, evms.NewParityVM(s))
+	}
+	for _, s := range conf.Aleth {
+		vms = append(vms, evms.NewAlethVM(s))
+	}
+	return vms, nil
+}
+
+type config struct {
+	Geth         []string
+	Besu         []string
+	OpenEthereum []string
+	Nethermind   []string
+	Aleth        []string
+}
+
+func loadConfig(file string) (*config, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	conf := &config{}
+	err = tomlSettings.NewDecoder(bufio.NewReader(f)).Decode(conf)
+	// Add file name to errors that have a line number.
+	if _, ok := err.(*toml.LineError); ok {
+		err = errors.New(file + ", " + err.Error())
+	}
+	return conf, err
+}
+
+// These settings ensure that TOML keys use the same names as Go struct fields.
+var tomlSettings = toml.Config{
+	NormFieldName: func(rt reflect.Type, key string) string {
+		return key
+	},
+	FieldToKey: func(rt reflect.Type, field string) string {
+		return field
+	},
+	MissingField: func(rt reflect.Type, field string) error {
+		link := ""
+		if unicode.IsUpper(rune(rt.Name()[0])) && rt.PkgPath() != "main" {
+			link = fmt.Sprintf(", see https://godoc.org/%s#%s for available fields", rt.PkgPath(), rt.Name())
+		}
+		return fmt.Errorf("field '%s' is not defined in %s%s", field, rt.String(), link)
+	},
+}

config.toml

@@ -0,0 +1,5 @@
+Geth = ["/home/matematik/ethereum/FuzzyVM/vms/geth-evm", "/home/matematik/ethereum/FuzzyVM/vms/turbogeth-evm"]
+OpenEthereum = ["/home/matematik/ethereum/FuzzyVM/vms/openethereum-evm"]
+Nethermind = ["/home/matematik/ethereum/neth_test/nethermind/src/Nethermind/Nethermind.State.Test.Runner/bin/Release/netcoreapp3.1/nethtest"]
+Besu = ["/home/matematik/ethereum/besu/ethereum/evmtool/build/install/evmtool/bin/evm"]
+Aleth = []

executor/executor.go

@@ -31,19 +31,22 @@ import (
 	"github.com/pkg/errors"
 )
 
-var (
-	vms = []evms.Evm{
-		evms.NewGethEVM("/home/matematik/ethereum/FuzzyVM/vms/geth-evm"),
-		evms.NewParityVM("/home/matematik/ethereum/FuzzyVM/vms/openethereum-evm"),
-		evms.NewNethermindVM("/home/matematik/ethereum/neth_test/nethermind/src/Nethermind/Nethermind.State.Test.Runner/bin/Release/netcoreapp3.1/nethtest"),
-		evms.NewBesuVM("/home/matematik/ethereum/besu/ethereum/evmtool/build/install/evmtool/bin/evm"),
-		evms.NewGethEVM("/home/matematik/ethereum/FuzzyVM/vms/turbogeth-evm"),
+var PrintTrace = true
+
+type Executor struct {
+	Vms        []evms.Evm
+	PrintTrace bool
+}
+
+func NewExecutor(vms []evms.Evm, printTrace bool) *Executor {
+	return &Executor{
+		Vms:        vms,
+		PrintTrace: printTrace,
 	}
-	PrintTrace = true
-)
+}
 
 // Execute runs all tests in `dirName` and saves crashers in `outDir`
-func Execute(dirName, outDir string, threadlimit int) error {
+func (e *Executor) Execute(dirName, outDir string, threadlimit int) error {
 	infos, err := ioutil.ReadDir(dirName)
 	if err != nil {
 		return err
@@ -59,7 +62,7 @@ func Execute(dirName, outDir string, threadlimit int) error {
 			meter.Mark(1)
 			name := info.Name()
 			job := func() {
-				if err := ExecuteFullTest(dirName, outDir, name, true); err != nil {
+				if err := e.ExecuteFullTest(dirName, outDir, name, true); err != nil {
 					err := errors.Wrap(err, fmt.Sprintf("in file: %v", name))
 					fmt.Println(err)
 					//errChan <- err
@@ -81,25 +84,25 @@ func Execute(dirName, outDir string, threadlimit int) error {
 }
 
 // ExecuteFullTest executes a single test.
-func ExecuteFullTest(dirName, outDir, filename string, doPurge bool) error {
+func (e *Executor) ExecuteFullTest(dirName, outDir, filename string, doPurge bool) error {
 	var (
 		testFile  = fmt.Sprintf("%v/%v", dirName, filename)
 		testName  = strings.TrimRight(filename, ".json")
 		traceFile = fmt.Sprintf("%v/%v-trace.jsonl", dirName, testName)
 		outputs   [][]byte
 		err       error
 	)
-	outputs, err = ExecuteTest(testFile)
+	outputs, err = e.ExecuteTest(testFile)
 	if err != nil {
 		return err
 	}
-	return verifyAndPurge(traceFile, testName, outDir, testFile, outputs, doPurge)
+	return e.verifyAndPurge(traceFile, testName, outDir, testFile, outputs, doPurge)
 }
 
-func verifyAndPurge(traceFile, testName, outDir, testFile string, outputs [][]byte, doPurge bool) error {
-	if !Verify(traceFile, outputs) {
+func (e *Executor) verifyAndPurge(traceFile, testName, outDir, testFile string, outputs [][]byte, doPurge bool) error {
+	if !e.Verify(traceFile, outputs) {
 		fmt.Printf("Test %v failed, dumping\n", testName)
-		if err := dump(testName, outDir, vms, outputs); err != nil {
+		if err := dump(testName, outDir, e.Vms, outputs); err != nil {
 			return err
 		}
 	} else {
@@ -109,17 +112,17 @@ func verifyAndPurge(traceFile, testName, outDir, testFile string, outputs [][]by
 				fmt.Printf("Purging failed: %v\n", err)
 			}
 		} else if PrintTrace {
-			printOutputs(outputs)
+			e.printOutputs(outputs)
 		}
 	}
 	return nil
 }
 
 // ExecuteTest executes a state test.
-func ExecuteTest(testName string) ([][]byte, error) {
+func (e *Executor) ExecuteTest(testName string) ([][]byte, error) {
 	var buf [][]byte
 	var buffer bytes.Buffer
-	for _, vm := range vms {
+	for _, vm := range e.Vms {
 		buffer.Reset()
 		if _, err := vm.RunStateTest(testName, &buffer, false); err != nil {
 			return nil, err
@@ -130,7 +133,7 @@ func ExecuteTest(testName string) ([][]byte, error) {
 }
 
 // Verify checks if the traces match the default trace.
-func Verify(traceName string, outputs [][]byte) bool {
+func (e *Executor) Verify(traceName string, outputs [][]byte) bool {
 	var ioReaders []io.Reader
 	for _, out := range outputs {
 		ioReaders = append(ioReaders, bytes.NewReader(out))
@@ -143,7 +146,7 @@ func Verify(traceName string, outputs [][]byte) bool {
 		}
 		ioReaders = append(ioReaders, bytes.NewBuffer(ref))
 	*/
-	return evms.CompareFiles(vms, ioReaders)
+	return evms.CompareFiles(e.Vms, ioReaders)
 }
 
 // dump writes outputs to a file in case of a verification problem
@@ -168,11 +171,11 @@ func purge(filename, tracename string) error {
 }
 
 // printOutputs prints out the produced traces
-func printOutputs(outputs [][]byte) {
+func (e *Executor) printOutputs(outputs [][]byte) {
 	fmt.Println("TRACES:")
 	fmt.Println("--------------")
 	for i, out := range outputs {
-		fmt.Printf("%v: \n", vms[i].Name())
+		fmt.Printf("%v: \n", e.Vms[i].Name())
 		fmt.Print(string(out))
 		fmt.Println("--------------")
 	}

flags.go

@@ -64,4 +64,9 @@ var (
 		Name:  "corpus",
 		Usage: "Number of corpus elements that should be created",
 	}
+	configFileFlag = cli.StringFlag{
+		Name:  "config",
+		Usage: "Path to the config file required to run FuzzyVM",
+		Value: "config.toml",
+	}
 )