Merge pull request #7 from FortnoxAB/metrics

Added metrics and webserver

go.mod

@@ -3,22 +3,57 @@ module github.com/fortnoxab/renovator
 go 1.23.2
 
 require (
+	github.com/fortnoxab/ginprometheus v0.0.0-20211026110220-d3da4ce1dc2b
+	github.com/gin-contrib/pprof v1.5.1
+	github.com/gin-gonic/gin v1.10.0
+	github.com/jonaz/ginlogrus v0.0.0-20191118094232-2f4da50f5dd6
+	github.com/prometheus/client_golang v1.11.0
 	github.com/redis/go-redis/v9 v9.7.0
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/sirupsen/logrus v1.9.3
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.9.0
 	github.com/urfave/cli/v2 v2.27.5
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bytedance/sonic v1.12.4 // indirect
+	github.com/bytedance/sonic/loader v0.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cloudwego/base64x v0.1.4 // indirect
+	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/gabriel-vasile/mimetype v1.4.6 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.22.1 // indirect
+	github.com/goccy/go-json v0.10.3 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/stretchr/objx v0.1.0 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.12 // indirect
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
-	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
+	golang.org/x/arch v0.12.0 // indirect
+	golang.org/x/crypto v0.29.0 // indirect
+	golang.org/x/net v0.31.0 // indirect
+	golang.org/x/sys v0.27.0 // indirect
+	golang.org/x/text v0.20.0 // indirect
+	google.golang.org/protobuf v1.35.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

main.go

@@ -78,6 +78,11 @@ func app() *cli.App {
 					Name:  "run-first-time",
 					Usage: "run discovery directly, only applicable if a schedule is provided",
 				},
+				&cli.StringFlag{
+					Name:  "port",
+					Usage: "webserver port for pprof and metrics",
+					Value: "8080",
+				},
 			},
 		},
 		{
@@ -98,6 +103,11 @@ func app() *cli.App {
 					Value: 1,
 					Usage: "Defines the maximum amount of simultaneous renovate processes",
 				},
+				&cli.StringFlag{
+					Name:  "port",
+					Usage: "webserver port for pprof and metrics",
+					Value: "8080",
+				},
 			},
 		},
 	}

pkg/agent/agent.go

@@ -9,15 +9,27 @@ import (
 	"github.com/fortnoxab/renovator/pkg/command"
 	"github.com/fortnoxab/renovator/pkg/master"
 	"github.com/fortnoxab/renovator/pkg/renovate"
+	"github.com/fortnoxab/renovator/pkg/webserver"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/redis/go-redis/v9"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
 
+var renovateRuns = prometheus.NewCounterVec(prometheus.CounterOpts{
+	Name: "renovate_runs",
+	Help: "Number of renovate runs",
+}, []string{"result", "repo"})
+
+func init() {
+	prometheus.MustRegister(renovateRuns)
+}
+
 type Agent struct {
 	Renovator       *renovate.Runner
 	RedisClient     redis.Cmdable
 	MaxProcessCount int
+	Webserver       *webserver.Webserver
 }
 
 func NewAgentFromContext(cCtx *cli.Context) (*Agent, error) {
@@ -30,6 +42,7 @@ func NewAgentFromContext(cCtx *cli.Context) (*Agent, error) {
 		Renovator:       renovate.NewRunner(&command.Exec{}),
 		RedisClient:     rc,
 		MaxProcessCount: cCtx.Int("max-process-count"),
+		Webserver:       &webserver.Webserver{Port: cCtx.String("port"), EnableMetrics: true},
 	}, nil
 }
 
@@ -38,6 +51,13 @@ func (a *Agent) Run(ctx context.Context) {
 	reposToProcess := make(chan string)
 
 	wg := &sync.WaitGroup{}
+	if a.Webserver != nil {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			a.Webserver.Start(ctx)
+		}()
+	}
 
 	for i := 0; i < a.MaxProcessCount; i++ {
 		wg.Add(1)
@@ -52,20 +72,19 @@ func (a *Agent) Run(ctx context.Context) {
 					logrus.Infof("running renovate on repo: %s", repo)
 					err := a.Renovator.RunRenovate(repo)
 					if err != nil {
+						renovateRuns.WithLabelValues("error", repo).Inc()
 						logrus.Errorf("error renovating repo: %s err: %s", repo, err)
 						continue
 					}
+					renovateRuns.WithLabelValues("ok", repo).Inc()
 					logrus.Infof("finished renovating repo: %s", repo)
 				}
 
 			}
 		}()
 	}
 
-	for {
-		if ctx.Err() != nil {
-			break
-		}
+	for ctx.Err() == nil {
 		repos, err := a.RedisClient.BLPop(ctx, 0, master.RedisRepoListKey).Result() // 0 duration == block until key exists.
 		if err != nil {
 			logrus.Error("BLpop err: ", err)

pkg/agent/agent_test.go

@@ -35,23 +35,23 @@ func TestRun(t *testing.T) {
 		redisMockCall.ReturnArguments = mock.Arguments{redisMockList.LPop()}
 	}
 
-	commanderMock.On("Run", "renovate", "project1/repo1").
+	commanderMock.On("RunWithEnv", []string{}, "renovate", "project1/repo1").
 		Run(func(args mock.Arguments) {
 			time.Sleep(200 * time.Millisecond)
 		}).
-		Return("", "", 0, nil).
+		Return(nil).
 		Once()
-	commanderMock.On("Run", "renovate", "project1/repo2").
+	commanderMock.On("RunWithEnv", []string{}, "renovate", "project1/repo2").
 		Run(func(args mock.Arguments) {
 			time.Sleep(200 * time.Millisecond)
 		}).
-		Return("", "", 0, nil).
+		Return(nil).
 		Once()
-	commanderMock.On("Run", "renovate", "project2/repo1").
+	commanderMock.On("RunWithEnv", []string{}, "renovate", "project2/repo1").
 		Run(func(args mock.Arguments) {
 			time.Sleep(200 * time.Millisecond)
 		}).
-		Return("", "", 0, nil).
+		Return(nil).
 		Once()
 
 	ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)

pkg/command/run.go

@@ -3,33 +3,27 @@ package command
 import (
 	"os"
 	"os/exec"
-	"strings"
 )
 
 type Commander interface {
-	Run(head string, parts ...string) (stdOut, stdErr string, exitCode int, err error)
-	RunWithEnv(env []string, head string, parts ...string) (stdOut, stdErr string, exitCode int, err error)
+	Run(head string, parts ...string) error
+	RunWithEnv(env []string, head string, parts ...string) error
 }
 
 type Exec struct {
 }
 
-func (e *Exec) Run(head string, parts ...string) (stdOut, stdErr string, exitCode int, err error) {
+func (e *Exec) Run(head string, parts ...string) (err error) {
 	return e.RunWithEnv(nil, head, parts...)
 }
 
-func (e *Exec) RunWithEnv(env []string, head string, parts ...string) (stdOut, stdErr string, exitCode int, err error) {
+func (e *Exec) RunWithEnv(env []string, head string, parts ...string) (err error) {
 	cmd := exec.Command(head, parts...) // #nosec
-	outBuf := &strings.Builder{}
-	errBuf := &strings.Builder{}
-	cmd.Stdout = outBuf
-	cmd.Stderr = errBuf
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
 	cmd.Env = os.Environ()
 	cmd.Env = append(cmd.Env, env...)
 
 	err = cmd.Run()
-	exitCode = cmd.ProcessState.ExitCode()
-	stdOut = strings.TrimSpace(outBuf.String())
-	stdErr = strings.TrimSpace(errBuf.String())
 	return
 }