Make larger parts of the analysis work

Author: EdSchouten

.bazelignore

@@ -0,0 +1 @@
+starlark

MODULE.bazel

@@ -58,6 +58,7 @@ use_repo(
     "com_github_bluekeyes_go_gitdiff",
     "com_github_buildbarn_go_cdc",
     "com_github_google_uuid",
+    "com_github_kballard_go_shellquote",
     "com_github_prometheus_client_golang",
     "com_github_secure_io_siv_go",
     "com_github_seehuhn_mt19937",
@@ -93,8 +94,10 @@ go_deps_dev.module_override(
 )
 go_deps_dev.module_override(
     patches = [
+        "//:patches/net_starlark_go/thread.diff",
         "//:patches/net_starlark_go/dict-get-pop.diff",
         "//:patches/net_starlark_go/function-saving.diff",
+        "//:patches/net_starlark_go/getattr-error-checking.diff",
         "//:patches/net_starlark_go/octal.diff",
         "//:patches/net_starlark_go/unwrap.diff",
     ],

cmd/playground_storage_frontend/main.go

@@ -49,7 +49,6 @@ func main() {
 			return status.Error(codes.InvalidArgument, "No maximum unfinalized parents limit provided")
 		}
 		maximumUnfinalizedParentsLimit := object.NewLimit(configuration.MaximumUnfinalizedParentsLimit)
-		objectStoreSemaphore := semaphore.NewWeighted(configuration.ObjectStoreConcurrency)
 
 		// Construct object and tag stores for mirrored replicas.
 		objectStoreA, tagStoreA, err := createShardsForReplica(grpcClientFactory, configuration.GrpcClientsShardsReplicaA)
@@ -65,7 +64,7 @@ func main() {
 		objectDownloader := object_mirrored.NewMirroredDownloader(objectStoreA, objectStoreB)
 		objectUploader := object_leaserenewing.NewLeaseRenewingUploader(
 			object_mirrored.NewMirroredUploader(objectStoreA, objectStoreB),
-			objectStoreSemaphore,
+			semaphore.NewWeighted(configuration.ObjectStoreConcurrency),
 			maximumUnfinalizedParentsLimit,
 		)
 		dependenciesGroup.Go(func(ctx context.Context, siblingsGroup, dependenciesGroup program.Group) error {
@@ -101,7 +100,7 @@ func main() {
 					s,
 					dag.NewUploaderServer(
 						objectUploader,
-						objectStoreSemaphore,
+						semaphore.NewWeighted(configuration.ObjectStoreConcurrency),
 						tagUpdater,
 						configuration.MaximumUnfinalizedDagsCount,
 						maximumUnfinalizedParentsLimit,

deployments/demo/playground_scheduler.jsonnet

@@ -12,7 +12,7 @@ local statePath = std.extVar('STATE_PATH');
   actionRouter: {
     simple: {
       initialSizeClassAnalyzer: {
-        maximumExecutionTimeout: '7200s',
+        maximumExecutionTimeout: '86400s',
       },
     },
   },

deployments/demo/playground_storage_frontend.jsonnet

@@ -2,6 +2,14 @@ local replicasCount = 4;
 local statePath = std.extVar('STATE_PATH');
 
 {
+  global: { diagnosticsHttpServer: {
+    httpServers: [{
+      listenAddresses: [':9981'],
+      authenticationPolicy: { allow: {} },
+    }],
+    enablePrometheus: true,
+    enablePprof: true,
+  } },
   grpcServers: [{
     listenPaths: [statePath + '/playground_storage_frontend.sock'],
     authenticationPolicy: { allow: {} },

go.mod

@@ -14,12 +14,13 @@ require (
 	github.com/buildbarn/bb-storage v0.0.0-20241220082316-ff8c80c6eb4c
 	github.com/buildbarn/go-cdc v0.0.0-20240326143813-ab4a540e41c6
 	github.com/google/uuid v1.6.0
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/prometheus/client_golang v1.20.5
 	github.com/secure-io/siv-go v0.0.0-20180922214919-5ff40651e2c4
 	github.com/seehuhn/mt19937 v1.0.0
 	github.com/stretchr/testify v1.10.0
 	github.com/ulikunitz/xz v0.5.12
-	go.starlark.net v0.0.0-20241226192728-8dfa5b98479f
+	go.starlark.net v0.0.0-20250205221240-492d3672b3f4
 	go.uber.org/mock v0.5.0
 	golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8
 	golang.org/x/sync v0.10.0

go.sum

@@ -165,6 +165,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
@@ -252,8 +254,8 @@ go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qq
 go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck=
 go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4=
 go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4=
-go.starlark.net v0.0.0-20241226192728-8dfa5b98479f h1:Zs/py28HDFATSDzPcfIzrBFjVsV7HzDEGNNVZIGsjm0=
-go.starlark.net v0.0.0-20241226192728-8dfa5b98479f/go.mod h1:YKMCv9b1WrfWmeqdV5MAuEHWsu5iC+fe6kYl2sQjdI8=
+go.starlark.net v0.0.0-20250205221240-492d3672b3f4 h1:eBP+boBfJoGU3irqbxGTcTlKcbNwJCOdbmsnDq56nak=
+go.starlark.net v0.0.0-20250205221240-492d3672b3f4/go.mod h1:YKMCv9b1WrfWmeqdV5MAuEHWsu5iC+fe6kYl2sQjdI8=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
 go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=

patches/net_starlark_go/dict-get-pop.diff

@@ -1,10 +1,10 @@
 diff --git starlark/library.go starlark/library.go
-index 4e73a40..988ece2 100644
+index f03d552..eaf7159 100644
 --- starlark/library.go
 +++ starlark/library.go
-@@ -1203,7 +1203,11 @@ func zip(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error)
+@@ -1205,7 +1205,11 @@ func zip(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error)
  // https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·get
- func dict_get(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) {
+ func dict_get(thread *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) {
  	var key, dflt Value
 -	if err := UnpackPositionalArgs(b.Name(), args, kwargs, 1, &key, &dflt); err != nil {
 +	if err := UnpackArgs(
@@ -14,10 +14,10 @@ index 4e73a40..988ece2 100644
 +	); err != nil {
  		return nil, err
  	}
- 	if v, ok, err := b.Receiver().(*Dict).Get(key); err != nil {
-@@ -1248,7 +1252,11 @@ func dict_keys(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error)
+ 	if v, ok, err := b.Receiver().(*Dict).Get(thread, key); err != nil {
+@@ -1250,7 +1254,11 @@ func dict_keys(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error)
  // https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·pop
- func dict_pop(_ *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) {
+ func dict_pop(thread *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) {
  	var k, d Value
 -	if err := UnpackPositionalArgs(b.Name(), args, kwargs, 1, &k, &d); err != nil {
 +	if err := UnpackArgs(
@@ -27,4 +27,4 @@ index 4e73a40..988ece2 100644
 +	); err != nil {
  		return nil, err
  	}
- 	if v, found, err := b.Receiver().(*Dict).Delete(k); err != nil {
+ 	if v, found, err := b.Receiver().(*Dict).Delete(thread, k); err != nil {

patches/net_starlark_go/function-saving.diff

@@ -1,5 +1,5 @@
 diff --git internal/compile/compile.go internal/compile/compile.go
-index b257d70..c171dc6 100644
+index bc38200..fe7e58e 100644
 --- internal/compile/compile.go
 +++ internal/compile/compile.go
 @@ -29,6 +29,7 @@ import (
@@ -18,31 +18,31 @@ index b257d70..c171dc6 100644
 
  	// -- transient state --
 
-@@ -364,7 +366,6 @@ type pcomp struct {
+@@ -366,7 +368,6 @@ type pcomp struct {
 
  	names     map[string]uint32
  	constants map[interface{}]uint32
 -	functions map[*Funcode]uint32
  }
 
  // An fcomp holds the compiler state for a Funcode.
-@@ -503,7 +504,6 @@ func File(opts *syntax.FileOptions, stmts []syntax.Stmt, pos syntax.Position, na
+@@ -523,7 +524,6 @@ func File(opts *syntax.FileOptions, stmts []syntax.Stmt, pos syntax.Position, na
  		},
  		names:     make(map[string]uint32),
  		constants: make(map[interface{}]uint32),
 -		functions: make(map[*Funcode]uint32),
  	}
  	pcomp.prog.Toplevel = pcomp.function(name, pos, stmts, locals, nil)
 
-@@ -521,6 +521,7 @@ func (pcomp *pcomp) function(name string, pos syntax.Position, stmts []syntax.St
+@@ -541,6 +541,7 @@ func (pcomp *pcomp) function(name string, pos syntax.Position, stmts []syntax.St
  			Doc:      docStringFromBody(stmts),
  			Locals:   bindings(locals),
  			FreeVars: bindings(freevars),
 +			Index:    math.MaxUint32,
  		},
  	}
 
-@@ -980,13 +981,11 @@ func (pcomp *pcomp) constantIndex(v interface{}) uint32 {
+@@ -1001,13 +1002,11 @@ func (pcomp *pcomp) constantIndex(v interface{}) uint32 {
  // functionIndex returns the index of the specified function
  // AST the nestedfun pool, adding it if necessary.
  func (pcomp *pcomp) functionIndex(fn *Funcode) uint32 {
@@ -60,7 +60,7 @@ index b257d70..c171dc6 100644
 
  // string emits code to push the specified string.
 diff --git internal/compile/serial.go internal/compile/serial.go
-index 0dbae47..af003d7 100644
+index 6131322..0b27769 100644
 --- internal/compile/serial.go
 +++ internal/compile/serial.go
 @@ -267,10 +267,10 @@ func DecodeProgram(data []byte) (_ *Program, err error) {
@@ -85,15 +85,16 @@ index 0dbae47..af003d7 100644
  	id := d.binding()
  	doc := d.string()
  	code := d.bytes()
-@@ -395,5 +395,6 @@ func (d *decoder) function() *Funcode {
+@@ -395,6 +395,7 @@ func (d *decoder) function() *Funcode {
  		NumKwonlyParams: numKwonlyParams,
  		HasVarargs:      hasVarargs,
  		HasKwargs:       hasKwargs,
 +		Index:           index,
  	}
- }
+ 	fn.finish()
+ 	return fn
 diff --git starlark/eval.go starlark/eval.go
-index be3eb61..fe2b3bc 100644
+index a4640e4..16b75bb 100644
 --- starlark/eval.go
 +++ starlark/eval.go
 @@ -5,6 +5,7 @@
@@ -174,10 +175,10 @@ index be3eb61..fe2b3bc 100644
  // and global environment. This is a variant of ExecFile specialized to
  // the needs of a REPL, in which a sequence of input chunks, each
 diff --git starlark/value.go starlark/value.go
-index 5e323ca..345b664 100644
+index d89a9f5..62d679f 100644
 --- starlark/value.go
 +++ starlark/value.go
-@@ -696,6 +696,25 @@ type Function struct {
+@@ -701,6 +701,25 @@ type Function struct {
  	freevars Tuple
  }
 

patches/net_starlark_go/getattr-error-checking.diff

@@ -0,0 +1,25 @@
+diff --git starlark/library.go starlark/library.go
+index f03d552..3e057e5 100644
+--- starlark/library.go
++++ starlark/library.go
+@@ -463,7 +463,8 @@ func getattr(thread *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, err
+ 		if err != nil {
+ 			// An error could mean the field doesn't exist,
+ 			// or it exists but could not be computed.
+-			if dflt != nil {
++			var errNoSuchAttr NoSuchAttrError
++			if errors.As(err, &errNoSuchAttr) && dflt != nil {
+ 				return dflt, nil
+ 			}
+ 			return nil, nameErr(b, err)
+@@ -491,6 +492,10 @@ func hasattr(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, err
+ 		if err == nil {
+ 			return Bool(v != nil), nil
+ 		}
++		var errNoSuchAttr NoSuchAttrError
++		if !errors.As(err, &errNoSuchAttr) {
++			return nil, err
++		}
+ 
+ 		// An error does not conclusively indicate presence or
+ 		// absence of a field: it could occur while computing