Skip to content

Commit

Permalink
Cut down usage of dag.ExistingObjectContentsWalker
Browse files Browse the repository at this point in the history 
Use TMetadata everywhere, and only convert to dag.ObjectContentsWalker
at the point where we actually return the results of the function. This
will be necessary if we want to flush data to storage asynchronously.
  • Loading branch information
@EdSchouten
EdSchouten committed Mar 12, 2025
1 parent fcd16c8 commit 83004a9
Show file tree
Hide file tree
Showing 27 changed files with 262 additions and 308 deletions.
7 changes: 2 additions & 5 deletions pkg/evaluation/generate_computer/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ func (fd functionDefinition) getKeyType(functionName string, isPatched bool) str

func (fd functionDefinition) keyToPatchedMessage() string {
if fd.KeyContainsReferences {
return "model_core.PatchedMessage[proto.Message, dag.ObjectContentsWalker]{Message: key.Message, Patcher: key.Patcher}"
return "model_core.NewPatchedMessage[proto.Message](key.Message, key.Patcher)"
} else {
return "model_core.NewSimplePatchedMessage[dag.ObjectContentsWalker, proto.Message](key)"
}
Expand Down Expand Up @@ -203,10 +203,7 @@ func main() {
if functionDefinition.NativeValueType == nil {
fmt.Printf("\tcase *pb.%s_Key:\n", functionName)
fmt.Printf("\t\tm, err := c.base.Compute%sValue(ctx, %s, &typedE)\n", functionName, functionDefinition.typedKeyToArgument(functionName))
fmt.Printf("\t\treturn model_core.PatchedMessage[proto.Message, dag.ObjectContentsWalker]{\n")
fmt.Printf("\t\t\tMessage: m.Message,\n")
fmt.Printf("\t\t\tPatcher: m.Patcher,\n")
fmt.Printf("\t\t}, err\n")
fmt.Printf("\t\treturn model_core.NewPatchedMessage[proto.Message](m.Message, m.Patcher), err\n")
}
}
fmt.Printf("\tdefault:\n")
Expand Down
44 changes: 18 additions & 26 deletions pkg/model/analysis/base_computer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -367,29 +367,25 @@ func (c *baseComputer[TReference, TMetadata]) ComputeBuildResultValue(ctx contex

var iterErr error
for canonicalTargetLabel := range c.expandCanonicalTargetPattern(ctx, e, canonicalTargetPattern, &iterErr) {
visibleTargetPatcher := model_core.NewReferenceMessagePatcher[dag.ObjectContentsWalker]()
visibleTargetValue := e.GetVisibleTargetValue(
model_core.PatchedMessage[*model_analysis_pb.VisibleTarget_Key, dag.ObjectContentsWalker]{
Message: &model_analysis_pb.VisibleTarget_Key{
model_core.NewSimplePatchedMessage[dag.ObjectContentsWalker](
&model_analysis_pb.VisibleTarget_Key{
FromPackage: canonicalTargetLabel.GetCanonicalPackage().String(),
ToLabel: canonicalTargetLabel.String(),
},
Patcher: visibleTargetPatcher,
},
),
)
if !visibleTargetValue.IsSet() {
missingDependencies = true
continue
}

targetCompletionPatcher := model_core.NewReferenceMessagePatcher[dag.ObjectContentsWalker]()
targetCompletionValue := e.GetTargetCompletionValue(
model_core.PatchedMessage[*model_analysis_pb.TargetCompletion_Key, dag.ObjectContentsWalker]{
Message: &model_analysis_pb.TargetCompletion_Key{
model_core.NewSimplePatchedMessage[dag.ObjectContentsWalker](
&model_analysis_pb.TargetCompletion_Key{
Label: visibleTargetValue.Message.Label,
},
Patcher: targetCompletionPatcher,
},
),
)
if !targetCompletionValue.IsSet() {
missingDependencies = true
Expand All @@ -414,18 +410,16 @@ func (c *baseComputer[TReference, TMetadata]) ComputeBuildSpecificationValue(ctx
return PatchedBuildSpecificationValue{}, err
}

patchedBuildSpecification := model_core.NewPatchedMessageFromExisting(
patchedBuildSpecification := model_core.NewPatchedMessageFromExistingCaptured(
c.objectCapturer,
buildSpecification,
func(index int) dag.ObjectContentsWalker {
return dag.ExistingObjectContentsWalker
},
)
return PatchedBuildSpecificationValue{
Message: &model_analysis_pb.BuildSpecification_Value{
return model_core.NewPatchedMessage(
&model_analysis_pb.BuildSpecification_Value{
BuildSpecification: patchedBuildSpecification.Message,
},
Patcher: patchedBuildSpecification.Patcher,
}, nil
model_core.MapReferenceMetadataToWalkers(patchedBuildSpecification.Patcher),
), nil
}

func (c *baseComputer[TReference, TMetadata]) ComputeBuiltinsModuleNamesValue(ctx context.Context, key *model_analysis_pb.BuiltinsModuleNames_Key, e BuiltinsModuleNamesEnvironment[TReference]) (PatchedBuiltinsModuleNamesValue, error) {
Expand Down Expand Up @@ -503,20 +497,18 @@ func (c *baseComputer[TReference, TMetadata]) ComputeRepoDefaultAttrsValue(ctx c
}

func (c *baseComputer[TReference, TMetadata]) ComputeTargetCompletionValue(ctx context.Context, key model_core.Message[*model_analysis_pb.TargetCompletion_Key, TReference], e TargetCompletionEnvironment[TReference]) (PatchedTargetCompletionValue, error) {
configurationReference := model_core.NewPatchedMessageFromExisting(
configurationReference := model_core.NewPatchedMessageFromExistingCaptured(
c.objectCapturer,
model_core.NewNestedMessage(key, key.Message.ConfigurationReference),
func(index int) dag.ObjectContentsWalker {
return dag.ExistingObjectContentsWalker
},
)
configuredTarget := e.GetConfiguredTargetValue(
model_core.PatchedMessage[*model_analysis_pb.ConfiguredTarget_Key, dag.ObjectContentsWalker]{
Message: &model_analysis_pb.ConfiguredTarget_Key{
model_core.NewPatchedMessage(
&model_analysis_pb.ConfiguredTarget_Key{
Label: key.Message.Label,
ConfigurationReference: configurationReference.Message,
},
Patcher: configurationReference.Patcher,
},
model_core.MapReferenceMetadataToWalkers(configurationReference.Patcher),
),
)
if !configuredTarget.IsSet() {
return PatchedTargetCompletionValue{}, evaluation.ErrMissingDependency
Expand Down
2 changes: 1 addition & 1 deletion pkg/model/analysis/compatible_toolchains_for_type.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ func (c *baseComputer[TReference, TMetadata]) ComputeCompatibleToolchainsForType
platformInfoProvider, err := getProviderFromConfiguredTarget(
e,
platformLabel,
model_core.NewSimplePatchedMessage[dag.ObjectContentsWalker, *model_core_pb.Reference](nil),
model_core.NewSimplePatchedMessage[model_core.WalkableReferenceMetadata, *model_core_pb.Reference](nil),
platformInfoProviderIdentifier,
)
if err != nil {
Expand Down
23 changes: 10 additions & 13 deletions pkg/model/analysis/compiled_bzl_file.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
model_filesystem "github.com/buildbarn/bonanza/pkg/model/filesystem"
model_starlark "github.com/buildbarn/bonanza/pkg/model/starlark"
model_analysis_pb "github.com/buildbarn/bonanza/pkg/proto/model/analysis"
"github.com/buildbarn/bonanza/pkg/storage/dag"

"go.starlark.net/starlark"
"go.starlark.net/syntax"
Expand Down Expand Up @@ -89,12 +88,12 @@ func (c *baseComputer[TReference, TMetadata]) ComputeCompiledBzlFileValue(ctx co
if err != nil {
return PatchedCompiledBzlFileValue{}, err
}
return PatchedCompiledBzlFileValue{
Message: &model_analysis_pb.CompiledBzlFile_Value{
return model_core.NewPatchedMessage(
&model_analysis_pb.CompiledBzlFile_Value{
CompiledProgram: compiledProgram.Message,
},
Patcher: model_core.MapReferenceMetadataToWalkers(compiledProgram.Patcher),
}, nil
model_core.MapReferenceMetadataToWalkers(compiledProgram.Patcher),
), nil
}

func (c *baseComputer[TReference, TMetadata]) ComputeCompiledBzlFileDecodedGlobalsValue(ctx context.Context, key *model_analysis_pb.CompiledBzlFileDecodedGlobals_Key, e CompiledBzlFileDecodedGlobalsEnvironment[TReference]) (starlark.StringDict, error) {
Expand Down Expand Up @@ -183,18 +182,16 @@ func (c *baseComputer[TReference, TMetadata]) ComputeCompiledBzlFileGlobalValue(
return PatchedCompiledBzlFileGlobalValue{}, err
}

patchedGlobal := model_core.NewPatchedMessageFromExisting(
patchedGlobal := model_core.NewPatchedMessageFromExistingCaptured(
c.objectCapturer,
global,
func(index int) dag.ObjectContentsWalker {
return dag.ExistingObjectContentsWalker
},
)
return PatchedCompiledBzlFileGlobalValue{
Message: &model_analysis_pb.CompiledBzlFileGlobal_Value{
return model_core.NewPatchedMessage(
&model_analysis_pb.CompiledBzlFileGlobal_Value{
Global: patchedGlobal.Message,
},
Patcher: patchedGlobal.Patcher,
}, nil
model_core.MapReferenceMetadataToWalkers(patchedGlobal.Patcher),
), nil
}

var exportsBzlTargetName = label.MustNewTargetName("exports.bzl")
Expand Down
84 changes: 38 additions & 46 deletions pkg/model/analysis/configured_target.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ func (c *baseComputer[TReference, TMetadata]) constraintValuesToConstraints(ctx
constrainValueInfoProvider, err := getProviderFromConfiguredTarget(
e,
visibleTarget.Message.Label,
model_core.NewSimplePatchedMessage[dag.ObjectContentsWalker, *model_core_pb.Reference](nil),
model_core.NewSimplePatchedMessage[model_core.WalkableReferenceMetadata, *model_core_pb.Reference](nil),
constraintValueInfoProviderIdentifier,
)
if err != nil {
Expand Down Expand Up @@ -380,20 +380,18 @@ func (c *baseComputer[TReference, TMetadata]) ComputeConfiguredTargetValue(ctx c
// edge transition is specified, apply it.
configurationReference := model_core.NewNestedMessage(key, key.Message.ConfigurationReference)
if cfgTransitionIdentifier := ruleDefinition.Message.CfgTransitionIdentifier; cfgTransitionIdentifier != "" {
patchedConfigurationReference := model_core.NewPatchedMessageFromExisting(
patchedConfigurationReference := model_core.NewPatchedMessageFromExistingCaptured(
c.objectCapturer,
configurationReference,
func(index int) dag.ObjectContentsWalker {
return dag.ExistingObjectContentsWalker
},
)
incomingEdgeTransitionValue := e.GetUserDefinedTransitionValue(
model_core.PatchedMessage[*model_analysis_pb.UserDefinedTransition_Key, dag.ObjectContentsWalker]{
Message: &model_analysis_pb.UserDefinedTransition_Key{
model_core.NewPatchedMessage(
&model_analysis_pb.UserDefinedTransition_Key{
TransitionIdentifier: cfgTransitionIdentifier,
InputConfigurationReference: patchedConfigurationReference.Message,
},
Patcher: patchedConfigurationReference.Patcher,
},
model_core.MapReferenceMetadataToWalkers(patchedConfigurationReference.Patcher),
),
)
if !incomingEdgeTransitionValue.IsSet() {
return PatchedConfiguredTargetValue{}, evaluation.ErrMissingDependency
Expand Down Expand Up @@ -920,13 +918,13 @@ func (rc *ruleContext[TReference, TMetadata]) configureAttr(thread *starlark.Thr
// TODO: Should we cache this in the ruleContext?
configurationReference := rc.getPatchedConfigurationReference()
transitionValue := rc.environment.GetUserDefinedTransitionValue(
model_core.PatchedMessage[*model_analysis_pb.UserDefinedTransition_Key, dag.ObjectContentsWalker]{
Message: &model_analysis_pb.UserDefinedTransition_Key{
model_core.NewPatchedMessage(
&model_analysis_pb.UserDefinedTransition_Key{
TransitionIdentifier: tr.UserDefined,
InputConfigurationReference: configurationReference.Message,
},
Patcher: configurationReference.Patcher,
},
model_core.MapReferenceMetadataToWalkers(configurationReference.Patcher),
),
)
if !transitionValue.IsSet() {
return nil, evaluation.ErrMissingDependency
Expand Down Expand Up @@ -979,21 +977,19 @@ func (rc *ruleContext[TReference, TMetadata]) configureAttr(thread *starlark.Thr
if err != nil {
return nil, err
}
patchedConfigurationReference1 := model_core.NewPatchedMessageFromExisting(
patchedConfigurationReference1 := model_core.NewPatchedMessageFromExistingCaptured(
rc.computer.objectCapturer,
configurationReference,
func(index int) dag.ObjectContentsWalker {
return dag.ExistingObjectContentsWalker
},
)
resolvedLabelValue := rc.environment.GetVisibleTargetValue(
model_core.PatchedMessage[*model_analysis_pb.VisibleTarget_Key, dag.ObjectContentsWalker]{
Message: &model_analysis_pb.VisibleTarget_Key{
model_core.NewPatchedMessage(
&model_analysis_pb.VisibleTarget_Key{
FromPackage: visibilityFromPackage.String(),
ToLabel: canonicalLabel.String(),
ConfigurationReference: patchedConfigurationReference1.Message,
},
Patcher: patchedConfigurationReference1.Patcher,
},
model_core.MapReferenceMetadataToWalkers(patchedConfigurationReference1.Patcher),
),
)
if !resolvedLabelValue.IsSet() {
missingDependencies = true
Expand All @@ -1006,20 +1002,18 @@ func (rc *ruleContext[TReference, TMetadata]) configureAttr(thread *starlark.Thr
}

// Obtain the providers of the target.
patchedConfigurationReference2 := model_core.NewPatchedMessageFromExisting(
patchedConfigurationReference2 := model_core.NewPatchedMessageFromExistingCaptured(
rc.computer.objectCapturer,
configurationReference,
func(index int) dag.ObjectContentsWalker {
return dag.ExistingObjectContentsWalker
},
)
configuredTarget := rc.environment.GetConfiguredTargetValue(
model_core.PatchedMessage[*model_analysis_pb.ConfiguredTarget_Key, dag.ObjectContentsWalker]{
Message: &model_analysis_pb.ConfiguredTarget_Key{
model_core.NewPatchedMessage(
&model_analysis_pb.ConfiguredTarget_Key{
Label: resolvedLabelStr,
ConfigurationReference: patchedConfigurationReference2.Message,
},
Patcher: patchedConfigurationReference2.Patcher,
},
model_core.MapReferenceMetadataToWalkers(patchedConfigurationReference2.Patcher),
),
)
if !configuredTarget.IsSet() {
missingDependencies = true
Expand Down Expand Up @@ -1239,14 +1233,12 @@ func (rc *ruleContext[TReference, TMetadata]) getAttrValueParts(namedAttr *model
nil
}

func (rc *ruleContext[TReference, TMetadata]) getPatchedConfigurationReference() model_core.PatchedMessage[*model_core_pb.Reference, dag.ObjectContentsWalker] {
func (rc *ruleContext[TReference, TMetadata]) getPatchedConfigurationReference() model_core.PatchedMessage[*model_core_pb.Reference, TMetadata] {
// TODO: This function should likely not exist, as we need to
// take transitions into account.
return model_core.NewPatchedMessageFromExisting(
return model_core.NewPatchedMessageFromExistingCaptured(
rc.computer.objectCapturer,
rc.configurationReference,
func(index int) dag.ObjectContentsWalker {
return dag.ExistingObjectContentsWalker
},
)
}

Expand Down Expand Up @@ -1749,7 +1741,7 @@ func (rce *ruleContextExecutable[TReference, TMetadata]) Attr(thread *starlark.T
ToLabel: labelValue.Label,
ConfigurationReference: configurationReference.Message,
},
configurationReference.Patcher,
model_core.MapReferenceMetadataToWalkers(configurationReference.Patcher),
),
)
if !visibleTarget.IsSet() {
Expand Down Expand Up @@ -1876,7 +1868,7 @@ func (rcf *ruleContextFile[TReference, TMetadata]) Attr(thread *starlark.Thread,
ToLabel: labelValue.Label,
ConfigurationReference: configurationReference.Message,
},
configurationReference.Patcher,
model_core.MapReferenceMetadataToWalkers(configurationReference.Patcher),
),
)
if !visibleTarget.IsSet() {
Expand Down Expand Up @@ -2030,7 +2022,7 @@ func (rcf *ruleContextFiles[TReference, TMetadata]) Attr(thread *starlark.Thread
ToLabel: labelElement.Label,
ConfigurationReference: configurationReference.Message,
},
configurationReference.Patcher,
model_core.MapReferenceMetadataToWalkers(configurationReference.Patcher),
),
)
if !visibleTarget.IsSet() {
Expand Down Expand Up @@ -2316,14 +2308,14 @@ func (tc *toolchainContext[TReference, TMetadata]) Get(thread *starlark.Thread,
}
configurationReference := rc.getPatchedConfigurationReference()
resolvedToolchains := rc.environment.GetResolvedToolchainsValue(
model_core.PatchedMessage[*model_analysis_pb.ResolvedToolchains_Key, dag.ObjectContentsWalker]{
Message: &model_analysis_pb.ResolvedToolchains_Key{
model_core.NewPatchedMessage(
&model_analysis_pb.ResolvedToolchains_Key{
ExecCompatibleWith: execCompatibleWith,
ConfigurationReference: configurationReference.Message,
Toolchains: execGroupDefinition.Toolchains,
},
Patcher: configurationReference.Patcher,
},
model_core.MapReferenceMetadataToWalkers(configurationReference.Patcher),
),
)
if !resolvedToolchains.IsSet() {
return nil, true, evaluation.ErrMissingDependency
Expand Down Expand Up @@ -2389,15 +2381,15 @@ type getProviderFromConfiguredTargetEnvironment[TReference any] interface {

// getProviderFromConfiguredTarget looks up a single provider that is
// provided by a configured target
func getProviderFromConfiguredTarget[TReference any](e getProviderFromConfiguredTargetEnvironment[TReference], targetLabel string, configurationReference model_core.PatchedMessage[*model_core_pb.Reference, dag.ObjectContentsWalker], providerIdentifier label.CanonicalStarlarkIdentifier) (model_core.Message[*model_starlark_pb.Struct_Fields, TReference], error) {
func getProviderFromConfiguredTarget[TReference any, TMetadata model_core.WalkableReferenceMetadata](e getProviderFromConfiguredTargetEnvironment[TReference], targetLabel string, configurationReference model_core.PatchedMessage[*model_core_pb.Reference, TMetadata], providerIdentifier label.CanonicalStarlarkIdentifier) (model_core.Message[*model_starlark_pb.Struct_Fields, TReference], error) {
configuredTargetValue := e.GetConfiguredTargetValue(
model_core.PatchedMessage[*model_analysis_pb.ConfiguredTarget_Key, dag.ObjectContentsWalker]{
Message: &model_analysis_pb.ConfiguredTarget_Key{
model_core.NewPatchedMessage(
&model_analysis_pb.ConfiguredTarget_Key{
Label: targetLabel,
ConfigurationReference: configurationReference.Message,
},
Patcher: configurationReference.Patcher,
},
model_core.MapReferenceMetadataToWalkers(configurationReference.Patcher),
),
)
if !configuredTargetValue.IsSet() {
return model_core.Message[*model_starlark_pb.Struct_Fields, TReference]{}, evaluation.ErrMissingDependency
Expand Down
15 changes: 6 additions & 9 deletions pkg/model/analysis/file_properties_value.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import (
model_analysis_pb "github.com/buildbarn/bonanza/pkg/proto/model/analysis"
model_core_pb "github.com/buildbarn/bonanza/pkg/proto/model/core"
model_filesystem_pb "github.com/buildbarn/bonanza/pkg/proto/model/filesystem"
"github.com/buildbarn/bonanza/pkg/storage/dag"
"github.com/buildbarn/bonanza/pkg/storage/object"
)

Expand Down Expand Up @@ -219,16 +218,14 @@ func (c *baseComputer[TReference, TMetadata]) ComputeFilePropertiesValue(ctx con
return PatchedFilePropertiesValue{}, errors.New("path resolves to a directory")
}

patchedFileProperties := model_core.NewPatchedMessageFromExisting(
patchedFileProperties := model_core.NewPatchedMessageFromExistingCaptured(
c.objectCapturer,
resolver.fileProperties,
func(index int) dag.ObjectContentsWalker {
return dag.ExistingObjectContentsWalker
},
)
return PatchedFilePropertiesValue{
Message: &model_analysis_pb.FileProperties_Value{
return model_core.NewPatchedMessage(
&model_analysis_pb.FileProperties_Value{
Exists: patchedFileProperties.Message,
},
Patcher: patchedFileProperties.Patcher,
}, nil
model_core.MapReferenceMetadataToWalkers(patchedFileProperties.Patcher),
), nil
}
Loading

0 comments on commit 83004a9

Please sign in to comment.