From 31c90e7ddc5636994cb4151fb6d7fe3f08830192 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Thu, 6 Feb 2025 23:53:11 +0000 Subject: [PATCH 01/26] add model garden + claude models --- go/plugins/internal/gemini/gemini.go | 1 - go/plugins/vertexai/modelgarden/anthropic.go | 51 ++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 go/plugins/vertexai/modelgarden/anthropic.go diff --git a/go/plugins/internal/gemini/gemini.go b/go/plugins/internal/gemini/gemini.go index bfec6c6714..8780a60c1d 100644 --- a/go/plugins/internal/gemini/gemini.go +++ b/go/plugins/internal/gemini/gemini.go @@ -1,7 +1,6 @@ // Copyright 2024 Google LLC // SPDX-License-Identifier: Apache-2.0 - // Package gemini contains code that is common to both the googleai and vertexai plugins. // Most most cannot be shared in this way because the import paths are different. package gemini diff --git a/go/plugins/vertexai/modelgarden/anthropic.go b/go/plugins/vertexai/modelgarden/anthropic.go new file mode 100644 index 0000000000..24bf90b2ad --- /dev/null +++ b/go/plugins/vertexai/modelgarden/anthropic.go @@ -0,0 +1,51 @@ +// Copyright 2025 Google LLC +// SPDX-License-Identifier: Apache-2.0 + +package modelgarden + +import ( + "sync" + + "cloud.google.com/go/vertexai/genai" + "github.com/firebase/genkit/go/ai" + "github.com/firebase/genkit/go/plugins/internal/gemini" +) + +const ( + provider = "vertexai" + labelPrefix = "Claude" +) + +var state struct { + gclient, pclient *genai.Client + mu sync.Mutex + initted bool +} + +var supportedModels = map[string]ai.ModelInfo{ + "claude-3-5-sonnet-v2": { + Label: "Vertex AI Model Garden - Claude 3.5 Sonnet", + Supports: &gemini.Multimodal, + Versions: []string{"claude-3-5-sonnet-v2@20241022"}, + }, + "claude-3-5-sonnet": { + Label: "Vertex AI Model Garden - Claude 3.5 Sonnet", + Supports: &gemini.Multimodal, + Versions: []string{"claude-3-5-sonnet@20240620"}, + }, + "claude-3-sonnet": { + Label: "Vertex AI Model Garden - Claude 3 Sonnet", + Supports: &gemini.Multimodal, + Versions: []string{"claude-3-sonnet@20240229"}, + }, + "claude-3-haiku": { + Label: "Vertex AI Model Garden - Claude 3 Haiku", + Supports: &gemini.Multimodal, + Versions: []string{"claude-3-haiku@20240307"}, + }, + "claude-3-opus": { + Label: "Vertex AI Model Garden - Claude 3 Opus", + Supports: &gemini.Multimodal, + Versions: []string{"claude-3-opus@20240229"}, + }, +} From 09089e1fef3efe7e5b8323385e0a4534d6c3ae5f Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Sat, 8 Feb 2025 05:30:32 +0000 Subject: [PATCH 02/26] add client management and plugin arch --- go/plugins/vertexai/modelgarden/anthropic.go | 33 +++++---- .../vertexai/modelgarden/anthropic_test.go | 30 ++++++++ go/plugins/vertexai/modelgarden/client.go | 73 +++++++++++++++++++ .../vertexai/modelgarden/modelgarden.go | 42 +++++++++++ 4 files changed, 164 insertions(+), 14 deletions(-) create mode 100644 go/plugins/vertexai/modelgarden/anthropic_test.go create mode 100644 go/plugins/vertexai/modelgarden/client.go create mode 100644 go/plugins/vertexai/modelgarden/modelgarden.go diff --git a/go/plugins/vertexai/modelgarden/anthropic.go b/go/plugins/vertexai/modelgarden/anthropic.go index 24bf90b2ad..7c1a242c8b 100644 --- a/go/plugins/vertexai/modelgarden/anthropic.go +++ b/go/plugins/vertexai/modelgarden/anthropic.go @@ -4,25 +4,13 @@ package modelgarden import ( - "sync" + "log" - "cloud.google.com/go/vertexai/genai" "github.com/firebase/genkit/go/ai" "github.com/firebase/genkit/go/plugins/internal/gemini" ) -const ( - provider = "vertexai" - labelPrefix = "Claude" -) - -var state struct { - gclient, pclient *genai.Client - mu sync.Mutex - initted bool -} - -var supportedModels = map[string]ai.ModelInfo{ +var AnthropicModels = map[string]ai.ModelInfo{ "claude-3-5-sonnet-v2": { Label: "Vertex AI Model Garden - Claude 3.5 Sonnet", Supports: &gemini.Multimodal, @@ -49,3 +37,20 @@ var supportedModels = map[string]ai.ModelInfo{ Versions: []string{"claude-3-opus@20240229"}, }, } + +type AnthropicCloudClient struct { + region string + project string +} + +var AnthropicClient = func(region string, project string) (Client, error) { + return &AnthropicCloudClient{ + region: region, + project: project, + }, nil +} + +func (a *AnthropicCloudClient) DefineModel(name string, info *ai.ModelInfo) error { + log.Printf("created an anthropic model: %s, versions: %#v", name, info.Versions) + return nil +} diff --git a/go/plugins/vertexai/modelgarden/anthropic_test.go b/go/plugins/vertexai/modelgarden/anthropic_test.go new file mode 100644 index 0000000000..227d25db91 --- /dev/null +++ b/go/plugins/vertexai/modelgarden/anthropic_test.go @@ -0,0 +1,30 @@ +// Copyright 2025 Google LLC +// SPDX-License-Identifier: Apache-2.0 + +package modelgarden_test + +import ( + "testing" + + "github.com/firebase/genkit/go/ai" + "github.com/firebase/genkit/go/plugins/vertexai/modelgarden" +) + +// keep track of all clients +var clients = modelgarden.NewClientFactory() + +func TestAnthropicClient(t *testing.T) { + anthropicClient, err := clients.CreateClient(&modelgarden.ClientConfig{ + Region: "us-west-1", + Provider: "anthropic", + Project: "project-123", + }) + if err != nil { + t.Fatalf("unable to create anthropic client") + } + + err = anthropicClient.DefineModel("foo_model", &ai.ModelInfo{}) + if err != nil { + t.Fatalf("unable to define model: %v", err) + } +} diff --git a/go/plugins/vertexai/modelgarden/client.go b/go/plugins/vertexai/modelgarden/client.go new file mode 100644 index 0000000000..6fccd707b3 --- /dev/null +++ b/go/plugins/vertexai/modelgarden/client.go @@ -0,0 +1,73 @@ +// Copyright 2025 Google LLC +// SPDX-License-Identifier: Apache-2.0 + +package modelgarden + +import ( + "errors" + "fmt" + "sync" + + "github.com/firebase/genkit/go/ai" +) + +// Generic Client interface for supported provider clients +type Client interface { + DefineModel(name string, info *ai.ModelInfo) error +} + +type ClientFactory struct { + clients map[string]Client // cache for provider clients + mu sync.Mutex +} + +func NewClientFactory() *ClientFactory { + return &ClientFactory{ + clients: make(map[string]Client), + } +} + +const ( + Anthropic string = "anthropic" +) + +// Function type for creating clients from supported providers +type ClientCreator func(region string, project string) (Client, error) + +// Basic client configuration +type ClientConfig struct { + Creator ClientCreator + Provider string + Project string + Region string +} + +func (f *ClientFactory) CreateClient(config *ClientConfig) (Client, error) { + if config == nil { + return nil, errors.New("empty client config") + } + + f.mu.Lock() + defer f.mu.Unlock() + + // every client will be identified by its provider-region combination + key := fmt.Sprintf("%s-%s", config.Provider, config.Region) + if client, ok := f.clients[key]; ok { + return client, nil // return from cache + } + + var client Client + var err error + + switch config.Provider { + case Anthropic: + client, err = AnthropicClient(config.Region, config.Project) + if err != nil { + return nil, err + } + } + + f.clients[key] = client + + return client, nil +} diff --git a/go/plugins/vertexai/modelgarden/modelgarden.go b/go/plugins/vertexai/modelgarden/modelgarden.go new file mode 100644 index 0000000000..b329b09d0c --- /dev/null +++ b/go/plugins/vertexai/modelgarden/modelgarden.go @@ -0,0 +1,42 @@ +// Copyright 2025 Google LLC +// SPDX-License-Identifier: Apache-2.0 + +package modelgarden + +import ( + "context" + "fmt" + + "github.com/firebase/genkit/go/genkit" +) + +type ModelGardenOptions struct { + ProjectID string + Region string + Models []string +} + +// Init initializes the ModelGarden plugin +// After calling Init, you may call [DefineModel] to create and register +// any additional generative models +func Init(ctx context.Context, g *genkit.Genkit, cfg *ModelGardenOptions) error { + clients := NewClientFactory() + for _, m := range cfg.Models { + // ANTHROPIC + if info, ok := AnthropicModels[m]; ok { + anthropicClient, err := clients.CreateClient(&ClientConfig{ + Provider: "anthropic", + Project: "anthropic-project", + Region: "us-west-1", + }) + if err != nil { + return fmt.Errorf("unable to create client: %v", err) + } + + anthropicClient.DefineModel(m, &info) + continue + } + } + + return nil +} From 745b03858acf4c4c7200592bfc9b6fadcdc1b779 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Mon, 10 Feb 2025 17:47:52 +0000 Subject: [PATCH 03/26] add test cases and minor refactor --- go/plugins/vertexai/modelgarden/anthropic.go | 1 + go/plugins/vertexai/modelgarden/client.go | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/go/plugins/vertexai/modelgarden/anthropic.go b/go/plugins/vertexai/modelgarden/anthropic.go index 7c1a242c8b..6814c2abe7 100644 --- a/go/plugins/vertexai/modelgarden/anthropic.go +++ b/go/plugins/vertexai/modelgarden/anthropic.go @@ -44,6 +44,7 @@ type AnthropicCloudClient struct { } var AnthropicClient = func(region string, project string) (Client, error) { + // TODO find a way to init Anthropic client (SDK) return &AnthropicCloudClient{ region: region, project: project, diff --git a/go/plugins/vertexai/modelgarden/client.go b/go/plugins/vertexai/modelgarden/client.go index 6fccd707b3..428ceba020 100644 --- a/go/plugins/vertexai/modelgarden/client.go +++ b/go/plugins/vertexai/modelgarden/client.go @@ -31,12 +31,8 @@ const ( Anthropic string = "anthropic" ) -// Function type for creating clients from supported providers -type ClientCreator func(region string, project string) (Client, error) - // Basic client configuration type ClientConfig struct { - Creator ClientCreator Provider string Project string Region string From d910186ec5f53d22d98709e70b7ce7a915a925f4 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Tue, 11 Feb 2025 16:14:59 +0000 Subject: [PATCH 04/26] add claude client and test cases (genkit wiring missing) --- go/plugins/vertexai/modelgarden/anthropic.go | 122 ++++++++++++++++-- .../vertexai/modelgarden/anthropic_test.go | 30 ----- go/plugins/vertexai/modelgarden/client.go | 40 ++++-- .../vertexai/modelgarden/modelgarden.go | 13 +- .../vertexai/modelgarden/modelgarden_test.go | 87 +++++++++++++ 5 files changed, 232 insertions(+), 60 deletions(-) delete mode 100644 go/plugins/vertexai/modelgarden/anthropic_test.go create mode 100644 go/plugins/vertexai/modelgarden/modelgarden_test.go diff --git a/go/plugins/vertexai/modelgarden/anthropic.go b/go/plugins/vertexai/modelgarden/anthropic.go index 6814c2abe7..3cb3c80bab 100644 --- a/go/plugins/vertexai/modelgarden/anthropic.go +++ b/go/plugins/vertexai/modelgarden/anthropic.go @@ -4,10 +4,15 @@ package modelgarden import ( - "log" + "context" + "fmt" "github.com/firebase/genkit/go/ai" + "github.com/firebase/genkit/go/genkit" "github.com/firebase/genkit/go/plugins/internal/gemini" + + "github.com/anthropics/anthropic-sdk-go" + "github.com/anthropics/anthropic-sdk-go/vertex" ) var AnthropicModels = map[string]ai.ModelInfo{ @@ -38,20 +43,111 @@ var AnthropicModels = map[string]ai.ModelInfo{ }, } -type AnthropicCloudClient struct { - region string - project string +// AnthropicClientConfig is the required configuration to create an Anthropic +// client +type AnthropicClientConfig struct { + Region string + Project string +} + +// AnthropicClient is a mirror struct of Anthropic's client but implements +// [Client] interface +type AnthropicClient struct { + *anthropic.Client } -var AnthropicClient = func(region string, project string) (Client, error) { - // TODO find a way to init Anthropic client (SDK) - return &AnthropicCloudClient{ - region: region, - project: project, - }, nil +// Anthropic defines how an Anthropic client is created +var Anthropic = func(config any) (Client, error) { + cfg, ok := config.(*AnthropicClientConfig) + if !ok { + return nil, fmt.Errorf("invalid config for Anthropic %T", config) + } + c := anthropic.NewClient( + vertex.WithGoogleAuth(context.Background(), cfg.Region, cfg.Project), + ) + + return &AnthropicClient{c}, nil } -func (a *AnthropicCloudClient) DefineModel(name string, info *ai.ModelInfo) error { - log.Printf("created an anthropic model: %s, versions: %#v", name, info.Versions) - return nil +// DefineModel adds +func (a *AnthropicClient) DefineModel(g *genkit.Genkit, name string, info *ai.ModelInfo) (ai.Model, error) { + var mi ai.ModelInfo + if info == nil { + var ok bool + mi, ok = AnthropicModels[name] + if !ok { + return nil, fmt.Errorf("%s.DefineModel: called with unknown model %q and nil ModelInfo", "anthropic", name) + } + } else { + mi = *info + } + return defineModel(g, a, name, mi), nil +} + +func defineModel(g *genkit.Genkit, client *AnthropicClient, name string, info ai.ModelInfo) ai.Model { + meta := &ai.ModelInfo{ + Label: "Anthropic" + "-" + name, + Supports: info.Supports, + Versions: info.Versions, + } + return genkit.DefineModel(g, "anthropic", name, meta, func( + ctx context.Context, + input *ai.ModelRequest, + cb func(context.Context, *ai.ModelResponseChunk) error, + ) (*ai.ModelResponse, error) { + return generate(ctx, client, name, input, cb) + }) +} + +func generate( + ctx context.Context, + client *AnthropicClient, + model string, + input *ai.ModelRequest, + cb func(context.Context, *ai.ModelResponseChunk) error, +) (*ai.ModelResponse, error) { + // TODO: create toAnthropicRequest functions to translate Genkit -> + // Anthropic and viceversa + + // streaming off + if cb == nil { + msg, err := client.Messages.New(ctx, anthropic.MessageNewParams{ + Model: anthropic.F(anthropic.ModelClaude3_5SonnetLatest), + MaxTokens: anthropic.F(int64(1024)), + Messages: anthropic.F([]anthropic.MessageParam{ + anthropic.NewUserMessage(anthropic.NewTextBlock("What's a quaternion?")), + }), + }) + if err != nil { + return nil, err + } + + fmt.Printf("%+v\n", msg.Content) + } else { + stream := client.Messages.NewStreaming(ctx, anthropic.MessageNewParams{ + Model: anthropic.F(anthropic.ModelClaude3_5SonnetLatest), + MaxTokens: anthropic.Int(1024), + Messages: anthropic.F([]anthropic.MessageParam{ + anthropic.NewUserMessage(anthropic.NewTextBlock("What's purpose of life?")), + }), + }) + + message := anthropic.Message{} + for stream.Next() { + event := stream.Current() + message.Accumulate(event) + + switch delta := event.Delta.(type) { + case anthropic.ContentBlockDeltaEventDelta: + if delta.Text != "" { + print(delta.Text) + } + } + } + if stream.Err() != nil { + return nil, stream.Err() + } + } + + return nil, nil } diff --git a/go/plugins/vertexai/modelgarden/anthropic_test.go b/go/plugins/vertexai/modelgarden/anthropic_test.go deleted file mode 100644 index 227d25db91..0000000000 --- a/go/plugins/vertexai/modelgarden/anthropic_test.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2025 Google LLC -// SPDX-License-Identifier: Apache-2.0 - -package modelgarden_test - -import ( - "testing" - - "github.com/firebase/genkit/go/ai" - "github.com/firebase/genkit/go/plugins/vertexai/modelgarden" -) - -// keep track of all clients -var clients = modelgarden.NewClientFactory() - -func TestAnthropicClient(t *testing.T) { - anthropicClient, err := clients.CreateClient(&modelgarden.ClientConfig{ - Region: "us-west-1", - Provider: "anthropic", - Project: "project-123", - }) - if err != nil { - t.Fatalf("unable to create anthropic client") - } - - err = anthropicClient.DefineModel("foo_model", &ai.ModelInfo{}) - if err != nil { - t.Fatalf("unable to define model: %v", err) - } -} diff --git a/go/plugins/vertexai/modelgarden/client.go b/go/plugins/vertexai/modelgarden/client.go index 428ceba020..2b4c1be33b 100644 --- a/go/plugins/vertexai/modelgarden/client.go +++ b/go/plugins/vertexai/modelgarden/client.go @@ -9,28 +9,27 @@ import ( "sync" "github.com/firebase/genkit/go/ai" + "github.com/firebase/genkit/go/genkit" ) // Generic Client interface for supported provider clients type Client interface { - DefineModel(name string, info *ai.ModelInfo) error + DefineModel(g *genkit.Genkit, name string, info *ai.ModelInfo) (ai.Model, error) } type ClientFactory struct { - clients map[string]Client // cache for provider clients - mu sync.Mutex + creators map[string]ClientCreator // cache for client creation functions + clients map[string]Client // cache for provider clients + mu sync.Mutex } func NewClientFactory() *ClientFactory { return &ClientFactory{ - clients: make(map[string]Client), + creators: make(map[string]ClientCreator), + clients: make(map[string]Client), } } -const ( - Anthropic string = "anthropic" -) - // Basic client configuration type ClientConfig struct { Provider string @@ -38,6 +37,12 @@ type ClientConfig struct { Region string } +type ClientCreator func(config any) (Client, error) + +func (f *ClientFactory) Register(provider string, creator ClientCreator) { + f.creators[provider] = creator +} + func (f *ClientFactory) CreateClient(config *ClientConfig) (Client, error) { if config == nil { return nil, errors.New("empty client config") @@ -52,15 +57,22 @@ func (f *ClientFactory) CreateClient(config *ClientConfig) (Client, error) { return client, nil // return from cache } + creator, ok := f.creators[config.Provider] + if !ok { + return nil, fmt.Errorf("unknown client type: %s", key) + } + var client Client var err error - switch config.Provider { - case Anthropic: - client, err = AnthropicClient(config.Region, config.Project) - if err != nil { - return nil, err - } + case "anthropic": + client, err = creator(&AnthropicClientConfig{ + Region: config.Region, + Project: config.Project, + }) + } + if err != nil { + return nil, err } f.clients[key] = client diff --git a/go/plugins/vertexai/modelgarden/modelgarden.go b/go/plugins/vertexai/modelgarden/modelgarden.go index b329b09d0c..4196d9dce5 100644 --- a/go/plugins/vertexai/modelgarden/modelgarden.go +++ b/go/plugins/vertexai/modelgarden/modelgarden.go @@ -7,6 +7,7 @@ import ( "context" "fmt" + "github.com/firebase/genkit/go/ai" "github.com/firebase/genkit/go/genkit" ) @@ -24,19 +25,25 @@ func Init(ctx context.Context, g *genkit.Genkit, cfg *ModelGardenOptions) error for _, m := range cfg.Models { // ANTHROPIC if info, ok := AnthropicModels[m]; ok { + clients.Register("anthropic", Anthropic) + anthropicClient, err := clients.CreateClient(&ClientConfig{ Provider: "anthropic", - Project: "anthropic-project", - Region: "us-west-1", + Project: cfg.ProjectID, + Region: cfg.Region, }) if err != nil { return fmt.Errorf("unable to create client: %v", err) } - anthropicClient.DefineModel(m, &info) + anthropicClient.DefineModel(g, m, &info) continue } } return nil } + +func Model(g *genkit.Genkit, provider string, name string) ai.Model { + return genkit.LookupModel(g, provider, name) +} diff --git a/go/plugins/vertexai/modelgarden/modelgarden_test.go b/go/plugins/vertexai/modelgarden/modelgarden_test.go new file mode 100644 index 0000000000..26e5c06a8c --- /dev/null +++ b/go/plugins/vertexai/modelgarden/modelgarden_test.go @@ -0,0 +1,87 @@ +// Copyright 2025 Google LLC +// SPDX-License-Identifier: Apache-2.0 + +package modelgarden_test + +import ( + "context" + "flag" + "testing" + + "github.com/firebase/genkit/go/ai" + "github.com/firebase/genkit/go/genkit" + "github.com/firebase/genkit/go/plugins/vertexai/modelgarden" +) + +var ( + projectID = flag.String("projectid", "", "Modelgarden project") + location = flag.String("location", "us-central1", "Geographic location") +) + +func TestModelGarden(t *testing.T) { + if *projectID == "" { + t.Skipf("no -projectid provided") + } + + ctx := context.Background() + g, err := genkit.New(nil) + if err != nil { + t.Fatal(err) + } + + err = modelgarden.Init(ctx, g, &modelgarden.ModelGardenOptions{ + ProjectID: *projectID, + Region: *location, + Models: []string{"claude-3-5-sonnet-v2"}, + }) + if err != nil { + t.Fatal(err) + } + + t.Run("invalid model", func(t *testing.T) { + m := modelgarden.Model(g, "anthropic", "claude-not-valid-v2") + if m != nil { + t.Fatal("model should have been invalid") + } + }) + + t.Run("model", func(t *testing.T) { + m := modelgarden.Model(g, "anthropic", "claude-3-5-sonnet-v2") + resp, err := genkit.Generate(ctx, g, ai.WithTextPrompt("What's your name?"), ai.WithModel(m)) + if err != nil { + t.Fatal(err) + } + t.Fatal(resp.Message.Content[0].Text) + }) + + t.Run("streaming", func(t *testing.T) { + m := modelgarden.Model(g, "anthropic", "claude-3-5-sonnet-v2") + out := "" + parts := 0 + + final, err := genkit.Generate(ctx, g, + ai.WithTextPrompt("Tell me a short story about a frog and a princess"), + ai.WithModel(m), + ai.WithStreaming(func(ctx context.Context, c *ai.ModelResponseChunk) error { + parts++ + out += c.Content[0].Text + return nil + }), + ) + if err != nil { + t.Fatal(err) + } + + out2 := "" + for _, p := range final.Message.Content { + out2 += p.Text + } + + if out != out2 { + t.Fatalf("streaming and final should containt the same text.\nstreaming: %s\nfinal:%s\n", out, out2) + } + if final.Usage.InputTokens == 0 || final.Usage.OutputTokens == 0 || final.Usage.TotalTokens == 0 { + t.Fatalf("empty usage stats: %#v", *final.Usage) + } + }) +} From 966355eda0cabdd499083e3259ee6c198cc8c3a2 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Tue, 11 Feb 2025 21:37:36 +0000 Subject: [PATCH 05/26] wiring: system and user text messages + basic conf --- go/plugins/vertexai/modelgarden/anthropic.go | 84 ++++++++++++-------- 1 file changed, 52 insertions(+), 32 deletions(-) diff --git a/go/plugins/vertexai/modelgarden/anthropic.go b/go/plugins/vertexai/modelgarden/anthropic.go index 3cb3c80bab..8346004443 100644 --- a/go/plugins/vertexai/modelgarden/anthropic.go +++ b/go/plugins/vertexai/modelgarden/anthropic.go @@ -6,6 +6,7 @@ package modelgarden import ( "context" "fmt" + "log" "github.com/firebase/genkit/go/ai" "github.com/firebase/genkit/go/genkit" @@ -109,44 +110,63 @@ func generate( // TODO: create toAnthropicRequest functions to translate Genkit -> // Anthropic and viceversa - // streaming off + log.Printf("%s", model) + log.Printf("%#v", input) + // parse configuration + reqParams := anthropic.MessageNewParams{} + if c, ok := input.Config.(*ai.GenerationCommonConfig); ok && c != nil { + if c.MaxOutputTokens != 0 { + reqParams.MaxTokens = anthropic.F(int64(c.MaxOutputTokens)) + } + reqParams.Model = anthropic.F(anthropic.Model(model)) + // TODO: check if Version is needed + if c.Version != "" { + reqParams.Model = anthropic.F(anthropic.Model(c.Version)) + } + if c.Temperature != 0 { + reqParams.Temperature = anthropic.F(c.Temperature) + } + if c.TopK != 0 { + reqParams.TopK = anthropic.F(int64(c.TopK)) + } + if c.TopP != 0 { + reqParams.TopP = anthropic.F(float64(c.TopP)) + } + if len(c.StopSequences) > 0 { + reqParams.StopSequences = anthropic.F(c.StopSequences) + } + } + // system and user blocks + sysBlocks := []anthropic.TextBlockParam{} + userBlocks := []anthropic.TextBlockParam{} + for _, m := range input.Messages { + // TODO: convert messages to its types (text, media, toolResponse) + if m.Role == ai.RoleSystem { + sysBlocks = append(sysBlocks, anthropic.NewTextBlock(m.Text())) + } + if m.Role == ai.RoleUser { + userBlocks = append(userBlocks, anthropic.NewTextBlock(m.Text())) + } + } + if len(sysBlocks) > 0 { + reqParams.System = anthropic.F(sysBlocks) + } + if len(userBlocks) > 0 { + messageParam := make([]anthropic.MessageParam, 0) + for _, u := range userBlocks { + messageParam = append(messageParam, anthropic.NewUserMessage(u)) + } + reqParams.Messages = anthropic.F(messageParam) + } + + // no streaming if cb == nil { - msg, err := client.Messages.New(ctx, anthropic.MessageNewParams{ - Model: anthropic.F(anthropic.ModelClaude3_5SonnetLatest), - MaxTokens: anthropic.F(int64(1024)), - Messages: anthropic.F([]anthropic.MessageParam{ - anthropic.NewUserMessage(anthropic.NewTextBlock("What's a quaternion?")), - }), - }) + msg, err := client.Messages.New(ctx, reqParams) if err != nil { return nil, err } fmt.Printf("%+v\n", msg.Content) - } else { - stream := client.Messages.NewStreaming(ctx, anthropic.MessageNewParams{ - Model: anthropic.F(anthropic.ModelClaude3_5SonnetLatest), - MaxTokens: anthropic.Int(1024), - Messages: anthropic.F([]anthropic.MessageParam{ - anthropic.NewUserMessage(anthropic.NewTextBlock("What's purpose of life?")), - }), - }) - - message := anthropic.Message{} - for stream.Next() { - event := stream.Current() - message.Accumulate(event) - - switch delta := event.Delta.(type) { - case anthropic.ContentBlockDeltaEventDelta: - if delta.Text != "" { - print(delta.Text) - } - } - } - if stream.Err() != nil { - return nil, stream.Err() - } } return nil, nil From 4e257f587fbef4843c9624a73bb3c839fd984b60 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Tue, 11 Feb 2025 22:01:40 +0000 Subject: [PATCH 06/26] test: add model version test --- .../vertexai/modelgarden/modelgarden_test.go | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/go/plugins/vertexai/modelgarden/modelgarden_test.go b/go/plugins/vertexai/modelgarden/modelgarden_test.go index 26e5c06a8c..95ab1dd150 100644 --- a/go/plugins/vertexai/modelgarden/modelgarden_test.go +++ b/go/plugins/vertexai/modelgarden/modelgarden_test.go @@ -45,6 +45,34 @@ func TestModelGarden(t *testing.T) { } }) + t.Run("model version ok", func(t *testing.T) { + m := modelgarden.Model(g, "anthropic", "claude-3-5-sonnet-v2") + _, err := genkit.Generate(ctx, g, + ai.WithConfig(&ai.GenerationCommonConfig{ + Temperature: 1, + Version: "claude-3-5-sonnet-v2@20241022", + }), + ai.WithModel(m), + ) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("model version nok", func(t *testing.T) { + m := modelgarden.Model(g, "anthropic", "claude-3-5-sonnet-v2") + _, err := genkit.Generate(ctx, g, + ai.WithConfig(&ai.GenerationCommonConfig{ + Temperature: 1, + Version: "foo", + }), + ai.WithModel(m), + ) + if err == nil { + t.Fatal("should have failed due wrong model version") + } + }) + t.Run("model", func(t *testing.T) { m := modelgarden.Model(g, "anthropic", "claude-3-5-sonnet-v2") resp, err := genkit.Generate(ctx, g, ai.WithTextPrompt("What's your name?"), ai.WithModel(m)) From 817bc061b0e1ff2ff338f7696a401a1a7fa9202e Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Tue, 11 Feb 2025 22:32:08 +0000 Subject: [PATCH 07/26] tidy: added constants and docs --- go/plugins/vertexai/modelgarden/anthropic.go | 10 ++++++---- go/plugins/vertexai/modelgarden/client.go | 6 ++++-- go/plugins/vertexai/modelgarden/modelgarden.go | 13 ++++++++++--- go/plugins/vertexai/modelgarden/modelgarden_test.go | 10 +++++----- 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/go/plugins/vertexai/modelgarden/anthropic.go b/go/plugins/vertexai/modelgarden/anthropic.go index 8346004443..d8302cc876 100644 --- a/go/plugins/vertexai/modelgarden/anthropic.go +++ b/go/plugins/vertexai/modelgarden/anthropic.go @@ -16,6 +16,7 @@ import ( "github.com/anthropics/anthropic-sdk-go/vertex" ) +// supported anthropic models var AnthropicModels = map[string]ai.ModelInfo{ "claude-3-5-sonnet-v2": { Label: "Vertex AI Model Garden - Claude 3.5 Sonnet", @@ -70,14 +71,14 @@ var Anthropic = func(config any) (Client, error) { return &AnthropicClient{c}, nil } -// DefineModel adds +// DefineModel adds the model to the registry func (a *AnthropicClient) DefineModel(g *genkit.Genkit, name string, info *ai.ModelInfo) (ai.Model, error) { var mi ai.ModelInfo if info == nil { var ok bool mi, ok = AnthropicModels[name] if !ok { - return nil, fmt.Errorf("%s.DefineModel: called with unknown model %q and nil ModelInfo", "anthropic", name) + return nil, fmt.Errorf("%s.DefineModel: called with unknown model %q and nil ModelInfo", AnthropicProvider, name) } } else { mi = *info @@ -87,11 +88,11 @@ func (a *AnthropicClient) DefineModel(g *genkit.Genkit, name string, info *ai.Mo func defineModel(g *genkit.Genkit, client *AnthropicClient, name string, info ai.ModelInfo) ai.Model { meta := &ai.ModelInfo{ - Label: "Anthropic" + "-" + name, + Label: AnthropicProvider + "-" + name, Supports: info.Supports, Versions: info.Versions, } - return genkit.DefineModel(g, "anthropic", name, meta, func( + return genkit.DefineModel(g, AnthropicProvider, name, meta, func( ctx context.Context, input *ai.ModelRequest, cb func(context.Context, *ai.ModelResponseChunk) error, @@ -100,6 +101,7 @@ func defineModel(g *genkit.Genkit, client *AnthropicClient, name string, info ai }) } +// generate function defines how a generate request is done in Anthropic models func generate( ctx context.Context, client *AnthropicClient, diff --git a/go/plugins/vertexai/modelgarden/client.go b/go/plugins/vertexai/modelgarden/client.go index 2b4c1be33b..fc028836a0 100644 --- a/go/plugins/vertexai/modelgarden/client.go +++ b/go/plugins/vertexai/modelgarden/client.go @@ -40,7 +40,9 @@ type ClientConfig struct { type ClientCreator func(config any) (Client, error) func (f *ClientFactory) Register(provider string, creator ClientCreator) { - f.creators[provider] = creator + if _, ok := f.creators[provider]; !ok { + f.creators[provider] = creator + } } func (f *ClientFactory) CreateClient(config *ClientConfig) (Client, error) { @@ -65,7 +67,7 @@ func (f *ClientFactory) CreateClient(config *ClientConfig) (Client, error) { var client Client var err error switch config.Provider { - case "anthropic": + case AnthropicProvider: client, err = creator(&AnthropicClientConfig{ Region: config.Region, Project: config.Project, diff --git a/go/plugins/vertexai/modelgarden/modelgarden.go b/go/plugins/vertexai/modelgarden/modelgarden.go index 4196d9dce5..826e40ce6e 100644 --- a/go/plugins/vertexai/modelgarden/modelgarden.go +++ b/go/plugins/vertexai/modelgarden/modelgarden.go @@ -11,24 +11,31 @@ import ( "github.com/firebase/genkit/go/genkit" ) +const ( + AnthropicProvider = "anthropic" + MistralProvider = "mistral" +) + type ModelGardenOptions struct { ProjectID string Region string Models []string } +// A cache where all clients and its creators will be stored +var clients = NewClientFactory() + // Init initializes the ModelGarden plugin // After calling Init, you may call [DefineModel] to create and register // any additional generative models func Init(ctx context.Context, g *genkit.Genkit, cfg *ModelGardenOptions) error { - clients := NewClientFactory() for _, m := range cfg.Models { // ANTHROPIC if info, ok := AnthropicModels[m]; ok { - clients.Register("anthropic", Anthropic) + clients.Register(AnthropicProvider, Anthropic) anthropicClient, err := clients.CreateClient(&ClientConfig{ - Provider: "anthropic", + Provider: AnthropicProvider, Project: cfg.ProjectID, Region: cfg.Region, }) diff --git a/go/plugins/vertexai/modelgarden/modelgarden_test.go b/go/plugins/vertexai/modelgarden/modelgarden_test.go index 95ab1dd150..2015dd223a 100644 --- a/go/plugins/vertexai/modelgarden/modelgarden_test.go +++ b/go/plugins/vertexai/modelgarden/modelgarden_test.go @@ -39,14 +39,14 @@ func TestModelGarden(t *testing.T) { } t.Run("invalid model", func(t *testing.T) { - m := modelgarden.Model(g, "anthropic", "claude-not-valid-v2") + m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-not-valid-v2") if m != nil { t.Fatal("model should have been invalid") } }) t.Run("model version ok", func(t *testing.T) { - m := modelgarden.Model(g, "anthropic", "claude-3-5-sonnet-v2") + m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") _, err := genkit.Generate(ctx, g, ai.WithConfig(&ai.GenerationCommonConfig{ Temperature: 1, @@ -60,7 +60,7 @@ func TestModelGarden(t *testing.T) { }) t.Run("model version nok", func(t *testing.T) { - m := modelgarden.Model(g, "anthropic", "claude-3-5-sonnet-v2") + m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") _, err := genkit.Generate(ctx, g, ai.WithConfig(&ai.GenerationCommonConfig{ Temperature: 1, @@ -74,7 +74,7 @@ func TestModelGarden(t *testing.T) { }) t.Run("model", func(t *testing.T) { - m := modelgarden.Model(g, "anthropic", "claude-3-5-sonnet-v2") + m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") resp, err := genkit.Generate(ctx, g, ai.WithTextPrompt("What's your name?"), ai.WithModel(m)) if err != nil { t.Fatal(err) @@ -83,7 +83,7 @@ func TestModelGarden(t *testing.T) { }) t.Run("streaming", func(t *testing.T) { - m := modelgarden.Model(g, "anthropic", "claude-3-5-sonnet-v2") + m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") out := "" parts := 0 From 1caa09c8f6e2b0885e37a1421025cc4f508db943 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Tue, 11 Feb 2025 23:29:06 +0000 Subject: [PATCH 08/26] docs: client.go docs --- go/plugins/vertexai/modelgarden/client.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/go/plugins/vertexai/modelgarden/client.go b/go/plugins/vertexai/modelgarden/client.go index fc028836a0..f8281091a1 100644 --- a/go/plugins/vertexai/modelgarden/client.go +++ b/go/plugins/vertexai/modelgarden/client.go @@ -37,14 +37,19 @@ type ClientConfig struct { Region string } +// ClientCreator is a function type that will be defined on every provider in order to create its +// client type ClientCreator func(config any) (Client, error) +// Register adds the client creator function to a cache for later use func (f *ClientFactory) Register(provider string, creator ClientCreator) { if _, ok := f.creators[provider]; !ok { f.creators[provider] = creator } } +// CreateClient creates a client with the given configuration +// A [ClientCreator] must have been previously registered func (f *ClientFactory) CreateClient(config *ClientConfig) (Client, error) { if config == nil { return nil, errors.New("empty client config") @@ -67,6 +72,7 @@ func (f *ClientFactory) CreateClient(config *ClientConfig) (Client, error) { var client Client var err error switch config.Provider { + // TODO: add providers when needed case AnthropicProvider: client, err = creator(&AnthropicClientConfig{ Region: config.Region, From 78f759c584843fcedec6d79c264610362e6c8ff6 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Wed, 12 Feb 2025 19:05:51 +0000 Subject: [PATCH 09/26] feat: add support to claude models --- go/plugins/vertexai/modelgarden/anthropic.go | 62 ++++++++++++++--- go/plugins/vertexai/modelgarden/client.go | 8 +-- .../vertexai/modelgarden/modelgarden.go | 46 +++++++++++-- .../vertexai/modelgarden/modelgarden_test.go | 10 ++- go/samples/model-garden/main.go | 69 +++++++++++++++++++ 5 files changed, 173 insertions(+), 22 deletions(-) create mode 100644 go/samples/model-garden/main.go diff --git a/go/plugins/vertexai/modelgarden/anthropic.go b/go/plugins/vertexai/modelgarden/anthropic.go index d8302cc876..a39a39c766 100644 --- a/go/plugins/vertexai/modelgarden/anthropic.go +++ b/go/plugins/vertexai/modelgarden/anthropic.go @@ -5,8 +5,8 @@ package modelgarden import ( "context" + "encoding/json" "fmt" - "log" "github.com/firebase/genkit/go/ai" "github.com/firebase/genkit/go/genkit" @@ -48,8 +48,8 @@ var AnthropicModels = map[string]ai.ModelInfo{ // AnthropicClientConfig is the required configuration to create an Anthropic // client type AnthropicClientConfig struct { - Region string - Project string + Location string + Project string } // AnthropicClient is a mirror struct of Anthropic's client but implements @@ -64,10 +64,13 @@ var Anthropic = func(config any) (Client, error) { if !ok { return nil, fmt.Errorf("invalid config for Anthropic %T", config) } + fmt.Printf("%#v", cfg) c := anthropic.NewClient( - vertex.WithGoogleAuth(context.Background(), cfg.Region, cfg.Project), + vertex.WithGoogleAuth(context.Background(), cfg.Location, cfg.Project), ) + fmt.Printf("client: %#v", c) + return &AnthropicClient{c}, nil } @@ -109,14 +112,12 @@ func generate( input *ai.ModelRequest, cb func(context.Context, *ai.ModelResponseChunk) error, ) (*ai.ModelResponse, error) { - // TODO: create toAnthropicRequest functions to translate Genkit -> - // Anthropic and viceversa - - log.Printf("%s", model) - log.Printf("%#v", input) // parse configuration reqParams := anthropic.MessageNewParams{} if c, ok := input.Config.(*ai.GenerationCommonConfig); ok && c != nil { + + // defaulting to 1024 since this is a mandatory parameter + reqParams.MaxTokens = anthropic.F(int64(1024)) if c.MaxOutputTokens != 0 { reqParams.MaxTokens = anthropic.F(int64(c.MaxOutputTokens)) } @@ -141,7 +142,9 @@ func generate( // system and user blocks sysBlocks := []anthropic.TextBlockParam{} userBlocks := []anthropic.TextBlockParam{} + fmt.Printf("input.Messages: %#v", input.Messages) for _, m := range input.Messages { + fmt.Printf("input.Messages: %#v", input.Messages) // TODO: convert messages to its types (text, media, toolResponse) if m.Role == ai.RoleSystem { sysBlocks = append(sysBlocks, anthropic.NewTextBlock(m.Text())) @@ -168,8 +171,47 @@ func generate( return nil, err } - fmt.Printf("%+v\n", msg.Content) + r := toGenkitResponse(msg) + r.Request = input + return r, nil } return nil, nil } + +func toGenkitResponse(m *anthropic.Message) *ai.ModelResponse { + r := &ai.ModelResponse{} + + switch m.StopReason { + case anthropic.MessageStopReasonMaxTokens: + r.FinishReason = ai.FinishReasonLength + case anthropic.MessageStopReasonStopSequence: + r.FinishReason = ai.FinishReasonStop + case anthropic.MessageStopReasonEndTurn: + case anthropic.MessageStopReasonToolUse: + r.FinishReason = ai.FinishReasonOther + } + + msg := &ai.Message{} + msg.Role = ai.Role(m.Role) + for _, part := range m.Content { + var p *ai.Part + switch part.Type { + case anthropic.ContentBlockTypeText: + p = ai.NewTextPart(string(part.Text)) + case anthropic.ContentBlockTypeToolUse: + t := &ai.ToolRequest{} + err := json.Unmarshal([]byte(part.Input), &t.Input) + if err != nil { + return nil + } + p = ai.NewToolRequestPart(t) + default: + panic(fmt.Sprintf("unknown part: %#v", part)) + } + msg.Content = append(msg.Content, p) + } + + r.Message = msg + return r +} diff --git a/go/plugins/vertexai/modelgarden/client.go b/go/plugins/vertexai/modelgarden/client.go index f8281091a1..cf029b11e4 100644 --- a/go/plugins/vertexai/modelgarden/client.go +++ b/go/plugins/vertexai/modelgarden/client.go @@ -34,7 +34,7 @@ func NewClientFactory() *ClientFactory { type ClientConfig struct { Provider string Project string - Region string + Location string } // ClientCreator is a function type that will be defined on every provider in order to create its @@ -59,7 +59,7 @@ func (f *ClientFactory) CreateClient(config *ClientConfig) (Client, error) { defer f.mu.Unlock() // every client will be identified by its provider-region combination - key := fmt.Sprintf("%s-%s", config.Provider, config.Region) + key := fmt.Sprintf("%s-%s", config.Provider, config.Location) if client, ok := f.clients[key]; ok { return client, nil // return from cache } @@ -75,8 +75,8 @@ func (f *ClientFactory) CreateClient(config *ClientConfig) (Client, error) { // TODO: add providers when needed case AnthropicProvider: client, err = creator(&AnthropicClientConfig{ - Region: config.Region, - Project: config.Project, + Location: config.Location, + Project: config.Project, }) } if err != nil { diff --git a/go/plugins/vertexai/modelgarden/modelgarden.go b/go/plugins/vertexai/modelgarden/modelgarden.go index 826e40ce6e..67c49152b4 100644 --- a/go/plugins/vertexai/modelgarden/modelgarden.go +++ b/go/plugins/vertexai/modelgarden/modelgarden.go @@ -6,6 +6,8 @@ package modelgarden import ( "context" "fmt" + "os" + "sync" "github.com/firebase/genkit/go/ai" "github.com/firebase/genkit/go/genkit" @@ -16,19 +18,53 @@ const ( MistralProvider = "mistral" ) -type ModelGardenOptions struct { +type Config struct { ProjectID string - Region string + Location string Models []string } // A cache where all clients and its creators will be stored var clients = NewClientFactory() +var state struct { + initted bool + clients *ClientFactory + projectID string + location string + mu sync.Mutex +} + // Init initializes the ModelGarden plugin // After calling Init, you may call [DefineModel] to create and register // any additional generative models -func Init(ctx context.Context, g *genkit.Genkit, cfg *ModelGardenOptions) error { +func Init(ctx context.Context, g *genkit.Genkit, cfg *Config) error { + if cfg == nil { + cfg = &Config{} + } + + state.mu.Lock() + defer state.mu.Unlock() + if state.initted { + panic("modelgarden.init already called") + } + + state.projectID = cfg.ProjectID + if state.projectID == "" { + state.projectID = os.Getenv("GCLOUD_PROJECT") + } + if state.projectID == "" { + state.projectID = os.Getenv("GOOGLE_CLOUD_PROJECT") + } + if state.projectID == "" { + return fmt.Errorf("modelgarden.Init: Model Garden requires setting GCLOUD_PROJECT or GOOGLE_CLOUD_PROJECT in the environment") + } + + state.location = cfg.Location + if state.location == "" { + state.location = "us-central1" + } + for _, m := range cfg.Models { // ANTHROPIC if info, ok := AnthropicModels[m]; ok { @@ -36,8 +72,8 @@ func Init(ctx context.Context, g *genkit.Genkit, cfg *ModelGardenOptions) error anthropicClient, err := clients.CreateClient(&ClientConfig{ Provider: AnthropicProvider, - Project: cfg.ProjectID, - Region: cfg.Region, + Project: state.projectID, + Location: state.location, }) if err != nil { return fmt.Errorf("unable to create client: %v", err) diff --git a/go/plugins/vertexai/modelgarden/modelgarden_test.go b/go/plugins/vertexai/modelgarden/modelgarden_test.go index 2015dd223a..e6bc0823e7 100644 --- a/go/plugins/vertexai/modelgarden/modelgarden_test.go +++ b/go/plugins/vertexai/modelgarden/modelgarden_test.go @@ -15,7 +15,7 @@ import ( var ( projectID = flag.String("projectid", "", "Modelgarden project") - location = flag.String("location", "us-central1", "Geographic location") + location = flag.String("location", "us-east5", "Geographic location") ) func TestModelGarden(t *testing.T) { @@ -29,9 +29,9 @@ func TestModelGarden(t *testing.T) { t.Fatal(err) } - err = modelgarden.Init(ctx, g, &modelgarden.ModelGardenOptions{ + err = modelgarden.Init(ctx, g, &modelgarden.Config{ ProjectID: *projectID, - Region: *location, + Location: *location, Models: []string{"claude-3-5-sonnet-v2"}, }) if err != nil { @@ -53,6 +53,7 @@ func TestModelGarden(t *testing.T) { Version: "claude-3-5-sonnet-v2@20241022", }), ai.WithModel(m), + ai.WithTextPrompt("Hello there..."), ) if err != nil { t.Fatal(err) @@ -60,6 +61,7 @@ func TestModelGarden(t *testing.T) { }) t.Run("model version nok", func(t *testing.T) { + t.Skipf("no -projectid provided") m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") _, err := genkit.Generate(ctx, g, ai.WithConfig(&ai.GenerationCommonConfig{ @@ -74,6 +76,7 @@ func TestModelGarden(t *testing.T) { }) t.Run("model", func(t *testing.T) { + t.Skipf("no -projectid provided") m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") resp, err := genkit.Generate(ctx, g, ai.WithTextPrompt("What's your name?"), ai.WithModel(m)) if err != nil { @@ -83,6 +86,7 @@ func TestModelGarden(t *testing.T) { }) t.Run("streaming", func(t *testing.T) { + t.Skipf("no -projectid provided") m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") out := "" parts := 0 diff --git a/go/samples/model-garden/main.go b/go/samples/model-garden/main.go new file mode 100644 index 0000000000..0aab200465 --- /dev/null +++ b/go/samples/model-garden/main.go @@ -0,0 +1,69 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "fmt" + "log" + + "github.com/firebase/genkit/go/ai" + "github.com/firebase/genkit/go/genkit" + "github.com/firebase/genkit/go/plugins/vertexai/modelgarden" +) + +func main() { + ctx := context.Background() + + g, err := genkit.New(nil) + if err != nil { + log.Fatal(err) + } + + cfg := &modelgarden.Config{ + Location: "us-east5", + Models: []string{"claude-3-5-sonnet-v2"}, + } + if err := modelgarden.Init(ctx, g, cfg); err != nil { + log.Fatal(err) + } + + // Define a simple flow that generates jokes about a given topic + genkit.DefineFlow(g, "jokesFlow", func(ctx context.Context, input string) (string, error) { + m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") + if m == nil { + return "", errors.New("jokesFlow: failed to find model") + } + + resp, err := genkit.Generate(ctx, g, + ai.WithModel(m), + ai.WithConfig(&ai.GenerationCommonConfig{ + Temperature: 0.1, + Version: "claude-3-5-sonnet-v2@20241022", + }), + ai.WithTextPrompt(fmt.Sprintf(`Tell silly short jokes about %s`, input))) + if err != nil { + return "", err + } + + text := resp.Text() + return text, nil + }) + + if err := g.Start(ctx, nil); err != nil { + log.Fatal(err) + } +} From 45321426cac657ca13d82e8f414ce6df6774136b Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Wed, 12 Feb 2025 19:15:26 +0000 Subject: [PATCH 10/26] use state.clients instead of standalone variable --- go/plugins/vertexai/modelgarden/modelgarden.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/go/plugins/vertexai/modelgarden/modelgarden.go b/go/plugins/vertexai/modelgarden/modelgarden.go index 67c49152b4..28a14fd475 100644 --- a/go/plugins/vertexai/modelgarden/modelgarden.go +++ b/go/plugins/vertexai/modelgarden/modelgarden.go @@ -18,18 +18,16 @@ const ( MistralProvider = "mistral" ) +// Config for Model Garden type Config struct { ProjectID string Location string Models []string } -// A cache where all clients and its creators will be stored -var clients = NewClientFactory() - var state struct { initted bool - clients *ClientFactory + clients *ClientFactory // cache for all clients for available providers projectID string location string mu sync.Mutex @@ -65,12 +63,13 @@ func Init(ctx context.Context, g *genkit.Genkit, cfg *Config) error { state.location = "us-central1" } + state.clients = NewClientFactory() for _, m := range cfg.Models { // ANTHROPIC if info, ok := AnthropicModels[m]; ok { - clients.Register(AnthropicProvider, Anthropic) + state.clients.Register(AnthropicProvider, Anthropic) - anthropicClient, err := clients.CreateClient(&ClientConfig{ + anthropicClient, err := state.clients.CreateClient(&ClientConfig{ Provider: AnthropicProvider, Project: state.projectID, Location: state.location, From c27a30765e4a193479786c60ab45976c168515c0 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Wed, 12 Feb 2025 19:21:24 +0000 Subject: [PATCH 11/26] docs: refine anthropic docs --- go/plugins/vertexai/modelgarden/anthropic.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/go/plugins/vertexai/modelgarden/anthropic.go b/go/plugins/vertexai/modelgarden/anthropic.go index a39a39c766..172daff19c 100644 --- a/go/plugins/vertexai/modelgarden/anthropic.go +++ b/go/plugins/vertexai/modelgarden/anthropic.go @@ -122,7 +122,6 @@ func generate( reqParams.MaxTokens = anthropic.F(int64(c.MaxOutputTokens)) } reqParams.Model = anthropic.F(anthropic.Model(model)) - // TODO: check if Version is needed if c.Version != "" { reqParams.Model = anthropic.F(anthropic.Model(c.Version)) } @@ -142,9 +141,7 @@ func generate( // system and user blocks sysBlocks := []anthropic.TextBlockParam{} userBlocks := []anthropic.TextBlockParam{} - fmt.Printf("input.Messages: %#v", input.Messages) for _, m := range input.Messages { - fmt.Printf("input.Messages: %#v", input.Messages) // TODO: convert messages to its types (text, media, toolResponse) if m.Role == ai.RoleSystem { sysBlocks = append(sysBlocks, anthropic.NewTextBlock(m.Text())) @@ -175,10 +172,13 @@ func generate( r.Request = input return r, nil } + // TODO: add streaming support return nil, nil } +// toGenkitResponse translates an Anthropic Message response to a Genkit +// [ai.ModelResponse] func toGenkitResponse(m *anthropic.Message) *ai.ModelResponse { r := &ai.ModelResponse{} @@ -190,6 +190,8 @@ func toGenkitResponse(m *anthropic.Message) *ai.ModelResponse { case anthropic.MessageStopReasonEndTurn: case anthropic.MessageStopReasonToolUse: r.FinishReason = ai.FinishReasonOther + default: + r.FinishReason = ai.FinishReasonUnknown } msg := &ai.Message{} From 6458f88cfc60c732f9cbb21c204d3a01c9f07602 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Wed, 12 Feb 2025 21:37:09 +0000 Subject: [PATCH 12/26] use the least req params --- go/plugins/vertexai/modelgarden/anthropic.go | 7 ++++--- go/plugins/vertexai/modelgarden/modelgarden.go | 3 ++- go/samples/model-garden/main.go | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/go/plugins/vertexai/modelgarden/anthropic.go b/go/plugins/vertexai/modelgarden/anthropic.go index 172daff19c..fbdd1cb942 100644 --- a/go/plugins/vertexai/modelgarden/anthropic.go +++ b/go/plugins/vertexai/modelgarden/anthropic.go @@ -64,13 +64,10 @@ var Anthropic = func(config any) (Client, error) { if !ok { return nil, fmt.Errorf("invalid config for Anthropic %T", config) } - fmt.Printf("%#v", cfg) c := anthropic.NewClient( vertex.WithGoogleAuth(context.Background(), cfg.Location, cfg.Project), ) - fmt.Printf("client: %#v", c) - return &AnthropicClient{c}, nil } @@ -114,6 +111,10 @@ func generate( ) (*ai.ModelResponse, error) { // parse configuration reqParams := anthropic.MessageNewParams{} + + reqParams.Model = anthropic.F(anthropic.Model(model)) + reqParams.MaxTokens = anthropic.F(int64(1024)) + if c, ok := input.Config.(*ai.GenerationCommonConfig); ok && c != nil { // defaulting to 1024 since this is a mandatory parameter diff --git a/go/plugins/vertexai/modelgarden/modelgarden.go b/go/plugins/vertexai/modelgarden/modelgarden.go index 28a14fd475..947860f14a 100644 --- a/go/plugins/vertexai/modelgarden/modelgarden.go +++ b/go/plugins/vertexai/modelgarden/modelgarden.go @@ -44,7 +44,7 @@ func Init(ctx context.Context, g *genkit.Genkit, cfg *Config) error { state.mu.Lock() defer state.mu.Unlock() if state.initted { - panic("modelgarden.init already called") + panic("modelgarden.Init already called") } state.projectID = cfg.ProjectID @@ -64,6 +64,7 @@ func Init(ctx context.Context, g *genkit.Genkit, cfg *Config) error { } state.clients = NewClientFactory() + state.initted = true for _, m := range cfg.Models { // ANTHROPIC if info, ok := AnthropicModels[m]; ok { diff --git a/go/samples/model-garden/main.go b/go/samples/model-garden/main.go index 0aab200465..5212de9d3a 100644 --- a/go/samples/model-garden/main.go +++ b/go/samples/model-garden/main.go @@ -35,7 +35,7 @@ func main() { cfg := &modelgarden.Config{ Location: "us-east5", - Models: []string{"claude-3-5-sonnet-v2"}, + Models: []string{"claude-3-5-sonnet-v2", "claude-3-5-sonnet"}, } if err := modelgarden.Init(ctx, g, cfg); err != nil { log.Fatal(err) From b425e02aa6a212b169c1d2e0c8e4a7095d01322b Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Thu, 13 Feb 2025 00:46:34 +0000 Subject: [PATCH 13/26] fix: minor fixes and docs --- go/plugins/vertexai/modelgarden/anthropic.go | 82 ++++++++++++-------- go/samples/model-garden/main.go | 2 +- 2 files changed, 51 insertions(+), 33 deletions(-) diff --git a/go/plugins/vertexai/modelgarden/anthropic.go b/go/plugins/vertexai/modelgarden/anthropic.go index fbdd1cb942..431607abc1 100644 --- a/go/plugins/vertexai/modelgarden/anthropic.go +++ b/go/plugins/vertexai/modelgarden/anthropic.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "fmt" + "math" "github.com/firebase/genkit/go/ai" "github.com/firebase/genkit/go/genkit" @@ -110,39 +111,70 @@ func generate( cb func(context.Context, *ai.ModelResponseChunk) error, ) (*ai.ModelResponse, error) { // parse configuration - reqParams := anthropic.MessageNewParams{} + req := toAnthropicRequest(model, input) - reqParams.Model = anthropic.F(anthropic.Model(model)) - reqParams.MaxTokens = anthropic.F(int64(1024)) + // no streaming + if cb == nil { + msg, err := client.Messages.New(ctx, req) + if err != nil { + return nil, err + } - if c, ok := input.Config.(*ai.GenerationCommonConfig); ok && c != nil { + r := toGenkitResponse(msg) + r.Request = input + return r, nil + } else { + stream := client.Messages.NewStreaming(ctx, req) + msg := anthropic.Message{} + for stream.Next() { + event := stream.Current() + msg.Accumulate(event) + + switch delta := event.Delta.(type) { + case anthropic.ContentBlockDeltaEventDelta: + if delta.Text != "" { + fmt.Printf(delta.Text) + } + } + } + } + + return nil, nil +} - // defaulting to 1024 since this is a mandatory parameter - reqParams.MaxTokens = anthropic.F(int64(1024)) +// toAnthropicRequest translates [ai.ModelRequest] to an Anthropic request +func toAnthropicRequest(model string, i *ai.ModelRequest) anthropic.MessageNewParams { + req := anthropic.MessageNewParams{} + + // minimum required data to perform a request + req.Model = anthropic.F(anthropic.Model(model)) + req.MaxTokens = anthropic.F(int64(math.MaxInt64)) + + if c, ok := i.Config.(*ai.GenerationCommonConfig); ok && c != nil { if c.MaxOutputTokens != 0 { - reqParams.MaxTokens = anthropic.F(int64(c.MaxOutputTokens)) + req.MaxTokens = anthropic.F(int64(c.MaxOutputTokens)) } - reqParams.Model = anthropic.F(anthropic.Model(model)) + req.Model = anthropic.F(anthropic.Model(model)) if c.Version != "" { - reqParams.Model = anthropic.F(anthropic.Model(c.Version)) + req.Model = anthropic.F(anthropic.Model(c.Version)) } if c.Temperature != 0 { - reqParams.Temperature = anthropic.F(c.Temperature) + req.Temperature = anthropic.F(c.Temperature) } if c.TopK != 0 { - reqParams.TopK = anthropic.F(int64(c.TopK)) + req.TopK = anthropic.F(int64(c.TopK)) } if c.TopP != 0 { - reqParams.TopP = anthropic.F(float64(c.TopP)) + req.TopP = anthropic.F(float64(c.TopP)) } if len(c.StopSequences) > 0 { - reqParams.StopSequences = anthropic.F(c.StopSequences) + req.StopSequences = anthropic.F(c.StopSequences) } } // system and user blocks sysBlocks := []anthropic.TextBlockParam{} userBlocks := []anthropic.TextBlockParam{} - for _, m := range input.Messages { + for _, m := range i.Messages { // TODO: convert messages to its types (text, media, toolResponse) if m.Role == ai.RoleSystem { sysBlocks = append(sysBlocks, anthropic.NewTextBlock(m.Text())) @@ -152,34 +184,20 @@ func generate( } } if len(sysBlocks) > 0 { - reqParams.System = anthropic.F(sysBlocks) + req.System = anthropic.F(sysBlocks) } if len(userBlocks) > 0 { messageParam := make([]anthropic.MessageParam, 0) for _, u := range userBlocks { messageParam = append(messageParam, anthropic.NewUserMessage(u)) } - reqParams.Messages = anthropic.F(messageParam) - } - - // no streaming - if cb == nil { - msg, err := client.Messages.New(ctx, reqParams) - if err != nil { - return nil, err - } - - r := toGenkitResponse(msg) - r.Request = input - return r, nil + req.Messages = anthropic.F(messageParam) } - // TODO: add streaming support - return nil, nil + return req } -// toGenkitResponse translates an Anthropic Message response to a Genkit -// [ai.ModelResponse] +// toGenkitResponse translates an Anthropic Message to [ai.ModelResponse] func toGenkitResponse(m *anthropic.Message) *ai.ModelResponse { r := &ai.ModelResponse{} diff --git a/go/samples/model-garden/main.go b/go/samples/model-garden/main.go index 5212de9d3a..72708d60db 100644 --- a/go/samples/model-garden/main.go +++ b/go/samples/model-garden/main.go @@ -34,7 +34,7 @@ func main() { } cfg := &modelgarden.Config{ - Location: "us-east5", + Location: "us-east5", // or us-central1 Models: []string{"claude-3-5-sonnet-v2", "claude-3-5-sonnet"}, } if err := modelgarden.Init(ctx, g, cfg); err != nil { From b95226e93036ebf155ea17f76565949c1d0af96b Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Thu, 13 Feb 2025 16:36:50 +0000 Subject: [PATCH 14/26] draft: add tooling, media and system prompts features to anthropic --- go/plugins/vertexai/modelgarden/anthropic.go | 93 ++++++++++++++++---- 1 file changed, 77 insertions(+), 16 deletions(-) diff --git a/go/plugins/vertexai/modelgarden/anthropic.go b/go/plugins/vertexai/modelgarden/anthropic.go index 431607abc1..03817f01e2 100644 --- a/go/plugins/vertexai/modelgarden/anthropic.go +++ b/go/plugins/vertexai/modelgarden/anthropic.go @@ -12,6 +12,7 @@ import ( "github.com/firebase/genkit/go/ai" "github.com/firebase/genkit/go/genkit" "github.com/firebase/genkit/go/plugins/internal/gemini" + "github.com/firebase/genkit/go/plugins/internal/uri" "github.com/anthropics/anthropic-sdk-go" "github.com/anthropics/anthropic-sdk-go/vertex" @@ -171,27 +172,87 @@ func toAnthropicRequest(model string, i *ai.ModelRequest) anthropic.MessageNewPa req.StopSequences = anthropic.F(c.StopSequences) } } - // system and user blocks - sysBlocks := []anthropic.TextBlockParam{} - userBlocks := []anthropic.TextBlockParam{} + + /* + * // check all messages and split system and user in different blocks + * for _, m := range i.Messages { + * parts, err := convertParts(m.Content) + * if err != nil { + * return nil, err + * } + * + * if m.Role == system { + * systemBlocks = append(parts) + * continue + * } + * + * userBlocks = append(parts) + * } + * + * func convertParts(c []*ai.Part) ([]anthropic.ContentBlockParamUnion, error) { + * parts := make([]anthropic.ContentBlockParamUnion, 0, len(c)) + * for _, p := range c { + * switch { + * detect part type and append it to a list + * } + * } + * + * return parts + * } + */ + // find a way to make this generic for both user and system blocks + sysBlocks := []anthropic.ContentBlockParamUnion{} for _, m := range i.Messages { - // TODO: convert messages to its types (text, media, toolResponse) + // system parts if m.Role == ai.RoleSystem { - sysBlocks = append(sysBlocks, anthropic.NewTextBlock(m.Text())) - } - if m.Role == ai.RoleUser { - userBlocks = append(userBlocks, anthropic.NewTextBlock(m.Text())) + for _, p := range m.Content { + switch { + case p.IsText(): + // TODO: check this + sysBlocks = append(sysBlocks, anthropic.NewTextBlock(p.Text)) + case p.IsMedia(): + // TODO: check this + contentType, data, _ := uri.Data(p) + sysBlocks = append(sysBlocks, anthropic.NewImageBlockBase64(contentType, string(data))) + case p.IsData(): + // todo: what is this? is this related to ContentBlocks? + panic("data content is unsupported by anthropic models") + case p.IsToolResponse(): + // TODO: check this + toolResp := p.ToolResponse + if toolResp.Output == nil { + panic("tool response is empty") + } + data, err := json.Marshal(toolResp.Output) + if err != nil { + panic("unable to parse tool response") + } + sysBlocks = append(sysBlocks, anthropic.NewToolResultBlock(toolResp.Name, string(data), false)) + case p.IsToolRequest(): + // TODO: check this + toolReq := p.ToolRequest + sysBlocks = append(sysBlocks, anthropic.NewToolUseBlockParam(toolReq.Name, toolReq.Name, toolReq.Input)) + default: + panic("unknown part type in the request") + } + } } } - if len(sysBlocks) > 0 { - req.System = anthropic.F(sysBlocks) - } - if len(userBlocks) > 0 { - messageParam := make([]anthropic.MessageParam, 0) - for _, u := range userBlocks { - messageParam = append(messageParam, anthropic.NewUserMessage(u)) + + if len(i.Messages) > 0 { + // check why this is required + last := i.Messages[len(i.Messages)-1] + for _, p := range last.Content { + switch { + case p.IsText(): + case p.IsMedia(): + case p.IsData(): + case p.IsToolResponse(): + case p.IsToolRequest(): + default: + panic("unknown part type in the request") + } } - req.Messages = anthropic.F(messageParam) } return req From 946b360b82dc4d7687e9c9eeda7ddb2eea44bc84 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Thu, 13 Feb 2025 21:27:30 +0000 Subject: [PATCH 15/26] feat: add system and user roles + media --- go/plugins/vertexai/modelgarden/anthropic.go | 139 ++++++++---------- .../vertexai/modelgarden/modelgarden_test.go | 10 +- 2 files changed, 70 insertions(+), 79 deletions(-) diff --git a/go/plugins/vertexai/modelgarden/anthropic.go b/go/plugins/vertexai/modelgarden/anthropic.go index 03817f01e2..44980a8faf 100644 --- a/go/plugins/vertexai/modelgarden/anthropic.go +++ b/go/plugins/vertexai/modelgarden/anthropic.go @@ -7,7 +7,6 @@ import ( "context" "encoding/json" "fmt" - "math" "github.com/firebase/genkit/go/ai" "github.com/firebase/genkit/go/genkit" @@ -18,6 +17,8 @@ import ( "github.com/anthropics/anthropic-sdk-go/vertex" ) +const MaxNumberOfTokens = 8192 + // supported anthropic models var AnthropicModels = map[string]ai.ModelInfo{ "claude-3-5-sonnet-v2": { @@ -134,7 +135,7 @@ func generate( switch delta := event.Delta.(type) { case anthropic.ContentBlockDeltaEventDelta: if delta.Text != "" { - fmt.Printf(delta.Text) + fmt.Print(delta.Text) } } } @@ -149,7 +150,7 @@ func toAnthropicRequest(model string, i *ai.ModelRequest) anthropic.MessageNewPa // minimum required data to perform a request req.Model = anthropic.F(anthropic.Model(model)) - req.MaxTokens = anthropic.F(int64(math.MaxInt64)) + req.MaxTokens = anthropic.F(int64(MaxNumberOfTokens)) if c, ok := i.Config.(*ai.GenerationCommonConfig); ok && c != nil { if c.MaxOutputTokens != 0 { @@ -173,91 +174,75 @@ func toAnthropicRequest(model string, i *ai.ModelRequest) anthropic.MessageNewPa } } - /* - * // check all messages and split system and user in different blocks - * for _, m := range i.Messages { - * parts, err := convertParts(m.Content) - * if err != nil { - * return nil, err - * } - * - * if m.Role == system { - * systemBlocks = append(parts) - * continue - * } - * - * userBlocks = append(parts) - * } - * - * func convertParts(c []*ai.Part) ([]anthropic.ContentBlockParamUnion, error) { - * parts := make([]anthropic.ContentBlockParamUnion, 0, len(c)) - * for _, p := range c { - * switch { - * detect part type and append it to a list - * } - * } - * - * return parts - * } - */ - // find a way to make this generic for both user and system blocks - sysBlocks := []anthropic.ContentBlockParamUnion{} + // check user and system messages + sysBlocks := []anthropic.TextBlockParam{} + userBlocks := []anthropic.ContentBlockParamUnion{} for _, m := range i.Messages { - // system parts - if m.Role == ai.RoleSystem { - for _, p := range m.Content { - switch { - case p.IsText(): - // TODO: check this - sysBlocks = append(sysBlocks, anthropic.NewTextBlock(p.Text)) - case p.IsMedia(): - // TODO: check this - contentType, data, _ := uri.Data(p) - sysBlocks = append(sysBlocks, anthropic.NewImageBlockBase64(contentType, string(data))) - case p.IsData(): - // todo: what is this? is this related to ContentBlocks? - panic("data content is unsupported by anthropic models") - case p.IsToolResponse(): - // TODO: check this - toolResp := p.ToolResponse - if toolResp.Output == nil { - panic("tool response is empty") - } - data, err := json.Marshal(toolResp.Output) - if err != nil { - panic("unable to parse tool response") - } - sysBlocks = append(sysBlocks, anthropic.NewToolResultBlock(toolResp.Name, string(data), false)) - case p.IsToolRequest(): - // TODO: check this - toolReq := p.ToolRequest - sysBlocks = append(sysBlocks, anthropic.NewToolUseBlockParam(toolReq.Name, toolReq.Name, toolReq.Input)) - default: - panic("unknown part type in the request") - } + switch m.Role { + case ai.RoleSystem: + // text blocks only supported for system messages + sysBlocks = append(sysBlocks, anthropic.NewTextBlock(m.Text())) + case ai.RoleUser: + parts, err := convertParts(m.Content) + if err != nil { + return req } + userBlocks = append(userBlocks, parts...) } } - if len(i.Messages) > 0 { - // check why this is required - last := i.Messages[len(i.Messages)-1] - for _, p := range last.Content { - switch { - case p.IsText(): - case p.IsMedia(): - case p.IsData(): - case p.IsToolResponse(): - case p.IsToolRequest(): - default: - panic("unknown part type in the request") - } + if len(sysBlocks) > 0 { + req.System = anthropic.F(sysBlocks) + } + if len(userBlocks) > 0 { + messageParam := make([]anthropic.MessageParam, 0, len(userBlocks)) + for _, m := range userBlocks { + messageParam = append(messageParam, anthropic.NewUserMessage(m)) } + req.Messages = anthropic.F(messageParam) } return req } +// convertParts translates [ai.Part] to an anthropic.ContentBlockParamUnion type +func convertParts(parts []*ai.Part) ([]anthropic.ContentBlockParamUnion, error) { + blocks := []anthropic.ContentBlockParamUnion{} + + for _, p := range parts { + switch { + case p.IsText(): + blocks = append(blocks, anthropic.NewTextBlock(p.Text)) + case p.IsMedia(): + // TODO: check this + contentType, data, _ := uri.Data(p) + blocks = append(blocks, anthropic.NewImageBlockBase64(contentType, string(data))) + case p.IsData(): + // todo: what is this? is this related to ContentBlocks? + panic("data content is unsupported by anthropic models") + case p.IsToolResponse(): + // TODO: check this + toolResp := p.ToolResponse + if toolResp.Output == nil { + panic("tool response is empty") + } + data, err := json.Marshal(toolResp.Output) + if err != nil { + panic("unable to parse tool response") + } + blocks = append(blocks, anthropic.NewToolResultBlock(toolResp.Name, string(data), false)) + case p.IsToolRequest(): + // TODO: check this + toolReq := p.ToolRequest + blocks = append(blocks, anthropic.NewToolUseBlockParam(toolReq.Name, toolReq.Name, toolReq.Input)) + default: + panic("unknown part type in the request") + } + } + + return blocks, nil +} + // toGenkitResponse translates an Anthropic Message to [ai.ModelResponse] func toGenkitResponse(m *anthropic.Message) *ai.ModelResponse { r := &ai.ModelResponse{} diff --git a/go/plugins/vertexai/modelgarden/modelgarden_test.go b/go/plugins/vertexai/modelgarden/modelgarden_test.go index e6bc0823e7..bf27bd1541 100644 --- a/go/plugins/vertexai/modelgarden/modelgarden_test.go +++ b/go/plugins/vertexai/modelgarden/modelgarden_test.go @@ -6,6 +6,7 @@ package modelgarden_test import ( "context" "flag" + "strings" "testing" "github.com/firebase/genkit/go/ai" @@ -47,17 +48,22 @@ func TestModelGarden(t *testing.T) { t.Run("model version ok", func(t *testing.T) { m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") - _, err := genkit.Generate(ctx, g, + resp, err := genkit.Generate(ctx, g, ai.WithConfig(&ai.GenerationCommonConfig{ Temperature: 1, Version: "claude-3-5-sonnet-v2@20241022", }), ai.WithModel(m), - ai.WithTextPrompt("Hello there..."), + ai.WithSystemPrompt("talk to me like an evil pirate and say ARR several times"), + ai.WithMessages(ai.NewUserMessage(ai.NewTextPart("I'm a fish"))), ) if err != nil { t.Fatal(err) } + + if !strings.Contains(resp.Text(), "ARR") { + t.Fatalf("not a pirate :( :%s", resp.Text()) + } }) t.Run("model version nok", func(t *testing.T) { From 90ca4a65cc3dedaf5a3322b1173b6d9d06a527ae Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Thu, 13 Feb 2025 22:10:28 +0000 Subject: [PATCH 16/26] feat: add media support and test cases --- go/plugins/vertexai/modelgarden/anthropic.go | 3 +- .../vertexai/modelgarden/modelgarden_test.go | 49 ++++++++++++++++--- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/go/plugins/vertexai/modelgarden/anthropic.go b/go/plugins/vertexai/modelgarden/anthropic.go index 44980a8faf..913840e2bc 100644 --- a/go/plugins/vertexai/modelgarden/anthropic.go +++ b/go/plugins/vertexai/modelgarden/anthropic.go @@ -5,6 +5,7 @@ package modelgarden import ( "context" + "encoding/base64" "encoding/json" "fmt" @@ -216,7 +217,7 @@ func convertParts(parts []*ai.Part) ([]anthropic.ContentBlockParamUnion, error) case p.IsMedia(): // TODO: check this contentType, data, _ := uri.Data(p) - blocks = append(blocks, anthropic.NewImageBlockBase64(contentType, string(data))) + blocks = append(blocks, anthropic.NewImageBlockBase64(contentType, base64.StdEncoding.EncodeToString(data))) case p.IsData(): // todo: what is this? is this related to ContentBlocks? panic("data content is unsupported by anthropic models") diff --git a/go/plugins/vertexai/modelgarden/modelgarden_test.go b/go/plugins/vertexai/modelgarden/modelgarden_test.go index bf27bd1541..a807459ddb 100644 --- a/go/plugins/vertexai/modelgarden/modelgarden_test.go +++ b/go/plugins/vertexai/modelgarden/modelgarden_test.go @@ -5,7 +5,10 @@ package modelgarden_test import ( "context" + "encoding/base64" "flag" + "io" + "net/http" "strings" "testing" @@ -67,7 +70,6 @@ func TestModelGarden(t *testing.T) { }) t.Run("model version nok", func(t *testing.T) { - t.Skipf("no -projectid provided") m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") _, err := genkit.Generate(ctx, g, ai.WithConfig(&ai.GenerationCommonConfig{ @@ -81,18 +83,30 @@ func TestModelGarden(t *testing.T) { } }) - t.Run("model", func(t *testing.T) { - t.Skipf("no -projectid provided") + t.Run("media content", func(t *testing.T) { + i, err := fetchImgAsBase64() + if err != nil { + t.Fatal(err) + } m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") - resp, err := genkit.Generate(ctx, g, ai.WithTextPrompt("What's your name?"), ai.WithModel(m)) + resp, err := genkit.Generate(ctx, g, + ai.WithSystemPrompt("You are a professional image detective that talks like an evil pirate that does not like tv shows, your task is to detect what's in the image"), + ai.WithModel(m), + ai.WithMessages( + ai.NewUserMessage( + ai.NewTextPart("do you know who's in the image?"), + ai.NewMediaPart("", "data:image/png;base64,"+i)))) if err != nil { t.Fatal(err) } - t.Fatal(resp.Message.Content[0].Text) + + if !strings.Contains(resp.Text(), "Bluey") { + t.Fatalf("it should've said Bluey but got: %s", resp.Text()) + } }) t.Run("streaming", func(t *testing.T) { - t.Skipf("no -projectid provided") + t.Skipf("no streaming support yet") m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") out := "" parts := 0 @@ -123,3 +137,26 @@ func TestModelGarden(t *testing.T) { } }) } + +// Bluey rocks +func fetchImgAsBase64() (string, error) { + imgUrl := "https://www.bluey.tv/wp-content/uploads/2023/07/Bluey.png" + resp, err := http.Get(imgUrl) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", err + } + + // keep the img in memory + imageBytes, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + base64string := base64.StdEncoding.EncodeToString(imageBytes) + return base64string, nil +} From cf917e1e98f038f84e1cc4acbe74029ff47209eb Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Tue, 18 Feb 2025 02:40:58 +0000 Subject: [PATCH 17/26] misc: tool flow complete, disabled tests -- prepare for refactor --- go/plugins/vertexai/modelgarden/anthropic.go | 93 +++++++++++-------- .../vertexai/modelgarden/modelgarden_test.go | 29 +++++- 2 files changed, 83 insertions(+), 39 deletions(-) diff --git a/go/plugins/vertexai/modelgarden/anthropic.go b/go/plugins/vertexai/modelgarden/anthropic.go index 913840e2bc..d3ac20e2ef 100644 --- a/go/plugins/vertexai/modelgarden/anthropic.go +++ b/go/plugins/vertexai/modelgarden/anthropic.go @@ -6,19 +6,23 @@ package modelgarden import ( "context" "encoding/base64" - "encoding/json" "fmt" + "regexp" "github.com/firebase/genkit/go/ai" "github.com/firebase/genkit/go/genkit" "github.com/firebase/genkit/go/plugins/internal/gemini" "github.com/firebase/genkit/go/plugins/internal/uri" + "github.com/invopop/jsonschema" "github.com/anthropics/anthropic-sdk-go" "github.com/anthropics/anthropic-sdk-go/vertex" ) -const MaxNumberOfTokens = 8192 +const ( + MaxNumberOfTokens = 8192 + ToolNameRegex = `^[a-zA-Z0-9_-]{1,64}$` +) // supported anthropic models var AnthropicModels = map[string]ai.ModelInfo{ @@ -126,20 +130,6 @@ func generate( r := toGenkitResponse(msg) r.Request = input return r, nil - } else { - stream := client.Messages.NewStreaming(ctx, req) - msg := anthropic.Message{} - for stream.Next() { - event := stream.Current() - msg.Accumulate(event) - - switch delta := event.Delta.(type) { - case anthropic.ContentBlockDeltaEventDelta: - if delta.Text != "" { - fmt.Print(delta.Text) - } - } - } } return nil, nil @@ -189,6 +179,8 @@ func toAnthropicRequest(model string, i *ai.ModelRequest) anthropic.MessageNewPa return req } userBlocks = append(userBlocks, parts...) + case ai.RoleTool: + fmt.Printf("toAnthropicRequest: ai.RoleTool message found: %s\n", m.Text()) } } @@ -203,9 +195,48 @@ func toAnthropicRequest(model string, i *ai.ModelRequest) anthropic.MessageNewPa req.Messages = anthropic.F(messageParam) } + // check tools + tools, err := convertTools(i.Tools) + if err != nil { + return req + } + req.Tools = anthropic.F(tools) + return req } +// convertTools translates [ai.ToolDefinition] to an anthropic.ToolParam type +func convertTools(tools []*ai.ToolDefinition) ([]anthropic.ToolParam, error) { + resp := make([]anthropic.ToolParam, 0) + regex := regexp.MustCompile(ToolNameRegex) + + for _, t := range tools { + if t.Name == "" { + return nil, fmt.Errorf("tool name is required") + } + if !regex.MatchString(t.Name) { + return nil, fmt.Errorf("tool name must match regex: %s", ToolNameRegex) + } + + resp = append(resp, anthropic.ToolParam{ + Name: anthropic.F(t.Name), + Description: anthropic.F(t.Description), + InputSchema: anthropic.F(generateSchema[map[string]any]()), + }) + } + + return resp, nil +} + +func generateSchema[T any]() interface{} { + reflector := jsonschema.Reflector{ + AllowAdditionalProperties: false, + DoNotReference: true, + } + var v T + return reflector.Reflect(v) +} + // convertParts translates [ai.Part] to an anthropic.ContentBlockParamUnion type func convertParts(parts []*ai.Part) ([]anthropic.ContentBlockParamUnion, error) { blocks := []anthropic.ContentBlockParamUnion{} @@ -215,27 +246,11 @@ func convertParts(parts []*ai.Part) ([]anthropic.ContentBlockParamUnion, error) case p.IsText(): blocks = append(blocks, anthropic.NewTextBlock(p.Text)) case p.IsMedia(): - // TODO: check this contentType, data, _ := uri.Data(p) blocks = append(blocks, anthropic.NewImageBlockBase64(contentType, base64.StdEncoding.EncodeToString(data))) case p.IsData(): // todo: what is this? is this related to ContentBlocks? panic("data content is unsupported by anthropic models") - case p.IsToolResponse(): - // TODO: check this - toolResp := p.ToolResponse - if toolResp.Output == nil { - panic("tool response is empty") - } - data, err := json.Marshal(toolResp.Output) - if err != nil { - panic("unable to parse tool response") - } - blocks = append(blocks, anthropic.NewToolResultBlock(toolResp.Name, string(data), false)) - case p.IsToolRequest(): - // TODO: check this - toolReq := p.ToolRequest - blocks = append(blocks, anthropic.NewToolUseBlockParam(toolReq.Name, toolReq.Name, toolReq.Input)) default: panic("unknown part type in the request") } @@ -268,12 +283,10 @@ func toGenkitResponse(m *anthropic.Message) *ai.ModelResponse { case anthropic.ContentBlockTypeText: p = ai.NewTextPart(string(part.Text)) case anthropic.ContentBlockTypeToolUse: - t := &ai.ToolRequest{} - err := json.Unmarshal([]byte(part.Input), &t.Input) - if err != nil { - return nil - } - p = ai.NewToolRequestPart(t) + p = ai.NewToolResponsePart(&ai.ToolResponse{ + Name: part.Name, + Output: part.JSON, + }) default: panic(fmt.Sprintf("unknown part: %#v", part)) } @@ -281,5 +294,9 @@ func toGenkitResponse(m *anthropic.Message) *ai.ModelResponse { } r.Message = msg + r.Usage = &ai.GenerationUsage{ + InputTokens: int(m.Usage.InputTokens), + OutputTokens: int(m.Usage.OutputTokens), + } return r } diff --git a/go/plugins/vertexai/modelgarden/modelgarden_test.go b/go/plugins/vertexai/modelgarden/modelgarden_test.go index a807459ddb..0281ea4bfd 100644 --- a/go/plugins/vertexai/modelgarden/modelgarden_test.go +++ b/go/plugins/vertexai/modelgarden/modelgarden_test.go @@ -7,6 +7,7 @@ import ( "context" "encoding/base64" "flag" + "fmt" "io" "net/http" "strings" @@ -22,6 +23,7 @@ var ( location = flag.String("location", "us-east5", "Geographic location") ) +// go test . -v -projectid="my_projectId" func TestModelGarden(t *testing.T) { if *projectID == "" { t.Skipf("no -projectid provided") @@ -43,6 +45,7 @@ func TestModelGarden(t *testing.T) { } t.Run("invalid model", func(t *testing.T) { + t.Skipf("no streaming support yet") m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-not-valid-v2") if m != nil { t.Fatal("model should have been invalid") @@ -50,6 +53,7 @@ func TestModelGarden(t *testing.T) { }) t.Run("model version ok", func(t *testing.T) { + t.Skipf("no streaming support yet") m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") resp, err := genkit.Generate(ctx, g, ai.WithConfig(&ai.GenerationCommonConfig{ @@ -70,6 +74,7 @@ func TestModelGarden(t *testing.T) { }) t.Run("model version nok", func(t *testing.T) { + t.Skipf("no streaming support yet") m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") _, err := genkit.Generate(ctx, g, ai.WithConfig(&ai.GenerationCommonConfig{ @@ -84,13 +89,14 @@ func TestModelGarden(t *testing.T) { }) t.Run("media content", func(t *testing.T) { + t.Skipf("no streaming support yet") i, err := fetchImgAsBase64() if err != nil { t.Fatal(err) } m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") resp, err := genkit.Generate(ctx, g, - ai.WithSystemPrompt("You are a professional image detective that talks like an evil pirate that does not like tv shows, your task is to detect what's in the image"), + ai.WithSystemPrompt("You are a professional image detective that talks like an evil pirate that does not like tv shows, your task is to tell the name of the character in the image"), ai.WithModel(m), ai.WithMessages( ai.NewUserMessage( @@ -105,6 +111,27 @@ func TestModelGarden(t *testing.T) { } }) + t.Run("tools", func(t *testing.T) { + m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") + myJokeTool := genkit.DefineTool( + g, + "myJoke", + "When the user asks for a joke, this tool must be used to tell a joke", + func(ctx *ai.ToolContext, input *any) (string, error) { + return "huehue joke: nil", nil + }, + ) + resp, err := genkit.Generate(ctx, g, + ai.WithModel(m), + ai.WithTextPrompt("tell me a joke"), + ai.WithTools(myJokeTool)) + if err != nil { + t.Fatal(err) + } + + fmt.Printf("resp: %s\n\n", resp.Text()) + }) + t.Run("streaming", func(t *testing.T) { t.Skipf("no streaming support yet") m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") From 5a0758e5261a6e3fe795823975e84c648dd0fd0d Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Wed, 19 Feb 2025 02:26:57 +0000 Subject: [PATCH 18/26] feat: add tools support (no response yet) --- go/plugins/vertexai/modelgarden/anthropic.go | 98 ++++++++++++------- .../vertexai/modelgarden/modelgarden_test.go | 6 +- 2 files changed, 66 insertions(+), 38 deletions(-) diff --git a/go/plugins/vertexai/modelgarden/anthropic.go b/go/plugins/vertexai/modelgarden/anthropic.go index d3ac20e2ef..10b42ad16d 100644 --- a/go/plugins/vertexai/modelgarden/anthropic.go +++ b/go/plugins/vertexai/modelgarden/anthropic.go @@ -6,6 +6,7 @@ package modelgarden import ( "context" "encoding/base64" + "encoding/json" "fmt" "regexp" @@ -117,8 +118,10 @@ func generate( input *ai.ModelRequest, cb func(context.Context, *ai.ModelResponseChunk) error, ) (*ai.ModelResponse, error) { - // parse configuration - req := toAnthropicRequest(model, input) + req, err := toAnthropicRequest(model, input) + if err != nil { + panic(fmt.Sprintf("unable to generate anthropic request: %v", err)) + } // no streaming if cb == nil { @@ -129,15 +132,30 @@ func generate( r := toGenkitResponse(msg) r.Request = input + return r, nil } return nil, nil } +func toAnthropicRole(role ai.Role) anthropic.MessageParamRole { + switch role { + case ai.RoleUser: + return anthropic.MessageParamRoleUser + case ai.RoleModel: + return anthropic.MessageParamRoleAssistant + case ai.RoleTool: + return anthropic.MessageParamRoleAssistant + default: + panic(fmt.Sprintf("unsupported role type: %v", role)) + } +} + // toAnthropicRequest translates [ai.ModelRequest] to an Anthropic request -func toAnthropicRequest(model string, i *ai.ModelRequest) anthropic.MessageNewParams { +func toAnthropicRequest(model string, i *ai.ModelRequest) (anthropic.MessageNewParams, error) { req := anthropic.MessageNewParams{} + messages := make([]anthropic.MessageParam, 0) // minimum required data to perform a request req.Model = anthropic.F(anthropic.Model(model)) @@ -165,44 +183,44 @@ func toAnthropicRequest(model string, i *ai.ModelRequest) anthropic.MessageNewPa } } - // check user and system messages + // configure system prompt (if given) sysBlocks := []anthropic.TextBlockParam{} - userBlocks := []anthropic.ContentBlockParamUnion{} - for _, m := range i.Messages { - switch m.Role { - case ai.RoleSystem: - // text blocks only supported for system messages - sysBlocks = append(sysBlocks, anthropic.NewTextBlock(m.Text())) - case ai.RoleUser: - parts, err := convertParts(m.Content) + // userBlocks := []anthropic.ContentBlockParamUnion{} + toolBlocks := []anthropic.ContentBlockParamUnion{} + for _, message := range i.Messages { + if message.Role == ai.RoleSystem { + // only text is supported for system messages + sysBlocks = append(sysBlocks, anthropic.NewTextBlock(message.Text())) + } else if message.Content[len(message.Content)-1].IsToolResponse() { + parts, err := convertParts(message.Content) + if err != nil { + return req, err + } + toolBlocks = append(toolBlocks, parts...) + messages = append(messages, anthropic.NewUserMessage(toolBlocks...)) + } else { + parts, err := convertParts(message.Content) if err != nil { - return req + return req, err } - userBlocks = append(userBlocks, parts...) - case ai.RoleTool: - fmt.Printf("toAnthropicRequest: ai.RoleTool message found: %s\n", m.Text()) + messages = append(messages, anthropic.MessageParam{ + Role: anthropic.F(toAnthropicRole(message.Role)), + Content: anthropic.F(parts), + }) } } - if len(sysBlocks) > 0 { - req.System = anthropic.F(sysBlocks) - } - if len(userBlocks) > 0 { - messageParam := make([]anthropic.MessageParam, 0, len(userBlocks)) - for _, m := range userBlocks { - messageParam = append(messageParam, anthropic.NewUserMessage(m)) - } - req.Messages = anthropic.F(messageParam) - } + req.System = anthropic.F(sysBlocks) + req.Messages = anthropic.F(messages) // check tools tools, err := convertTools(i.Tools) if err != nil { - return req + return req, err } req.Tools = anthropic.F(tools) - return req + return req, nil } // convertTools translates [ai.ToolDefinition] to an anthropic.ToolParam type @@ -251,6 +269,16 @@ func convertParts(parts []*ai.Part) ([]anthropic.ContentBlockParamUnion, error) case p.IsData(): // todo: what is this? is this related to ContentBlocks? panic("data content is unsupported by anthropic models") + case p.IsToolRequest(): + toolReq := p.ToolRequest + blocks = append(blocks, anthropic.NewToolUseBlockParam(toolReq.Ref, toolReq.Name, toolReq.Input)) + case p.IsToolResponse(): + toolResp := p.ToolResponse + output, err := json.Marshal(toolResp.Output) + if err != nil { + panic(fmt.Sprintf("unable to parse tool response: %v", err)) + } + blocks = append(blocks, anthropic.NewToolResultBlock(toolResp.Ref, string(output), false)) default: panic("unknown part type in the request") } @@ -269,23 +297,26 @@ func toGenkitResponse(m *anthropic.Message) *ai.ModelResponse { case anthropic.MessageStopReasonStopSequence: r.FinishReason = ai.FinishReasonStop case anthropic.MessageStopReasonEndTurn: + r.FinishReason = ai.FinishReasonStop case anthropic.MessageStopReasonToolUse: - r.FinishReason = ai.FinishReasonOther + r.FinishReason = ai.FinishReasonStop default: r.FinishReason = ai.FinishReasonUnknown } msg := &ai.Message{} - msg.Role = ai.Role(m.Role) + msg.Role = ai.RoleModel for _, part := range m.Content { var p *ai.Part switch part.Type { case anthropic.ContentBlockTypeText: p = ai.NewTextPart(string(part.Text)) + fmt.Printf("part: %#v\n\n", p.Text) case anthropic.ContentBlockTypeToolUse: - p = ai.NewToolResponsePart(&ai.ToolResponse{ - Name: part.Name, - Output: part.JSON, + p = ai.NewToolRequestPart(&ai.ToolRequest{ + Ref: part.ID, + Input: part.Input, + Name: part.Name, }) default: panic(fmt.Sprintf("unknown part: %#v", part)) @@ -294,6 +325,7 @@ func toGenkitResponse(m *anthropic.Message) *ai.ModelResponse { } r.Message = msg + fmt.Printf("r.Message: %#v\n\n", r.Message) r.Usage = &ai.GenerationUsage{ InputTokens: int(m.Usage.InputTokens), OutputTokens: int(m.Usage.OutputTokens), diff --git a/go/plugins/vertexai/modelgarden/modelgarden_test.go b/go/plugins/vertexai/modelgarden/modelgarden_test.go index 0281ea4bfd..78963b62b0 100644 --- a/go/plugins/vertexai/modelgarden/modelgarden_test.go +++ b/go/plugins/vertexai/modelgarden/modelgarden_test.go @@ -45,7 +45,6 @@ func TestModelGarden(t *testing.T) { } t.Run("invalid model", func(t *testing.T) { - t.Skipf("no streaming support yet") m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-not-valid-v2") if m != nil { t.Fatal("model should have been invalid") @@ -53,7 +52,6 @@ func TestModelGarden(t *testing.T) { }) t.Run("model version ok", func(t *testing.T) { - t.Skipf("no streaming support yet") m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") resp, err := genkit.Generate(ctx, g, ai.WithConfig(&ai.GenerationCommonConfig{ @@ -74,7 +72,6 @@ func TestModelGarden(t *testing.T) { }) t.Run("model version nok", func(t *testing.T) { - t.Skipf("no streaming support yet") m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") _, err := genkit.Generate(ctx, g, ai.WithConfig(&ai.GenerationCommonConfig{ @@ -89,7 +86,6 @@ func TestModelGarden(t *testing.T) { }) t.Run("media content", func(t *testing.T) { - t.Skipf("no streaming support yet") i, err := fetchImgAsBase64() if err != nil { t.Fatal(err) @@ -118,7 +114,7 @@ func TestModelGarden(t *testing.T) { "myJoke", "When the user asks for a joke, this tool must be used to tell a joke", func(ctx *ai.ToolContext, input *any) (string, error) { - return "huehue joke: nil", nil + return "do you want a joke? okay, here it is: do you want to hear about pizza? nevermind, it's to cheessy", nil }, ) resp, err := genkit.Generate(ctx, g, From e13790052d9cfa57f39795d8a044d0d68b8c5d28 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Wed, 19 Feb 2025 03:06:08 +0000 Subject: [PATCH 19/26] docs: refined logs --- go/plugins/vertexai/modelgarden/anthropic.go | 11 +++++------ go/plugins/vertexai/modelgarden/modelgarden_test.go | 12 ++++++++---- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/go/plugins/vertexai/modelgarden/anthropic.go b/go/plugins/vertexai/modelgarden/anthropic.go index 10b42ad16d..3d9bee5408 100644 --- a/go/plugins/vertexai/modelgarden/anthropic.go +++ b/go/plugins/vertexai/modelgarden/anthropic.go @@ -185,20 +185,21 @@ func toAnthropicRequest(model string, i *ai.ModelRequest) (anthropic.MessageNewP // configure system prompt (if given) sysBlocks := []anthropic.TextBlockParam{} - // userBlocks := []anthropic.ContentBlockParamUnion{} - toolBlocks := []anthropic.ContentBlockParamUnion{} for _, message := range i.Messages { if message.Role == ai.RoleSystem { // only text is supported for system messages sysBlocks = append(sysBlocks, anthropic.NewTextBlock(message.Text())) } else if message.Content[len(message.Content)-1].IsToolResponse() { + // if the last message is a ToolResponse, the conversation must continue + // and the ToolResponse message must be sent as a user + // see: https://docs.anthropic.com/en/docs/build-with-claude/tool-use#handling-tool-use-and-tool-result-content-blocks parts, err := convertParts(message.Content) if err != nil { return req, err } - toolBlocks = append(toolBlocks, parts...) - messages = append(messages, anthropic.NewUserMessage(toolBlocks...)) + messages = append(messages, anthropic.NewUserMessage(parts...)) } else { + // handle the rest of the messages parts, err := convertParts(message.Content) if err != nil { return req, err @@ -311,7 +312,6 @@ func toGenkitResponse(m *anthropic.Message) *ai.ModelResponse { switch part.Type { case anthropic.ContentBlockTypeText: p = ai.NewTextPart(string(part.Text)) - fmt.Printf("part: %#v\n\n", p.Text) case anthropic.ContentBlockTypeToolUse: p = ai.NewToolRequestPart(&ai.ToolRequest{ Ref: part.ID, @@ -325,7 +325,6 @@ func toGenkitResponse(m *anthropic.Message) *ai.ModelResponse { } r.Message = msg - fmt.Printf("r.Message: %#v\n\n", r.Message) r.Usage = &ai.GenerationUsage{ InputTokens: int(m.Usage.InputTokens), OutputTokens: int(m.Usage.OutputTokens), diff --git a/go/plugins/vertexai/modelgarden/modelgarden_test.go b/go/plugins/vertexai/modelgarden/modelgarden_test.go index 78963b62b0..4ed964bd82 100644 --- a/go/plugins/vertexai/modelgarden/modelgarden_test.go +++ b/go/plugins/vertexai/modelgarden/modelgarden_test.go @@ -45,6 +45,7 @@ func TestModelGarden(t *testing.T) { } t.Run("invalid model", func(t *testing.T) { + t.Skipf("no streaming support yet") m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-not-valid-v2") if m != nil { t.Fatal("model should have been invalid") @@ -52,6 +53,7 @@ func TestModelGarden(t *testing.T) { }) t.Run("model version ok", func(t *testing.T) { + t.Skipf("no streaming support yet") m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") resp, err := genkit.Generate(ctx, g, ai.WithConfig(&ai.GenerationCommonConfig{ @@ -59,7 +61,7 @@ func TestModelGarden(t *testing.T) { Version: "claude-3-5-sonnet-v2@20241022", }), ai.WithModel(m), - ai.WithSystemPrompt("talk to me like an evil pirate and say ARR several times"), + ai.WithSystemPrompt("talk to me like an evil pirate and say ARR several times but be very short"), ai.WithMessages(ai.NewUserMessage(ai.NewTextPart("I'm a fish"))), ) if err != nil { @@ -72,6 +74,7 @@ func TestModelGarden(t *testing.T) { }) t.Run("model version nok", func(t *testing.T) { + t.Skipf("no streaming support yet") m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") _, err := genkit.Generate(ctx, g, ai.WithConfig(&ai.GenerationCommonConfig{ @@ -86,13 +89,14 @@ func TestModelGarden(t *testing.T) { }) t.Run("media content", func(t *testing.T) { + t.Skipf("no streaming support yet") i, err := fetchImgAsBase64() if err != nil { t.Fatal(err) } m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") resp, err := genkit.Generate(ctx, g, - ai.WithSystemPrompt("You are a professional image detective that talks like an evil pirate that does not like tv shows, your task is to tell the name of the character in the image"), + ai.WithSystemPrompt("You are a professional image detective that talks like an evil pirate that does not like tv shows, your task is to tell the name of the character in the image but be very short"), ai.WithModel(m), ai.WithMessages( ai.NewUserMessage( @@ -112,9 +116,9 @@ func TestModelGarden(t *testing.T) { myJokeTool := genkit.DefineTool( g, "myJoke", - "When the user asks for a joke, this tool must be used to tell a joke", + "When the user asks for a joke, this tool must be used to generate a joke, try to come up with a joke that uses the output of the tool", func(ctx *ai.ToolContext, input *any) (string, error) { - return "do you want a joke? okay, here it is: do you want to hear about pizza? nevermind, it's to cheessy", nil + return "why did the chicken cross the road?", nil }, ) resp, err := genkit.Generate(ctx, g, From e77de851e57b0aed37399b7ea76fbb8b343ac2ef Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Wed, 19 Feb 2025 03:50:16 +0000 Subject: [PATCH 20/26] feat: added streaming support --- go/plugins/vertexai/modelgarden/anthropic.go | 28 +++++++++++++++++++ .../vertexai/modelgarden/modelgarden_test.go | 9 ++---- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/go/plugins/vertexai/modelgarden/anthropic.go b/go/plugins/vertexai/modelgarden/anthropic.go index 3d9bee5408..7845e41948 100644 --- a/go/plugins/vertexai/modelgarden/anthropic.go +++ b/go/plugins/vertexai/modelgarden/anthropic.go @@ -134,6 +134,34 @@ func generate( r.Request = input return r, nil + } else { + stream := client.Messages.NewStreaming(ctx, req) + message := anthropic.Message{} + for stream.Next() { + event := stream.Current() + err := message.Accumulate(event) + if err != nil { + panic(err) + } + + switch event := event.AsUnion().(type) { + case anthropic.ContentBlockDeltaEvent: + cb(ctx, &ai.ModelResponseChunk{ + Content: []*ai.Part{ + { + Text: event.Delta.Text, + }, + }, + }) + case anthropic.MessageStopEvent: + r := toGenkitResponse(&message) + r.Request = input + return r, nil + } + } + if stream.Err() != nil { + panic(stream.Err()) + } } return nil, nil diff --git a/go/plugins/vertexai/modelgarden/modelgarden_test.go b/go/plugins/vertexai/modelgarden/modelgarden_test.go index 4ed964bd82..5a5fca7610 100644 --- a/go/plugins/vertexai/modelgarden/modelgarden_test.go +++ b/go/plugins/vertexai/modelgarden/modelgarden_test.go @@ -45,7 +45,6 @@ func TestModelGarden(t *testing.T) { } t.Run("invalid model", func(t *testing.T) { - t.Skipf("no streaming support yet") m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-not-valid-v2") if m != nil { t.Fatal("model should have been invalid") @@ -53,7 +52,6 @@ func TestModelGarden(t *testing.T) { }) t.Run("model version ok", func(t *testing.T) { - t.Skipf("no streaming support yet") m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") resp, err := genkit.Generate(ctx, g, ai.WithConfig(&ai.GenerationCommonConfig{ @@ -74,7 +72,6 @@ func TestModelGarden(t *testing.T) { }) t.Run("model version nok", func(t *testing.T) { - t.Skipf("no streaming support yet") m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") _, err := genkit.Generate(ctx, g, ai.WithConfig(&ai.GenerationCommonConfig{ @@ -89,7 +86,6 @@ func TestModelGarden(t *testing.T) { }) t.Run("media content", func(t *testing.T) { - t.Skipf("no streaming support yet") i, err := fetchImgAsBase64() if err != nil { t.Fatal(err) @@ -133,7 +129,6 @@ func TestModelGarden(t *testing.T) { }) t.Run("streaming", func(t *testing.T) { - t.Skipf("no streaming support yet") m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") out := "" parts := 0 @@ -157,9 +152,9 @@ func TestModelGarden(t *testing.T) { } if out != out2 { - t.Fatalf("streaming and final should containt the same text.\nstreaming: %s\nfinal:%s\n", out, out2) + t.Fatalf("streaming and final should contain the same text.\nstreaming: %s\nfinal:%s\n", out, out2) } - if final.Usage.InputTokens == 0 || final.Usage.OutputTokens == 0 || final.Usage.TotalTokens == 0 { + if final.Usage.InputTokens == 0 || final.Usage.OutputTokens == 0 { t.Fatalf("empty usage stats: %#v", *final.Usage) } }) From 6493ef039ed5b11b1a0e39b9e645dd3da538d576 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Wed, 19 Feb 2025 18:34:09 +0000 Subject: [PATCH 21/26] test: add tools streaming test --- .../vertexai/modelgarden/modelgarden_test.go | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/go/plugins/vertexai/modelgarden/modelgarden_test.go b/go/plugins/vertexai/modelgarden/modelgarden_test.go index 5a5fca7610..59730a82ab 100644 --- a/go/plugins/vertexai/modelgarden/modelgarden_test.go +++ b/go/plugins/vertexai/modelgarden/modelgarden_test.go @@ -158,6 +158,47 @@ func TestModelGarden(t *testing.T) { t.Fatalf("empty usage stats: %#v", *final.Usage) } }) + + t.Run("tools streaming", func(t *testing.T) { + m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") + out := "" + parts := 0 + + myStoryTool := genkit.DefineTool( + g, + "myStory", + "When the user asks for a story, create a story about a frog and a fox that are good friends", + func(ctx *ai.ToolContext, input *any) (string, error) { + return "the fox is named Goph and the frog is called Fred", nil + }, + ) + + final, err := genkit.Generate(ctx, g, + ai.WithTextPrompt("Tell me a short story about a frog and a princess"), + ai.WithModel(m), + ai.WithTools(myStoryTool), + ai.WithStreaming(func(ctx context.Context, c *ai.ModelResponseChunk) error { + parts++ + out += c.Content[0].Text + return nil + }), + ) + if err != nil { + t.Fatal(err) + } + + out2 := "" + for _, p := range final.Message.Content { + out2 += p.Text + } + + if out != out2 { + t.Fatalf("streaming and final should contain the same text.\nstreaming: %s\nfinal:%s\n", out, out2) + } + if final.Usage.InputTokens == 0 || final.Usage.OutputTokens == 0 { + t.Fatalf("empty usage stats: %#v", *final.Usage) + } + }) } // Bluey rocks From f59eb8658104ec9dabcc8570815aeca00a227556 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Wed, 19 Feb 2025 20:38:27 +0000 Subject: [PATCH 22/26] fix: refactor to independant packages --- .../modelgarden/{ => anthropic}/anthropic.go | 18 +++++++++-------- .../modelgarden/{ => client}/client.go | 13 ++---------- .../vertexai/modelgarden/modelgarden.go | 19 ++++++++---------- .../vertexai/modelgarden/modelgarden_test.go | 20 ++++++++++--------- 4 files changed, 31 insertions(+), 39 deletions(-) rename go/plugins/vertexai/modelgarden/{ => anthropic}/anthropic.go (95%) rename go/plugins/vertexai/modelgarden/{ => client}/client.go (88%) diff --git a/go/plugins/vertexai/modelgarden/anthropic.go b/go/plugins/vertexai/modelgarden/anthropic/anthropic.go similarity index 95% rename from go/plugins/vertexai/modelgarden/anthropic.go rename to go/plugins/vertexai/modelgarden/anthropic/anthropic.go index 7845e41948..1843d8f5ba 100644 --- a/go/plugins/vertexai/modelgarden/anthropic.go +++ b/go/plugins/vertexai/modelgarden/anthropic/anthropic.go @@ -1,7 +1,7 @@ // Copyright 2025 Google LLC // SPDX-License-Identifier: Apache-2.0 -package modelgarden +package anthropic import ( "context" @@ -14,6 +14,7 @@ import ( "github.com/firebase/genkit/go/genkit" "github.com/firebase/genkit/go/plugins/internal/gemini" "github.com/firebase/genkit/go/plugins/internal/uri" + "github.com/firebase/genkit/go/plugins/vertexai/modelgarden/client" "github.com/invopop/jsonschema" "github.com/anthropics/anthropic-sdk-go" @@ -21,6 +22,7 @@ import ( ) const ( + ProviderName = "anthropic" MaxNumberOfTokens = 8192 ToolNameRegex = `^[a-zA-Z0-9_-]{1,64}$` ) @@ -57,8 +59,8 @@ var AnthropicModels = map[string]ai.ModelInfo{ // AnthropicClientConfig is the required configuration to create an Anthropic // client type AnthropicClientConfig struct { - Location string - Project string + client.ClientConfig + // expand configuration as required } // AnthropicClient is a mirror struct of Anthropic's client but implements @@ -68,8 +70,8 @@ type AnthropicClient struct { } // Anthropic defines how an Anthropic client is created -var Anthropic = func(config any) (Client, error) { - cfg, ok := config.(*AnthropicClientConfig) +var Anthropic = func(config any) (client.Client, error) { + cfg, ok := config.(*client.ClientConfig) if !ok { return nil, fmt.Errorf("invalid config for Anthropic %T", config) } @@ -87,7 +89,7 @@ func (a *AnthropicClient) DefineModel(g *genkit.Genkit, name string, info *ai.Mo var ok bool mi, ok = AnthropicModels[name] if !ok { - return nil, fmt.Errorf("%s.DefineModel: called with unknown model %q and nil ModelInfo", AnthropicProvider, name) + return nil, fmt.Errorf("%s.DefineModel: called with unknown model %q and nil ModelInfo", ProviderName, name) } } else { mi = *info @@ -97,11 +99,11 @@ func (a *AnthropicClient) DefineModel(g *genkit.Genkit, name string, info *ai.Mo func defineModel(g *genkit.Genkit, client *AnthropicClient, name string, info ai.ModelInfo) ai.Model { meta := &ai.ModelInfo{ - Label: AnthropicProvider + "-" + name, + Label: ProviderName + "-" + name, Supports: info.Supports, Versions: info.Versions, } - return genkit.DefineModel(g, AnthropicProvider, name, meta, func( + return genkit.DefineModel(g, ProviderName, name, meta, func( ctx context.Context, input *ai.ModelRequest, cb func(context.Context, *ai.ModelResponseChunk) error, diff --git a/go/plugins/vertexai/modelgarden/client.go b/go/plugins/vertexai/modelgarden/client/client.go similarity index 88% rename from go/plugins/vertexai/modelgarden/client.go rename to go/plugins/vertexai/modelgarden/client/client.go index cf029b11e4..573ca6ddde 100644 --- a/go/plugins/vertexai/modelgarden/client.go +++ b/go/plugins/vertexai/modelgarden/client/client.go @@ -1,7 +1,7 @@ // Copyright 2025 Google LLC // SPDX-License-Identifier: Apache-2.0 -package modelgarden +package client import ( "errors" @@ -69,16 +69,7 @@ func (f *ClientFactory) CreateClient(config *ClientConfig) (Client, error) { return nil, fmt.Errorf("unknown client type: %s", key) } - var client Client - var err error - switch config.Provider { - // TODO: add providers when needed - case AnthropicProvider: - client, err = creator(&AnthropicClientConfig{ - Location: config.Location, - Project: config.Project, - }) - } + client, err := creator(config) if err != nil { return nil, err } diff --git a/go/plugins/vertexai/modelgarden/modelgarden.go b/go/plugins/vertexai/modelgarden/modelgarden.go index 947860f14a..f6041148f2 100644 --- a/go/plugins/vertexai/modelgarden/modelgarden.go +++ b/go/plugins/vertexai/modelgarden/modelgarden.go @@ -11,11 +11,8 @@ import ( "github.com/firebase/genkit/go/ai" "github.com/firebase/genkit/go/genkit" -) - -const ( - AnthropicProvider = "anthropic" - MistralProvider = "mistral" + "github.com/firebase/genkit/go/plugins/vertexai/modelgarden/anthropic" + "github.com/firebase/genkit/go/plugins/vertexai/modelgarden/client" ) // Config for Model Garden @@ -27,7 +24,7 @@ type Config struct { var state struct { initted bool - clients *ClientFactory // cache for all clients for available providers + clients *client.ClientFactory // cache for all clients for available providers projectID string location string mu sync.Mutex @@ -63,15 +60,15 @@ func Init(ctx context.Context, g *genkit.Genkit, cfg *Config) error { state.location = "us-central1" } - state.clients = NewClientFactory() + state.clients = client.NewClientFactory() state.initted = true for _, m := range cfg.Models { // ANTHROPIC - if info, ok := AnthropicModels[m]; ok { - state.clients.Register(AnthropicProvider, Anthropic) + if info, ok := anthropic.AnthropicModels[m]; ok { + state.clients.Register(anthropic.ProviderName, anthropic.Anthropic) - anthropicClient, err := state.clients.CreateClient(&ClientConfig{ - Provider: AnthropicProvider, + anthropicClient, err := state.clients.CreateClient(&client.ClientConfig{ + Provider: anthropic.ProviderName, Project: state.projectID, Location: state.location, }) diff --git a/go/plugins/vertexai/modelgarden/modelgarden_test.go b/go/plugins/vertexai/modelgarden/modelgarden_test.go index 59730a82ab..f6c6dbe6ad 100644 --- a/go/plugins/vertexai/modelgarden/modelgarden_test.go +++ b/go/plugins/vertexai/modelgarden/modelgarden_test.go @@ -7,7 +7,6 @@ import ( "context" "encoding/base64" "flag" - "fmt" "io" "net/http" "strings" @@ -16,6 +15,7 @@ import ( "github.com/firebase/genkit/go/ai" "github.com/firebase/genkit/go/genkit" "github.com/firebase/genkit/go/plugins/vertexai/modelgarden" + "github.com/firebase/genkit/go/plugins/vertexai/modelgarden/anthropic" ) var ( @@ -45,14 +45,14 @@ func TestModelGarden(t *testing.T) { } t.Run("invalid model", func(t *testing.T) { - m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-not-valid-v2") + m := modelgarden.Model(g, anthropic.ProviderName, "claude-not-valid-v2") if m != nil { t.Fatal("model should have been invalid") } }) t.Run("model version ok", func(t *testing.T) { - m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") + m := modelgarden.Model(g, anthropic.ProviderName, "claude-3-5-sonnet-v2") resp, err := genkit.Generate(ctx, g, ai.WithConfig(&ai.GenerationCommonConfig{ Temperature: 1, @@ -72,7 +72,7 @@ func TestModelGarden(t *testing.T) { }) t.Run("model version nok", func(t *testing.T) { - m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") + m := modelgarden.Model(g, anthropic.ProviderName, "claude-3-5-sonnet-v2") _, err := genkit.Generate(ctx, g, ai.WithConfig(&ai.GenerationCommonConfig{ Temperature: 1, @@ -90,7 +90,7 @@ func TestModelGarden(t *testing.T) { if err != nil { t.Fatal(err) } - m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") + m := modelgarden.Model(g, anthropic.ProviderName, "claude-3-5-sonnet-v2") resp, err := genkit.Generate(ctx, g, ai.WithSystemPrompt("You are a professional image detective that talks like an evil pirate that does not like tv shows, your task is to tell the name of the character in the image but be very short"), ai.WithModel(m), @@ -108,7 +108,7 @@ func TestModelGarden(t *testing.T) { }) t.Run("tools", func(t *testing.T) { - m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") + m := modelgarden.Model(g, anthropic.ProviderName, "claude-3-5-sonnet-v2") myJokeTool := genkit.DefineTool( g, "myJoke", @@ -125,11 +125,13 @@ func TestModelGarden(t *testing.T) { t.Fatal(err) } - fmt.Printf("resp: %s\n\n", resp.Text()) + if len(resp.Text()) == 0 { + t.Fatal("expected a response but nothing was returned") + } }) t.Run("streaming", func(t *testing.T) { - m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") + m := modelgarden.Model(g, anthropic.ProviderName, "claude-3-5-sonnet-v2") out := "" parts := 0 @@ -160,7 +162,7 @@ func TestModelGarden(t *testing.T) { }) t.Run("tools streaming", func(t *testing.T) { - m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") + m := modelgarden.Model(g, anthropic.ProviderName, "claude-3-5-sonnet-v2") out := "" parts := 0 From 28825e89ad5931cf00ad7c26ecc576fb2ec81338 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Wed, 19 Feb 2025 20:47:41 +0000 Subject: [PATCH 23/26] fix: update modelgarden sample --- go/samples/model-garden/main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/samples/model-garden/main.go b/go/samples/model-garden/main.go index 72708d60db..771d89241b 100644 --- a/go/samples/model-garden/main.go +++ b/go/samples/model-garden/main.go @@ -23,6 +23,7 @@ import ( "github.com/firebase/genkit/go/ai" "github.com/firebase/genkit/go/genkit" "github.com/firebase/genkit/go/plugins/vertexai/modelgarden" + "github.com/firebase/genkit/go/plugins/vertexai/modelgarden/anthropic" ) func main() { @@ -43,7 +44,7 @@ func main() { // Define a simple flow that generates jokes about a given topic genkit.DefineFlow(g, "jokesFlow", func(ctx context.Context, input string) (string, error) { - m := modelgarden.Model(g, modelgarden.AnthropicProvider, "claude-3-5-sonnet-v2") + m := modelgarden.Model(g, anthropic.ProviderName, "claude-3-5-sonnet-v2") if m == nil { return "", errors.New("jokesFlow: failed to find model") } From aed1aea07f7eaff91bc07ed5e4181e954f2f5eaf Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Wed, 19 Feb 2025 20:49:30 +0000 Subject: [PATCH 24/26] add go.mod and go.sum --- go/go.mod | 5 +++++ go/go.sum | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/go/go.mod b/go/go.mod index 677dcc6bb7..bd901c6a5e 100644 --- a/go/go.mod +++ b/go/go.mod @@ -15,6 +15,7 @@ require ( firebase.google.com/go/v4 v4.14.1 github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.46.0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.22.0 + github.com/anthropics/anthropic-sdk-go v0.2.0-alpha.10 github.com/aymerick/raymond v2.0.2+incompatible github.com/google/generative-ai-go v0.16.1-0.20240711222609-09946422abc6 github.com/google/go-cmp v0.6.0 @@ -81,6 +82,10 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/tidwall/gjson v1.14.4 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect go.mongodb.org/mongo-driver v1.14.0 // indirect diff --git a/go/go.sum b/go/go.sum index cc53089587..1c8703a4b2 100644 --- a/go/go.sum +++ b/go/go.sum @@ -48,6 +48,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/ankane/disco-go v0.1.0 h1:nkz+y4O+UFKnEGH8FkJ8wcVwX5boZvaRzJN6EMK7NVw= github.com/ankane/disco-go v0.1.0/go.mod h1:nkR7DLW+KkXeRRAsWk6poMTpTOWp9/4iKYGDwg8dSS0= +github.com/anthropics/anthropic-sdk-go v0.2.0-alpha.10 h1:myWicO7qECViRePrrsSijlakZK3q7vzHBCoS2hL+8V0= +github.com/anthropics/anthropic-sdk-go v0.2.0-alpha.10/go.mod h1:GJxtdOs9K4neo8Gg65CjJ7jNautmldGli5/OFNabOoo= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= @@ -258,7 +260,17 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= github.com/uptrace/bun v1.1.12 h1:sOjDVHxNTuM6dNGaba0wUuz7KvDE1BmNu9Gqs2gJSXQ= From 0482fc420bd37e9b187646e693926acd0a74ed7a Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Mon, 24 Feb 2025 19:52:39 +0000 Subject: [PATCH 25/26] docs: updated license in samples --- go/samples/basic-gemini/main.go | 13 +------------ go/samples/model-garden/main.go | 13 +------------ 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/go/samples/basic-gemini/main.go b/go/samples/basic-gemini/main.go index daf2a67083..43d50a9d93 100644 --- a/go/samples/basic-gemini/main.go +++ b/go/samples/basic-gemini/main.go @@ -1,16 +1,5 @@ // Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package main diff --git a/go/samples/model-garden/main.go b/go/samples/model-garden/main.go index 771d89241b..62645b54dc 100644 --- a/go/samples/model-garden/main.go +++ b/go/samples/model-garden/main.go @@ -1,16 +1,5 @@ // Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package main From d3b29c6626564f64ce7e4526d6dc12a7624dc2b0 Mon Sep 17 00:00:00 2001 From: Hugo Aguirre Parra Date: Tue, 25 Feb 2025 16:56:00 +0000 Subject: [PATCH 26/26] feat: add support for claude-3-7-sonnet --- .../vertexai/modelgarden/anthropic/anthropic.go | 5 +++++ .../vertexai/modelgarden/modelgarden_test.go | 14 +++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/go/plugins/vertexai/modelgarden/anthropic/anthropic.go b/go/plugins/vertexai/modelgarden/anthropic/anthropic.go index 1843d8f5ba..4dd52ebfd2 100644 --- a/go/plugins/vertexai/modelgarden/anthropic/anthropic.go +++ b/go/plugins/vertexai/modelgarden/anthropic/anthropic.go @@ -54,6 +54,11 @@ var AnthropicModels = map[string]ai.ModelInfo{ Supports: &gemini.Multimodal, Versions: []string{"claude-3-opus@20240229"}, }, + "claude-3-7-sonnet": { + Label: "Vertex AI Model Garden - Claude 3.7 Sonnet", + Supports: &gemini.Multimodal, + Versions: []string{"claude-3-7-sonnet@20250219"}, + }, } // AnthropicClientConfig is the required configuration to create an Anthropic diff --git a/go/plugins/vertexai/modelgarden/modelgarden_test.go b/go/plugins/vertexai/modelgarden/modelgarden_test.go index f6c6dbe6ad..e3e70a2052 100644 --- a/go/plugins/vertexai/modelgarden/modelgarden_test.go +++ b/go/plugins/vertexai/modelgarden/modelgarden_test.go @@ -38,7 +38,7 @@ func TestModelGarden(t *testing.T) { err = modelgarden.Init(ctx, g, &modelgarden.Config{ ProjectID: *projectID, Location: *location, - Models: []string{"claude-3-5-sonnet-v2"}, + Models: []string{"claude-3-7-sonnet"}, }) if err != nil { t.Fatal(err) @@ -52,11 +52,11 @@ func TestModelGarden(t *testing.T) { }) t.Run("model version ok", func(t *testing.T) { - m := modelgarden.Model(g, anthropic.ProviderName, "claude-3-5-sonnet-v2") + m := modelgarden.Model(g, anthropic.ProviderName, "claude-3-7-sonnet") resp, err := genkit.Generate(ctx, g, ai.WithConfig(&ai.GenerationCommonConfig{ Temperature: 1, - Version: "claude-3-5-sonnet-v2@20241022", + Version: "claude-3-7-sonnet@20250219", }), ai.WithModel(m), ai.WithSystemPrompt("talk to me like an evil pirate and say ARR several times but be very short"), @@ -90,7 +90,7 @@ func TestModelGarden(t *testing.T) { if err != nil { t.Fatal(err) } - m := modelgarden.Model(g, anthropic.ProviderName, "claude-3-5-sonnet-v2") + m := modelgarden.Model(g, anthropic.ProviderName, "claude-3-7-sonnet") resp, err := genkit.Generate(ctx, g, ai.WithSystemPrompt("You are a professional image detective that talks like an evil pirate that does not like tv shows, your task is to tell the name of the character in the image but be very short"), ai.WithModel(m), @@ -108,7 +108,7 @@ func TestModelGarden(t *testing.T) { }) t.Run("tools", func(t *testing.T) { - m := modelgarden.Model(g, anthropic.ProviderName, "claude-3-5-sonnet-v2") + m := modelgarden.Model(g, anthropic.ProviderName, "claude-3-7-sonnet") myJokeTool := genkit.DefineTool( g, "myJoke", @@ -131,7 +131,7 @@ func TestModelGarden(t *testing.T) { }) t.Run("streaming", func(t *testing.T) { - m := modelgarden.Model(g, anthropic.ProviderName, "claude-3-5-sonnet-v2") + m := modelgarden.Model(g, anthropic.ProviderName, "claude-3-7-sonnet") out := "" parts := 0 @@ -162,7 +162,7 @@ func TestModelGarden(t *testing.T) { }) t.Run("tools streaming", func(t *testing.T) { - m := modelgarden.Model(g, anthropic.ProviderName, "claude-3-5-sonnet-v2") + m := modelgarden.Model(g, anthropic.ProviderName, "claude-3-7-sonnet") out := "" parts := 0