diff --git a/docs/docs/plugins/groq.md b/docs/docs/plugins/groq.md index 821dd71..3902232 100644 --- a/docs/docs/plugins/groq.md +++ b/docs/docs/plugins/groq.md @@ -41,14 +41,14 @@ Install the plugin in your project with your favorite package manager: ```typescript import dotenv from 'dotenv'; import { genkit } from 'genkit'; -import { groq, gemma7b } from 'genkitx-groq'; +import { groq, gemma2x9b } from 'genkitx-groq'; dotenv.config(); const ai = genkit({ plugins: [groq({ apiKey: process.env.GROQ_API_KEY })], // optional: default model for generate calls - model: gemma7b, + model: gemma2x9b, }); ``` diff --git a/examples/groq/src/index.ts b/examples/groq/src/index.ts index a8f3b92..171a261 100644 --- a/examples/groq/src/index.ts +++ b/examples/groq/src/index.ts @@ -16,13 +16,13 @@ import dotenv from 'dotenv'; import { genkit, z } from 'genkit'; -import { groq, gemma7b } from 'genkitx-groq'; +import { groq, gemma2x9b } from 'genkitx-groq'; dotenv.config(); const ai = genkit({ plugins: [groq({ apiKey: process.env.GROQ_API_KEY })], - model: gemma7b, + model: gemma2x9b, }); // genkit flow:run jokeFlow \"chicken\" diff --git a/lerna.json b/lerna.json index b31a121..a11366e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,9 +1,5 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", "version": "0.15.0", - "packages": [ - "plugins/*", - "docs", - "examples/*" - ] -} \ No newline at end of file + "packages": ["plugins/*", "docs", "examples/*"] +} diff --git a/package-lock.json b/package-lock.json index c7f1a07..73346ad 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11832,11 +11832,6 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "license": "MIT" }, - "node_modules/base-64": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz", - "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==" - }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -12460,15 +12455,6 @@ "dev": true, "license": "MIT" }, - "node_modules/charenc": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", - "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", - "license": "BSD-3-Clause", - "engines": { - "node": "*" - } - }, "node_modules/cheerio": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0.tgz", @@ -13822,15 +13808,6 @@ "node": ">= 8" } }, - "node_modules/crypt": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", - "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", - "license": "BSD-3-Clause", - "engines": { - "node": "*" - } - }, "node_modules/crypto-random-string": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-4.0.0.tgz", @@ -14677,16 +14654,6 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/digest-fetch": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz", - "integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==", - "license": "ISC", - "dependencies": { - "base-64": "^0.1.0", - "md5": "^2.3.0" - } - }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -17676,20 +17643,18 @@ } }, "node_modules/groq-sdk": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/groq-sdk/-/groq-sdk-0.3.3.tgz", - "integrity": "sha512-wdOeZ2QymPjjP3tmFpUAnfMisoLbt7xF2MfpROeFAngcqWbfTyB9j9pMWSEAMF/E4gZx8f2Y+5zswO0q92CSxA==", + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/groq-sdk/-/groq-sdk-0.15.0.tgz", + "integrity": "sha512-aYDEdr4qczx3cLCRRe+Beb37I7g/9bD5kHF+EEDxcrREWw1vKoRcfP3vHEkJB7Ud/8oOuF0scRwDpwWostTWuQ==", "license": "Apache-2.0", "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", - "digest-fetch": "^1.3.0", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7", - "web-streams-polyfill": "^3.2.1" + "node-fetch": "^2.6.7" } }, "node_modules/groq-sdk/node_modules/@types/node": { @@ -19072,12 +19037,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-buffer": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", - "license": "MIT" - }, "node_modules/is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", @@ -22209,17 +22168,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/md5": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", - "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", - "license": "BSD-3-Clause", - "dependencies": { - "charenc": "0.0.2", - "crypt": "0.0.2", - "is-buffer": "~1.1.6" - } - }, "node_modules/mdast-util-directive": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/mdast-util-directive/-/mdast-util-directive-3.0.0.tgz", @@ -34757,7 +34705,7 @@ "version": "0.15.0", "license": "Apache-2.0", "dependencies": { - "groq-sdk": "^0.3.3" + "groq-sdk": "^0.15.0" }, "devDependencies": { "@types/hast": "^3.0.4", diff --git a/plugins/groq/README.md b/plugins/groq/README.md index a318ee9..f59600c 100644 --- a/plugins/groq/README.md +++ b/plugins/groq/README.md @@ -38,14 +38,14 @@ Install the plugin in your project with your favorite package manager: ```typescript import dotenv from 'dotenv'; import { genkit } from 'genkit'; -import { groq, gemma7b } from 'genkitx-groq'; +import { groq, gemma2x9b } from 'genkitx-groq'; dotenv.config(); const ai = genkit({ plugins: [groq({ apiKey: process.env.GROQ_API_KEY })], // optional: default model for generate calls - model: gemma7b, + model: gemma2x9b, }); ``` diff --git a/plugins/groq/package.json b/plugins/groq/package.json index 457e81b..7e85b17 100644 --- a/plugins/groq/package.json +++ b/plugins/groq/package.json @@ -24,7 +24,7 @@ "author": "TheFireCo", "license": "Apache-2.0", "dependencies": { - "groq-sdk": "^0.3.3" + "groq-sdk": "^0.15.0" }, "peerDependencies": { "genkit": "^0.9.0 || ^1.0.0" diff --git a/plugins/groq/src/groq_models.ts b/plugins/groq/src/groq_models.ts index 7c7f2c5..7d0316d 100644 --- a/plugins/groq/src/groq_models.ts +++ b/plugins/groq/src/groq_models.ts @@ -14,12 +14,14 @@ * limitations under the License. */ import Groq from 'groq-sdk'; -import { ChatCompletionChunk } from 'groq-sdk/lib/chat_completions_ext.mjs'; -import { ChatCompletionCreateParamsBase } from 'groq-sdk/resources/chat/completions.mjs'; import { - ChatCompletion, - CompletionCreateParams, -} from 'groq-sdk/resources/chat/index.mjs'; + ChatCompletionChunk, + ChatCompletionCreateParamsBase, + ChatCompletionMessageParam, + ChatCompletionMessageToolCall, + ChatCompletionTool, +} from 'groq-sdk/resources/chat/completions.mjs'; +import { ChatCompletion } from 'groq-sdk/resources/chat/index.mjs'; import { GenerateRequest, GenerationCommonConfigSchema, @@ -66,6 +68,142 @@ export const llama3x8b = modelRef({ version: 'llama3-8b-8192', }); +export const llamaGuard3x8b = modelRef({ + name: 'groq/llama-guard-3-8b', + info: { + versions: ['llama-guard-3-8b'], + label: 'Llama Guard 3 8B', + supports: { + multiturn: true, + tools: false, // Could be true but not recommended + media: false, + systemRole: true, + output: ['text', 'json'], // JSON mode does not support streaming or stop sequences + }, + }, + configSchema: GroqConfigSchema, + version: 'llama-guard-3-8b', +}); + +export const llama33x70bVersatile = modelRef({ + name: 'groq/llama-3.3-70b-versatile', + info: { + versions: ['llama-3.3-70b-versatile'], + label: 'Llama 3.3 70B Versatile', + supports: { + multiturn: true, + tools: false, // Could be true but not recommended + media: false, + systemRole: true, + output: ['text', 'json'], // JSON mode does not support streaming or stop sequences + }, + }, + configSchema: GroqConfigSchema, + version: 'llama-3.3-70b-versatile', +}); + +export const llama33x70bSpecdec = modelRef({ + name: 'groq/llama-3.3-70b-specdec', + info: { + versions: ['llama-3.3-70b-specdec'], + label: 'Llama 3.3 70B SpecDec', + supports: { + multiturn: true, + tools: false, // Could be true but not recommended + media: false, + systemRole: true, + output: ['text', 'json'], // JSON mode does not support streaming or stop sequences + }, + }, + configSchema: GroqConfigSchema, + version: 'llama-3.3-70b-specdec', +}); + +export const llama32x90bVisionPreview = modelRef({ + name: 'groq/llama-3.2-90b-vision-preview', + info: { + versions: ['llama-3.2-90b-vision-preview'], + label: 'Llama 3.2 90B Vision Preview', + supports: { + multiturn: true, + tools: false, // Could be true but not recommended + media: false, + systemRole: true, + output: ['text', 'json'], // JSON mode does not support streaming or stop sequences + }, + }, + configSchema: GroqConfigSchema, + version: 'llama-3.2-90b-vision-preview', +}); + +export const llama32x11bVisionPreview = modelRef({ + name: 'groq/llama-3.2-11b-vision-preview', + info: { + versions: ['llama-3.2-11b-vision-preview'], + label: 'Llama 3.2 11B Vision Preview', + supports: { + multiturn: true, + tools: false, // Could be true but not recommended + media: false, + systemRole: true, + output: ['text', 'json'], // JSON mode does not support streaming or stop sequences + }, + }, + configSchema: GroqConfigSchema, + version: 'llama-3.2-11b-vision-preview', +}); + +export const llama32x3bPreview = modelRef({ + name: 'groq/llama-3.2-3b-preview', + info: { + versions: ['llama-3.2-3b-preview'], + label: 'Llama 3.2 3B Preview', + supports: { + multiturn: true, + tools: false, // Could be true but not recommended + media: false, + systemRole: true, + output: ['text', 'json'], // JSON mode does not support streaming or stop sequences + }, + }, + configSchema: GroqConfigSchema, + version: 'llama-3.2-3b-preview', +}); + +export const llama32x1bPreview = modelRef({ + name: 'groq/llama-3.2-1b-preview', + info: { + versions: ['llama-3.2-1b-preview'], + label: 'Llama 3.2 1B Preview', + supports: { + multiturn: true, + tools: false, // Could be true but not recommended + media: false, + systemRole: true, + output: ['text', 'json'], // JSON mode does not support streaming or stop sequences + }, + }, + configSchema: GroqConfigSchema, + version: 'llama-3.2-1b-preview', +}); + +export const llama31x8bInstant = modelRef({ + name: 'groq/llama-3.1-8b-instant', + info: { + versions: ['llama-3.1-8b-instant'], + label: 'Llama 3.1 8B Instant', + supports: { + multiturn: true, + tools: false, // Could be true but not recommended + media: false, + systemRole: true, + output: ['text', 'json'], // JSON mode does not support streaming or stop sequences + }, + }, + configSchema: GroqConfigSchema, + version: 'llama-3.1-8b-instant', +}); + // Worst at JSON mode // Only model recommended for Tool Use export const llama3x70b = modelRef({ @@ -104,11 +242,11 @@ export const mixtral8x7b = modelRef({ }); // Runner up at JSON mode -export const gemma7b = modelRef({ - name: 'groq/gemma-7b', +export const gemma2x9b = modelRef({ + name: 'groq/gemma2-9b', info: { - versions: ['gemma-7b-it'], - label: 'Gemma 7B IT', + versions: ['gemma2-9b-it'], + label: 'Gemma 2 9B', supports: { multiturn: true, tools: false, @@ -118,14 +256,94 @@ export const gemma7b = modelRef({ }, }, configSchema: GroqConfigSchema, - version: 'gemma-7b-it', + version: 'gemma2-9b-it', +}); + +export const qwen25x32b = modelRef({ + name: 'groq/qwen-2.5-32b', + info: { + versions: ['qwen-2.5-32b'], + label: 'Qwen 2.5 32B', + supports: { + multiturn: true, + tools: true, + media: false, + systemRole: true, + output: ['text', 'json'], // JSON mode does not support streaming or stop sequences + }, + }, + configSchema: GroqConfigSchema, + version: 'qwen-2.5-32b', +}); + +export const qwen25coderx32b = modelRef({ + name: 'groq/qwen-2.5-coder-32b', + info: { + versions: ['qwen-2.5-coder-32b'], + label: 'Qwen 2.5 Coder 32B', + supports: { + multiturn: true, + tools: true, + media: false, + systemRole: true, + output: ['text', 'json'], // JSON mode does not support streaming or stop sequences + }, + }, + configSchema: GroqConfigSchema, + version: 'qwen-2.5-coder-32b', +}); + +export const deepseekR1DistillQwenx32b = modelRef({ + name: 'groq/deepseek-r1-distill-qwen-32b', + info: { + versions: ['deepseek-r1-distill-qwen-32b'], + label: 'Deepseek R1 Distill Qwen 32B', + supports: { + multiturn: true, + tools: true, + media: false, + systemRole: true, + output: ['text', 'json'], // JSON mode does not support streaming or stop sequences + }, + }, + configSchema: GroqConfigSchema, + version: 'deepseek-r1-distill-qwen-32b', +}); + +export const deepseekR1DistillLlamax70b = modelRef({ + name: 'groq/deepseek-r1-distill-llama-70b', + info: { + versions: ['deepseek-r1-distill-llama-70b'], + label: 'Deepseek R1 Distill Llama 70B', + supports: { + multiturn: true, + tools: true, + media: false, + systemRole: true, + output: ['text', 'json'], // JSON mode does not support streaming or stop sequences + }, + }, + configSchema: GroqConfigSchema, + version: 'deepseek-r1-distill-llama-70b', }); export const SUPPORTED_GROQ_MODELS = { 'llama-3-8b': llama3x8b, 'llama-3-70b': llama3x70b, + 'llama-guard-3-8b': llamaGuard3x8b, + 'llama-3.3-70b-versatile': llama33x70bVersatile, + 'llama-3.3-70b-specdec': llama33x70bSpecdec, + 'llama-3.2-90b-vision-preview': llama32x90bVisionPreview, + 'llama-3.2-11b-vision-preview': llama32x11bVisionPreview, + 'llama-3.2-3b-preview': llama32x3bPreview, + 'llama-3.2-1b-preview': llama32x1bPreview, + 'llama-3.1-8b-instant': llama31x8bInstant, 'mixtral-8-7b': mixtral8x7b, - 'gemma-7b': gemma7b, + 'gemma2-9b': gemma2x9b, + 'qwen-2.5-32b': qwen25x32b, + 'qwen-2.5-coder-32b': qwen25coderx32b, + 'deepseek-r1-distill-qwen-32b': deepseekR1DistillQwenx32b, + 'deepseek-r1-distill-llama-70b': deepseekR1DistillLlamax70b, }; /** @@ -155,7 +373,7 @@ export function toGroqRole(role: Role): 'system' | 'user' | 'assistant' { * @param tool - The tool definition containing the name, description, and input schema. * @returns A Groq tool object formatted for use in completion creation parameters. */ -export function toGroqTool(tool: ToolDefinition): CompletionCreateParams.Tool { +export function toGroqTool(tool: ToolDefinition): ChatCompletionTool { return { type: 'function', function: { @@ -194,8 +412,8 @@ export function toGroqTextAndMedia(part: Part): string { */ export function toGroqMessages( messages: MessageData[] -): CompletionCreateParams.Message[] { - const groqMsgs: CompletionCreateParams.Message[] = []; +): ChatCompletionMessageParam[] { + const groqMsgs: ChatCompletionMessageParam[] = []; for (const message of messages) { const msg = new Message(message); switch (msg.role) { @@ -212,7 +430,7 @@ export function toGroqMessages( }); break; case 'model': - const toolCalls: CompletionCreateParams.Message.ToolCall[] = msg.content + const toolCalls: ChatCompletionMessageToolCall[] = msg.content .filter((part) => part.toolRequest) .map((part) => { if (!part.toolRequest) { @@ -247,7 +465,6 @@ export function toGroqMessages( toolResponseParts.map((part) => { groqMsgs.push({ role: toGroqRole(message.role), - tool_call_id: part.toolResponse.ref || '', content: typeof part.toolResponse.output === 'string' ? part.toolResponse.output @@ -273,7 +490,6 @@ const FINISH_REASON_MAP: Record< length: 'length', tool_calls: 'stop', function_call: 'stop', - content_filter: 'blocked', }; /** @@ -284,7 +500,7 @@ const FINISH_REASON_MAP: Record< */ function fromGroqToolCall( toolCall: - | ChatCompletion.Choice.Message.ToolCall + | ChatCompletionMessageToolCall | ChatCompletionChunk.Choice.Delta.ToolCall ) { if (!toolCall.function) { @@ -324,8 +540,8 @@ function fromGroqChoice( ? (toolRequestParts as ToolRequestPart[]) : [ jsonMode - ? { data: JSON.parse(choice.message.content) } - : { text: choice.message.content }, + ? { data: JSON.parse(choice.message.content || '{}') } + : { text: choice.message.content || '' }, ], }, custom: {}, @@ -370,18 +586,6 @@ export function toGroqRequestBody( modelName: string, request: GenerateRequest ): ChatCompletionCreateParamsBase { - const mapToSnakeCase = >( - obj: T - ): Record => { - return Object.entries(obj).reduce((acc, [key, value]) => { - const snakeCaseKey = key.replace( - /[A-Z]/g, - (letter) => `_${letter.toLowerCase()}` - ); - acc[snakeCaseKey] = value; - return acc; - }, {}); - }; const model = SUPPORTED_GROQ_MODELS[modelName]; if (!model) throw new Error(`Unsupported model: ${modelName}`); @@ -490,10 +694,12 @@ export function groqModel(ai: Genkit, name: string, client: Groq) { logprobs: choice.logprobs as ChatCompletion.Choice.Logprobs, message: { content: choice.delta.content || '', - role: 'model', - tool_calls: choice.delta.tool_calls, + role: 'assistant', + tool_calls: choice.delta.tool_calls?.filter( + (tc) => tc.type === 'function' && !!tc.function && !!tc.id + ) as ChatCompletionMessageToolCall[] | undefined, }, - finish_reason: choice.finish_reason || 'unknown', + finish_reason: choice.finish_reason || 'stop', }); const c = fromGroqChunkChoice(choice); streamingCallback({ @@ -511,6 +717,7 @@ export function groqModel(ai: Genkit, name: string, client: Groq) { completion_tokens: totalCompletionTokens, total_tokens: totalPromptTokens + totalCompletionTokens, }, + object: 'chat.completion', }; // TODO: find a way to get the final completion (current approach is a bit hacky) - issue here: https://github.com/groq/groq-typescript/issues/29 // response = await stream.finalChatCompletion(); diff --git a/plugins/groq/src/index.ts b/plugins/groq/src/index.ts index 79c4dc4..b867b9a 100644 --- a/plugins/groq/src/index.ts +++ b/plugins/groq/src/index.ts @@ -20,15 +20,44 @@ import { groqModel, llama3x70b, llama3x8b, - gemma7b, + llamaGuard3x8b, + llama33x70bVersatile, + llama33x70bSpecdec, + llama32x90bVisionPreview, + llama32x11bVisionPreview, + llama32x3bPreview, + llama32x1bPreview, + llama31x8bInstant, + gemma2x9b, mixtral8x7b, + qwen25x32b, + qwen25coderx32b, + deepseekR1DistillQwenx32b, + deepseekR1DistillLlamax70b, SUPPORTED_GROQ_MODELS, } from './groq_models'; import { Genkit } from 'genkit'; import { genkitPlugin } from 'genkit/plugin'; // Export models for direct access -export { llama3x70b, llama3x8b, gemma7b, mixtral8x7b }; +export { + llama3x70b, + llama3x8b, + llamaGuard3x8b, + llama33x70bVersatile, + llama33x70bSpecdec, + llama32x90bVisionPreview, + llama32x11bVisionPreview, + llama32x3bPreview, + llama32x1bPreview, + llama31x8bInstant, + gemma2x9b, + mixtral8x7b, + qwen25x32b, + qwen25coderx32b, + deepseekR1DistillQwenx32b, + deepseekR1DistillLlamax70b, +}; // Define the PluginOptions interface for customization of the Groq plugin. // This configuration provides flexibility and defaults for Groq API connectivity. diff --git a/plugins/groq/tests/groq_test.ts b/plugins/groq/tests/groq_test.ts index 75ee179..00f3eb0 100644 --- a/plugins/groq/tests/groq_test.ts +++ b/plugins/groq/tests/groq_test.ts @@ -43,10 +43,7 @@ describe('toGroqMessages', () => { { role: 'user', content: 'Hello, world!' }, { role: 'assistant', content: 'How can I assist you today?' }, ]; - assert.deepStrictEqual( - toGroqMessages(messages).map(({ tool_call_id, ...rest }) => rest), - expectedOutput - ); + assert.deepStrictEqual(toGroqMessages(messages), expectedOutput); }); });