From 48f77c5798e4728cf31ff1f2b293e4f294d6eb0e Mon Sep 17 00:00:00 2001 From: Justin Hayes Date: Fri, 28 Jun 2024 10:50:39 -0400 Subject: [PATCH 1/8] Fix Gemini --- .../providers/google_manifold_pipeline.py | 90 ++++++++++--------- 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/examples/pipelines/providers/google_manifold_pipeline.py b/examples/pipelines/providers/google_manifold_pipeline.py index 9b15e4d4..02c82e50 100644 --- a/examples/pipelines/providers/google_manifold_pipeline.py +++ b/examples/pipelines/providers/google_manifold_pipeline.py @@ -72,56 +72,62 @@ def update_pipelines(self) -> None: def pipe( self, user_message: str, model_id: str, messages: List[dict], body: dict ) -> Union[str, Iterator]: - """The pipe function (connects open-webui to google-genai) - - Args: - user_message (str): The last message input by the user - model_id (str): The model to use - messages (List[dict]): The chat history - body (dict): The raw request body in OpenAI's "chat/completions" style - - Returns: - str: The complete response - - Yields: - Iterator[str]: Yields a new message part every time it is received - """ - print(f"pipe:{__name__}") system_prompt = None google_messages = [] + for message in messages: - google_role = "" - if message["role"] == "user": - google_role = "user" - elif message["role"] == "assistant": - google_role = "model" - elif message["role"] == "system": + if message["role"] == "system": system_prompt = message["content"] - continue # System promt is not inyected as a message - google_messages.append( - genai.protos.Content( - role=google_role, - parts=[ - genai.protos.Part( - text=message["content"], - ), - ], + continue + + google_role = "user" if message["role"] == "user" else "model" + + try: + content = message.get("content", "") + if isinstance(content, list): + # Handle potential multi-modal content + parts = [] + for item in content: + if item["type"] == "text": + parts.append({"text": item["text"]}) + # Add handling for other content types if necessary + else: + parts = [{"text": content}] + + google_messages.append({ + "role": google_role, + "parts": parts + }) + except Exception as e: + print(f"Error processing message: {e}") + print(f"Problematic message: {message}") + # You might want to skip this message or handle the error differently + + try: + model = genai.GenerativeModel( + f"models/{model_id}", + generation_config=genai.GenerationConfig( + temperature=body.get("temperature", 0.7), + top_p=body.get("top_p", 1.0), + top_k=body.get("top_k", 1), + max_output_tokens=body.get("max_tokens", 1024), ) ) - response = genai.GenerativeModel( - f"models/{model_id}", # we have to add the "models/" part again - system_instruction=system_prompt, - ).generate_content( - google_messages, - stream=body["stream"], - ) + response = model.generate_content( + google_messages, + stream=body["stream"], + ) + + if body["stream"]: + for chunk in response: + yield chunk.text + return "" - if body["stream"]: - for chunk in response: - yield chunk.text - return "" + return response.text - return response.text + except Exception as e: + print(f"Error generating content: {e}") + return f"An error occurred: {str(e)}" From 50132989d0091f1e991d86447a97ccc9c487b32f Mon Sep 17 00:00:00 2001 From: Justin Hayes Date: Fri, 28 Jun 2024 10:52:05 -0400 Subject: [PATCH 2/8] Bump `max_output_tokens` --- examples/pipelines/providers/google_manifold_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pipelines/providers/google_manifold_pipeline.py b/examples/pipelines/providers/google_manifold_pipeline.py index 02c82e50..b4c3b12c 100644 --- a/examples/pipelines/providers/google_manifold_pipeline.py +++ b/examples/pipelines/providers/google_manifold_pipeline.py @@ -112,7 +112,7 @@ def pipe( temperature=body.get("temperature", 0.7), top_p=body.get("top_p", 1.0), top_k=body.get("top_k", 1), - max_output_tokens=body.get("max_tokens", 1024), + max_output_tokens=body.get("max_tokens", 8192), ) ) From c44217b5b8786819d2d47007bcbfb4ee8abc8115 Mon Sep 17 00:00:00 2001 From: Justin Hayes Date: Fri, 28 Jun 2024 10:59:34 -0400 Subject: [PATCH 3/8] Add logging --- .../providers/google_manifold_pipeline.py | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/examples/pipelines/providers/google_manifold_pipeline.py b/examples/pipelines/providers/google_manifold_pipeline.py index b4c3b12c..8686bce0 100644 --- a/examples/pipelines/providers/google_manifold_pipeline.py +++ b/examples/pipelines/providers/google_manifold_pipeline.py @@ -2,11 +2,15 @@ from typing import List, Union, Iterator import os +import logging from pydantic import BaseModel import google.generativeai as genai +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + class Pipeline: """Google GenAI pipeline""" @@ -72,7 +76,8 @@ def update_pipelines(self) -> None: def pipe( self, user_message: str, model_id: str, messages: List[dict], body: dict ) -> Union[str, Iterator]: - print(f"pipe:{__name__}") + logger.info(f"Pipe function called for model: {model_id}") + logger.info(f"Stream mode: {body['stream']}") system_prompt = None google_messages = [] @@ -87,12 +92,10 @@ def pipe( try: content = message.get("content", "") if isinstance(content, list): - # Handle potential multi-modal content parts = [] for item in content: if item["type"] == "text": parts.append({"text": item["text"]}) - # Add handling for other content types if necessary else: parts = [{"text": content}] @@ -101,9 +104,8 @@ def pipe( "parts": parts }) except Exception as e: - print(f"Error processing message: {e}") - print(f"Problematic message: {message}") - # You might want to skip this message or handle the error differently + logger.error(f"Error processing message: {e}") + logger.error(f"Problematic message: {message}") try: model = genai.GenerativeModel( @@ -112,7 +114,7 @@ def pipe( temperature=body.get("temperature", 0.7), top_p=body.get("top_p", 1.0), top_k=body.get("top_k", 1), - max_output_tokens=body.get("max_tokens", 8192), + max_output_tokens=body.get("max_tokens", 1024), ) ) @@ -122,12 +124,16 @@ def pipe( ) if body["stream"]: + logger.info("Streaming response") for chunk in response: yield chunk.text return "" - - return response.text + else: + logger.info("Non-streaming response") + result = response.text + logger.info(f"Generated content: {result}") + return result except Exception as e: - print(f"Error generating content: {e}") + logger.error(f"Error generating content: {e}") return f"An error occurred: {str(e)}" From b82d9021857ed5218fd003be1729c5bf247c1516 Mon Sep 17 00:00:00 2001 From: Justin Hayes Date: Fri, 28 Jun 2024 11:06:08 -0400 Subject: [PATCH 4/8] Print logging --- .../providers/google_manifold_pipeline.py | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/examples/pipelines/providers/google_manifold_pipeline.py b/examples/pipelines/providers/google_manifold_pipeline.py index 8686bce0..916a9611 100644 --- a/examples/pipelines/providers/google_manifold_pipeline.py +++ b/examples/pipelines/providers/google_manifold_pipeline.py @@ -2,15 +2,11 @@ from typing import List, Union, Iterator import os -import logging from pydantic import BaseModel import google.generativeai as genai -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - class Pipeline: """Google GenAI pipeline""" @@ -76,8 +72,8 @@ def update_pipelines(self) -> None: def pipe( self, user_message: str, model_id: str, messages: List[dict], body: dict ) -> Union[str, Iterator]: - logger.info(f"Pipe function called for model: {model_id}") - logger.info(f"Stream mode: {body['stream']}") + print(f"Pipe function called for model: {model_id}") + print(f"Stream mode: {body['stream']}") system_prompt = None google_messages = [] @@ -104,8 +100,8 @@ def pipe( "parts": parts }) except Exception as e: - logger.error(f"Error processing message: {e}") - logger.error(f"Problematic message: {message}") + print(f"Error processing message: {e}") + print(f"Problematic message: {message}") try: model = genai.GenerativeModel( @@ -124,16 +120,19 @@ def pipe( ) if body["stream"]: - logger.info("Streaming response") + print("Streaming response") for chunk in response: yield chunk.text return "" else: - logger.info("Non-streaming response") + print("Non-streaming response") result = response.text - logger.info(f"Generated content: {result}") + print(f"Generated content: {result}") return result except Exception as e: - logger.error(f"Error generating content: {e}") + print(f"Error generating content: {e}") return f"An error occurred: {str(e)}" + + finally: + print("Pipe function completed") From e9cedbfca8c6057898f7d76367997d49542a4437 Mon Sep 17 00:00:00 2001 From: Justin Hayes Date: Fri, 28 Jun 2024 11:09:45 -0400 Subject: [PATCH 5/8] Fix non-streaming response --- examples/pipelines/providers/google_manifold_pipeline.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/pipelines/providers/google_manifold_pipeline.py b/examples/pipelines/providers/google_manifold_pipeline.py index 916a9611..acae5ced 100644 --- a/examples/pipelines/providers/google_manifold_pipeline.py +++ b/examples/pipelines/providers/google_manifold_pipeline.py @@ -121,9 +121,7 @@ def pipe( if body["stream"]: print("Streaming response") - for chunk in response: - yield chunk.text - return "" + return (chunk.text for chunk in response) else: print("Non-streaming response") result = response.text From 20212561a19de22d0b880b12526b912bfd752411 Mon Sep 17 00:00:00 2001 From: Justin Hayes Date: Fri, 28 Jun 2024 11:14:37 -0400 Subject: [PATCH 6/8] Update frontmatter --- .../pipelines/providers/google_manifold_pipeline.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/examples/pipelines/providers/google_manifold_pipeline.py b/examples/pipelines/providers/google_manifold_pipeline.py index acae5ced..38fbfd22 100644 --- a/examples/pipelines/providers/google_manifold_pipeline.py +++ b/examples/pipelines/providers/google_manifold_pipeline.py @@ -1,4 +1,13 @@ -"""A manifold to integrate Google's GenAI models into Open-WebUI""" +""" +title: Google GenAI Manifold Pipeline +author: Marc Lopez +date: 2024-06-06 +version: 1.1 +license: MIT +description: A pipeline for generating text using Google's GenAI models in Open-WebUI. +requirements: google-generativeai +environment_variables: GOOGLE_API_KEY +""" from typing import List, Union, Iterator import os From 130e9cec3586295f05b2f369281ba942db266628 Mon Sep 17 00:00:00 2001 From: Justin Hayes Date: Fri, 28 Jun 2024 11:47:04 -0400 Subject: [PATCH 7/8] Refac --- .../providers/google_manifold_pipeline.py | 108 ++++++++++-------- 1 file changed, 61 insertions(+), 47 deletions(-) diff --git a/examples/pipelines/providers/google_manifold_pipeline.py b/examples/pipelines/providers/google_manifold_pipeline.py index 38fbfd22..c74c668e 100644 --- a/examples/pipelines/providers/google_manifold_pipeline.py +++ b/examples/pipelines/providers/google_manifold_pipeline.py @@ -15,6 +15,7 @@ from pydantic import BaseModel import google.generativeai as genai +from google.generativeai.types import GenerationConfig class Pipeline: @@ -81,65 +82,78 @@ def update_pipelines(self) -> None: def pipe( self, user_message: str, model_id: str, messages: List[dict], body: dict ) -> Union[str, Iterator]: - print(f"Pipe function called for model: {model_id}") - print(f"Stream mode: {body['stream']}") + if not self.valves.GOOGLE_API_KEY: + return "Error: GOOGLE_API_KEY is not set" - system_prompt = None - google_messages = [] + try: + genai.configure(api_key=self.valves.GOOGLE_API_KEY) - for message in messages: - if message["role"] == "system": - system_prompt = message["content"] - continue + if model_id.startswith("google_genai."): + model_id = model_id[12:] + model_id = model_id.lstrip(".") - google_role = "user" if message["role"] == "user" else "model" - - try: - content = message.get("content", "") - if isinstance(content, list): - parts = [] - for item in content: - if item["type"] == "text": - parts.append({"text": item["text"]}) - else: - parts = [{"text": content}] - - google_messages.append({ - "role": google_role, - "parts": parts - }) - except Exception as e: - print(f"Error processing message: {e}") - print(f"Problematic message: {message}") + if not model_id.startswith("gemini-"): + return f"Error: Invalid model name format: {model_id}" - try: - model = genai.GenerativeModel( - f"models/{model_id}", - generation_config=genai.GenerationConfig( - temperature=body.get("temperature", 0.7), - top_p=body.get("top_p", 1.0), - top_k=body.get("top_k", 1), - max_output_tokens=body.get("max_tokens", 1024), - ) + print(f"Pipe function called for model: {model_id}") + print(f"Stream mode: {body.get('stream', False)}") + + system_message = next((msg["content"] for msg in messages if msg["role"] == "system"), None) + + contents = [] + for message in messages: + if message["role"] != "system": + if isinstance(message.get("content"), list): + parts = [] + for content in message["content"]: + if content["type"] == "text": + parts.append({"text": content["text"]}) + elif content["type"] == "image_url": + image_url = content["image_url"]["url"] + if image_url.startswith("data:image"): + image_data = image_url.split(",")[1] + parts.append({"inline_data": {"mime_type": "image/jpeg", "data": image_data}}) + else: + parts.append({"image_url": image_url}) + contents.append({"role": message["role"], "parts": parts}) + else: + contents.append({ + "role": "user" if message["role"] == "user" else "model", + "parts": [{"text": message["content"]}] + }) + + if system_message: + contents.insert(0, {"role": "user", "parts": [{"text": f"System: {system_message}"}]}) + + model = genai.GenerativeModel(model_name=model_id) + + generation_config = GenerationConfig( + temperature=body.get("temperature", 0.7), + top_p=body.get("top_p", 0.9), + top_k=body.get("top_k", 40), + max_output_tokens=body.get("max_tokens", 8192), + stop_sequences=body.get("stop", []), ) + safety_settings = body.get("safety_settings") + response = model.generate_content( - google_messages, - stream=body["stream"], + contents, + generation_config=generation_config, + safety_settings=safety_settings, + stream=body.get("stream", False), ) - if body["stream"]: - print("Streaming response") - return (chunk.text for chunk in response) + if body.get("stream", False): + return self.stream_response(response) else: - print("Non-streaming response") - result = response.text - print(f"Generated content: {result}") - return result + return response.text except Exception as e: print(f"Error generating content: {e}") return f"An error occurred: {str(e)}" - finally: - print("Pipe function completed") + def stream_response(self, response): + for chunk in response: + if chunk.text: + yield chunk.text From c4e2d4933de66226fc8b4ecdc66f111a71d85a87 Mon Sep 17 00:00:00 2001 From: Justin Hayes Date: Fri, 28 Jun 2024 11:47:34 -0400 Subject: [PATCH 8/8] Update metadata --- examples/pipelines/providers/google_manifold_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pipelines/providers/google_manifold_pipeline.py b/examples/pipelines/providers/google_manifold_pipeline.py index c74c668e..66778eed 100644 --- a/examples/pipelines/providers/google_manifold_pipeline.py +++ b/examples/pipelines/providers/google_manifold_pipeline.py @@ -1,6 +1,6 @@ """ title: Google GenAI Manifold Pipeline -author: Marc Lopez +author: Marc Lopez (refactor by justinh-rahb) date: 2024-06-06 version: 1.1 license: MIT