From 814f50af3844b9303427d8c1283af065b1060667 Mon Sep 17 00:00:00 2001 From: pandyamarut Date: Fri, 6 Sep 2024 11:03:47 -0700 Subject: [PATCH] fix oai completion api error Signed-off-by: pandyamarut --- src/engine.py | 3 ++- src/utils.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/engine.py b/src/engine.py index 0c20dc3..d170edf 100644 --- a/src/engine.py +++ b/src/engine.py @@ -176,7 +176,8 @@ async def _handle_chat_or_completion_request(self, openai_request: JobInput): yield create_error_response(str(e)).model_dump() return - response_generator = await generator_function(request, raw_request=None) + dummy_request = DummyRequest() + response_generator = await generator_function(request, raw_request=dummy_request) if not openai_request.openai_input.get("stream") or isinstance(response_generator, ErrorResponse): yield response_generator.model_dump() diff --git a/src/utils.py b/src/utils.py index efe7611..220be90 100644 --- a/src/utils.py +++ b/src/utils.py @@ -49,9 +49,11 @@ def __init__(self, job): self.openai_input = job.get("openai_input") class DummyRequest: + def __init__(self): + self.headers = {} async def is_disconnected(self): return False - + class BatchSize: def __init__(self, max_batch_size, min_batch_size, batch_size_growth_factor): self.max_batch_size = max_batch_size