Merge branch 'main' into release

Author: bolasim

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "truss"
-version = "0.5.4"
+version = "0.5.5"
 description = "A seamless bridge from model development to model delivery"
 license = "MIT"
 readme = "README.md"

truss/templates/control/control/endpoints.py

@@ -5,7 +5,7 @@
 from fastapi import APIRouter
 from fastapi.responses import JSONResponse, StreamingResponse
 from helpers.errors import ModelLoadFailed, ModelNotReady
-from httpx import URL, ConnectError
+from httpx import URL, ConnectError, RemoteProtocolError
 from starlette.requests import Request
 from starlette.responses import Response
 from tenacity import Retrying, retry_if_exception_type, stop_after_attempt, wait_fixed
@@ -45,6 +45,7 @@ async def proxy(request: Request):
         retry=(
             retry_if_exception_type(ConnectError)
             | retry_if_exception_type(ModelNotReady)
+            | retry_if_exception_type(RemoteProtocolError)
         ),
         stop=stop_after_attempt(INFERENCE_SERVER_START_WAIT_SECS),
         wait=wait_fixed(1),
@@ -59,7 +60,7 @@ async def proxy(request: Request):
 
                 if await _is_model_not_ready(resp):
                     raise ModelNotReady("Model has started running, but not ready yet.")
-            except ConnectionError as exp:
+            except (RemoteProtocolError, ConnectError) as exp:
                 # This check is a bit expensive so we don't do it before every request, we
                 # do it only if request fails with connection error. If the inference server
                 # process is running then we continue waiting for it to start (by retrying),

truss/templates/control/control/helpers/inference_server_process_controller.py

@@ -19,6 +19,7 @@ class InferenceServerProcessController:
     _inference_server_home: str
     _app_logger: logging.Logger
     _inference_server_process_args: List[str]
+    _logged_unrecoverable_since_last_restart: bool
 
     def __init__(
         self,
@@ -33,6 +34,7 @@ def __init__(
         self._inference_server_started = False
         self._inference_server_ever_started = False
         self._inference_server_terminated = False
+        self._logged_unrecoverable_since_last_restart = False
         self._app_logger = app_logger
 
     def start(self, inf_env: dict):
@@ -45,6 +47,7 @@ def start(self, inf_env: dict):
 
             self._inference_server_started = True
             self._inference_server_ever_started = True
+            self._logged_unrecoverable_since_last_restart = False
 
     def stop(self):
         if self._inference_server_process is not None:
@@ -106,4 +109,8 @@ def check_and_recover_inference_server(self, inf_env: dict):
                 )
                 self.start(inf_env)
             else:
-                self._app_logger.warning("Inference server unrecoverable. Try patching")
+                if not self._logged_unrecoverable_since_last_restart:
+                    self._app_logger.warning(
+                        "Inference server unrecoverable. Try patching"
+                    )
+                    self._logged_unrecoverable_since_last_restart = True

truss/templates/server/model_wrapper.py

@@ -20,7 +20,7 @@
 
 MODEL_BASENAME = "model"
 
-NUM_LOAD_RETRIES = int(os.environ.get("NUM_LOAD_RETRIES_TRUSS", "3"))
+NUM_LOAD_RETRIES = int(os.environ.get("NUM_LOAD_RETRIES_TRUSS", "1"))
 STREAMING_RESPONSE_QUEUE_READ_TIMEOUT_SECS = 60
 DEFAULT_PREDICT_CONCURRENCY = 1