Merge pull request #673 from basetenlabs/bump-version-0.7.8

Release 0.7.8

Author: squidarth

docs/guides/base-images.mdx

@@ -0,0 +1,71 @@
+---
+title: How to use base images
+description: "A guide to configuring a base image for your truss"
+---
+
+
+Model serving enviroments will often be standardized as container images to avoid wrangling python, system, and other requirements needed to run your model on every deploy.
+Leverage your existing container artifacts by bringing your own base image to Truss.
+
+## Setting a base image in config.yaml
+
+To specify a base image to build a truss container image from in your `config.yaml` configure a `base_image`.
+
+```yaml config.yaml
+base_image:
+  image: <image_name:tag>
+  python_executable_path: <path-to-python>
+```
+
+where `python_executable_path` is a path to a python executable with which to run your server.
+
+## Example usage
+
+This [example truss](https://github.com/basetenlabs/truss/tree/main/examples/nemo-titanet) demonstrates how to properly configure a base image for Nvidia NeMo TitaNet:
+
+```yaml config.yaml
+base_image:
+  image: nvcr.io/nvidia/nemo:23.03
+  python_executable_path: /usr/bin/python
+apply_library_patches: true
+bundled_packages_dir: packages
+data_dir: data
+requirements:
+- PySoundFile
+live_reload: false
+resources:
+  accelerator: T4
+  cpu: 2500m
+  memory: 4512Mi
+  use_gpu: true
+secrets: {}
+spec_version: '2.0'
+system_packages:
+- python3.8-venv
+```
+
+## Configuring private base images with build time secrets
+
+Secrets of the form `DOCKER_REGISTRY_<REGISTRY_URL>` will be supplied to your model build to authenticate image pulls from private container registries.
+For information on where to store secret values see the [secrets guide](secrets#storing-secrets-on-your-remote).
+
+For example, to configure docker credentials to a private dockerhub repository your `config.yaml` should include the following secret and placeholder:
+
+```yaml config.yaml
+secrets:
+  DOCKER_REGISTRY_https://index.docker.io/v1/: null
+```
+
+along with a configured Baseten secret `DOCKER_REGISTRY_https://index.docker.io/v1/` with a base64 encoded `username:password` secret value:
+
+```sh
+echo -n 'username:password' | base64
+```
+
+To add docker credentials for gcloud artifact registry provide an [access token](https://cloud.google.com/artifact-registry/docs/docker/authentication#token) as the secret value.
+For example, to configure authentication for a repository in `us-west2` your `config.yaml` should include the following secret and placeholder:
+
+```yaml config.yaml
+secrets:
+  DOCKER_REGISTRY_us-west2-docker.pkg.dev: null
+```

docs/mint.json

@@ -67,7 +67,8 @@
     {
       "group": "Guides",
       "pages": [
-        "guides/secrets"
+        "guides/secrets",
+        "guides/base-images"
       ]
     },
     {

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "truss"
-version = "0.7.7"
+version = "0.7.8"
 description = "A seamless bridge from model development to model delivery"
 license = "MIT"
 readme = "README.md"

truss/contexts/image_builder/serving_image_builder.py

@@ -255,6 +255,7 @@ def create_tgi_build_dir(
     data_dir = build_dir / "data"
     credentials_file = data_dir / "service_account.json"
     dockerfile_content = dockerfile_template.render(
+        config=config,
         hf_access_token=hf_access_token,
         models=model_files,
         hf_cache=config.hf_cache,
@@ -312,6 +313,7 @@ def create_vllm_build_dir(
     data_dir = build_dir / "data"
     credentials_file = data_dir / "service_account.json"
     dockerfile_content = dockerfile_template.render(
+        config=config,
         hf_access_token=hf_access_token,
         models=model_files,
         should_install_server_requirements=True,

truss/templates/shared/secrets_resolver.py

@@ -1,7 +1,9 @@
 import os
 from collections.abc import Mapping
 from pathlib import Path
-from typing import Dict
+from typing import Dict, Optional
+
+SECRETS_DOC_LINK = "https://truss.baseten.co/docs/using-secrets"
 
 
 class SecretNotFound(Exception):
@@ -17,7 +19,7 @@ def get_secrets(config: Dict):
         return Secrets(config.get("secrets", {}))
 
     @staticmethod
-    def _resolve_secret(secret_name: str, default_value: str):
+    def _resolve_secret(secret_name: str, default_value: Optional[str]):
         secret_value = default_value
         secret_env_var_name = SecretsResolver.SECRET_ENV_VAR_PREFIX + secret_name
         if secret_env_var_name in os.environ:
@@ -39,15 +41,11 @@ def __init__(self, base_secrets: Dict[str, str]):
 
     def __getitem__(self, key: str) -> str:
         if key not in self._base_secrets:
-            # Note this is the case where the secrets are not specified in
-            # config.yaml
-            raise SecretNotFound(f"Secret '{key}' not specified in the config.")
+            raise SecretNotFound(_secret_missing_error_message(key))
 
         found_secret = SecretsResolver._resolve_secret(key, self._base_secrets[key])
         if not found_secret:
-            raise SecretNotFound(
-                f"Secret '{key}' not found. Please check available secrets."
-            )
+            raise SecretNotFound(_secret_missing_error_message(key))
 
         return found_secret
 
@@ -58,3 +56,13 @@ def __iter__(self):
 
     def __len__(self):
         return len(self._base_secrets)
+
+
+def _secret_missing_error_message(key: str) -> str:
+    return f"""
+Secret '{key}' not found. Please ensure that:
+  * Secret '{key}' is defined in the 'secrets' section of the Truss config file
+  * The model was pushed with the --trusted flag
+  * Secret '{key}' is defined in the secret manager
+ Read more about secrets here: {SECRETS_DOC_LINK}.
+    """

truss/templates/tgi/proxy.conf.jinja

@@ -1,5 +1,6 @@
 
 server {
+    # We use the proxy_read_timeout directive here (instead of proxy_send_timeout) as it sets the timeout for reading a response from the proxied server vs. setting a timeout for sending a request to the proxied server.
     listen 8080;
 
     # Liveness
@@ -11,6 +12,7 @@ server {
     # Readiness
     location ~ ^/v1/models/model$ {
         proxy_redirect off;
+        proxy_read_timeout 300s;
 
         rewrite ^/v1/models/model$ /health break;
 
@@ -20,6 +22,7 @@ server {
     # Predict
     location ~ ^/v1/models/model:predict$ {
         proxy_redirect off;
+        proxy_read_timeout 300s;
 
         rewrite ^/v1/models/model:predict$ /{{endpoint}} break;
 

truss/templates/tgi/tgi.Dockerfile.jinja

@@ -16,6 +16,10 @@ COPY ./proxy.conf /etc/nginx/conf.d/proxy.conf
 RUN mkdir -p /var/log/supervisor
 COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 
+{% for env_var_name, env_var_value in config.environment_variables.items() %}
+ENV {{ env_var_name }} {{ env_var_value }}
+{% endfor %}
+
 ENV SERVER_START_CMD /usr/bin/supervisord
 
 {%- if hf_cache %}

truss/templates/vllm/proxy.conf.jinja

@@ -1,5 +1,5 @@
-
 server {
+    # We use the proxy_read_timeout directive here (instead of proxy_send_timeout) as it sets the timeout for reading a response from the proxied server vs. setting a timeout for sending a request to the proxied server.
     listen 8080;
 
     # Liveness
@@ -11,6 +11,7 @@ server {
     # Readiness
     location ~ ^/v1/models/model$ {
         proxy_redirect off;
+        proxy_read_timeout 300s;
 
         rewrite ^/v1/models/model$ /v1/models break;
 
@@ -20,6 +21,7 @@ server {
     # Predict
     location ~ ^/v1/models/model:predict$ {
         proxy_redirect off;
+        proxy_read_timeout 300s;
 
         rewrite ^/v1/models/model:predict$ {{server_endpoint}} break;
 

truss/templates/vllm/vllm.Dockerfile.jinja

@@ -16,6 +16,10 @@ COPY ./proxy.conf /etc/nginx/conf.d/proxy.conf
 RUN mkdir -p /var/log/supervisor
 COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 
+{% for env_var_name, env_var_value in config.environment_variables.items() %}
+ENV {{ env_var_name }} {{ env_var_value }}
+{% endfor %}
+
 ENV SERVER_START_CMD /usr/bin/supervisord
 
 {%- if hf_cache %}

truss/tests/test_model_inference.py

@@ -336,6 +336,9 @@ def predict(self, request):
     """
 
     config_with_no_secret = "model_name: secrets-truss"
+    missing_secret_error_message = """Secret 'secret' not found. Please ensure that:
+  * Secret 'secret' is defined in the 'secrets' section of the Truss config file
+  * The model was pushed with the --trusted flag"""
 
     with ensure_kill_all(), tempfile.TemporaryDirectory(dir=".") as tmp_work_dir:
         truss_dir = Path(tmp_work_dir, "truss")
@@ -371,7 +374,7 @@ def predict(self, request):
 
         assert "error" in response.json()
 
-        assert_logs_contain_error(container.logs(), "not specified in the config")
+        assert_logs_contain_error(container.logs(), missing_secret_error_message)
         assert "Internal Server Error" in response.json()["error"]
 
     with ensure_kill_all(), tempfile.TemporaryDirectory(dir=".") as tmp_work_dir:
@@ -390,9 +393,7 @@ def predict(self, request):
         response = requests.post(full_url, json={})
         assert response.status_code == 500
 
-        assert_logs_contain_error(
-            container.logs(), "'secret' not found. Please check available secrets."
-        )
+        assert_logs_contain_error(container.logs(), missing_secret_error_message)
         assert "Internal Server Error" in response.json()["error"]
 
 

truss/truss_config.py

@@ -42,6 +42,7 @@
 
 class Accelerator(Enum):
     T4 = "T4"
+    L4 = "L4"
     A10G = "A10G"
     V100 = "V100"
     A100 = "A100"