Merge remote-tracking branch 'origin/main' into l4_support

Author: willosa

docs/guides/base-images.mdx

@@ -0,0 +1,71 @@
+---
+title: How to use base images
+description: "A guide to configuring a base image for your truss"
+---
+
+
+Model serving enviroments will often be standardized as container images to avoid wrangling python, system, and other requirements needed to run your model on every deploy.
+Leverage your existing container artifacts by bringing your own base image to Truss.
+
+## Setting a base image in config.yaml
+
+To specify a base image to build a truss container image from in your `config.yaml` configure a `base_image`.
+
+```yaml config.yaml
+base_image:
+  image: <image_name:tag>
+  python_executable_path: <path-to-python>
+```
+
+where `python_executable_path` is a path to a python executable with which to run your server.
+
+## Example usage
+
+This [example truss](https://github.com/basetenlabs/truss/tree/main/examples/nemo-titanet) demonstrates how to properly configure a base image for Nvidia NeMo TitaNet:
+
+```yaml config.yaml
+base_image:
+  image: nvcr.io/nvidia/nemo:23.03
+  python_executable_path: /usr/bin/python
+apply_library_patches: true
+bundled_packages_dir: packages
+data_dir: data
+requirements:
+- PySoundFile
+live_reload: false
+resources:
+  accelerator: T4
+  cpu: 2500m
+  memory: 4512Mi
+  use_gpu: true
+secrets: {}
+spec_version: '2.0'
+system_packages:
+- python3.8-venv
+```
+
+## Configuring private base images with build time secrets
+
+Secrets of the form `DOCKER_REGISTRY_<REGISTRY_URL>` will be supplied to your model build to authenticate image pulls from private container registries.
+For information on where to store secret values see the [secrets guide](secrets#storing-secrets-on-your-remote).
+
+For example, to configure docker credentials to a private dockerhub repository your `config.yaml` should include the following secret and placeholder:
+
+```yaml config.yaml
+secrets:
+  DOCKER_REGISTRY_https://index.docker.io/v1/: null
+```
+
+along with a configured Baseten secret `DOCKER_REGISTRY_https://index.docker.io/v1/` with a base64 encoded `username:password` secret value:
+
+```sh
+echo -n 'username:password' | base64
+```
+
+To add docker credentials for gcloud artifact registry provide an [access token](https://cloud.google.com/artifact-registry/docs/docker/authentication#token) as the secret value.
+For example, to configure authentication for a repository in `us-west2` your `config.yaml` should include the following secret and placeholder:
+
+```yaml config.yaml
+secrets:
+  DOCKER_REGISTRY_us-west2-docker.pkg.dev: null
+```

docs/mint.json

@@ -67,7 +67,8 @@
     {
       "group": "Guides",
       "pages": [
-        "guides/secrets"
+        "guides/secrets",
+        "guides/base-images"
       ]
     },
     {

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "truss"
-version = "0.7.8"
+version = "0.7.7rc5"
 description = "A seamless bridge from model development to model delivery"
 license = "MIT"
 readme = "README.md"

truss/templates/shared/secrets_resolver.py

@@ -1,7 +1,9 @@
 import os
 from collections.abc import Mapping
 from pathlib import Path
-from typing import Dict
+from typing import Dict, Optional
+
+SECRETS_DOC_LINK = "https://truss.baseten.co/docs/using-secrets"
 
 
 class SecretNotFound(Exception):
@@ -17,7 +19,7 @@ def get_secrets(config: Dict):
         return Secrets(config.get("secrets", {}))
 
     @staticmethod
-    def _resolve_secret(secret_name: str, default_value: str):
+    def _resolve_secret(secret_name: str, default_value: Optional[str]):
         secret_value = default_value
         secret_env_var_name = SecretsResolver.SECRET_ENV_VAR_PREFIX + secret_name
         if secret_env_var_name in os.environ:
@@ -39,15 +41,11 @@ def __init__(self, base_secrets: Dict[str, str]):
 
     def __getitem__(self, key: str) -> str:
         if key not in self._base_secrets:
-            # Note this is the case where the secrets are not specified in
-            # config.yaml
-            raise SecretNotFound(f"Secret '{key}' not specified in the config.")
+            raise SecretNotFound(_secret_missing_error_message(key))
 
         found_secret = SecretsResolver._resolve_secret(key, self._base_secrets[key])
         if not found_secret:
-            raise SecretNotFound(
-                f"Secret '{key}' not found. Please check available secrets."
-            )
+            raise SecretNotFound(_secret_missing_error_message(key))
 
         return found_secret
 
@@ -58,3 +56,13 @@ def __iter__(self):
 
     def __len__(self):
         return len(self._base_secrets)
+
+
+def _secret_missing_error_message(key: str) -> str:
+    return f"""
+Secret '{key}' not found. Please ensure that:
+  * Secret '{key}' is defined in the 'secrets' section of the Truss config file
+  * The model was pushed with the --trusted flag
+  * Secret '{key}' is defined in the secret manager
+ Read more about secrets here: {SECRETS_DOC_LINK}.
+    """

truss/tests/test_model_inference.py

@@ -336,6 +336,9 @@ def predict(self, request):
     """
 
     config_with_no_secret = "model_name: secrets-truss"
+    missing_secret_error_message = """Secret 'secret' not found. Please ensure that:
+  * Secret 'secret' is defined in the 'secrets' section of the Truss config file
+  * The model was pushed with the --trusted flag"""
 
     with ensure_kill_all(), tempfile.TemporaryDirectory(dir=".") as tmp_work_dir:
         truss_dir = Path(tmp_work_dir, "truss")
@@ -371,7 +374,7 @@ def predict(self, request):
 
         assert "error" in response.json()
 
-        assert_logs_contain_error(container.logs(), "not specified in the config")
+        assert_logs_contain_error(container.logs(), missing_secret_error_message)
         assert "Internal Server Error" in response.json()["error"]
 
     with ensure_kill_all(), tempfile.TemporaryDirectory(dir=".") as tmp_work_dir:
@@ -390,9 +393,7 @@ def predict(self, request):
         response = requests.post(full_url, json={})
         assert response.status_code == 500
 
-        assert_logs_contain_error(
-            container.logs(), "'secret' not found. Please check available secrets."
-        )
+        assert_logs_contain_error(container.logs(), missing_secret_error_message)
         assert "Internal Server Error" in response.json()["error"]