Allow nesting for local data dir, for external data (#583)

* Allow nesting.

* Remove warnings.

Author: squidarth

poetry.lock

@@ -93,6 +93,7 @@ mlflow = "^1.29.0"
 mypy = "^1.0.0"
 pytest-split = "^0.8.1"
 httpx = {extras = ["cli"], version = "^0.24.1"}
+requests-mock = "^1.11.0"
 
 [build-system]
 requires = ["poetry-core>=1.2.1"]

pyproject.toml

@@ -0,0 +1,38 @@
+import os
+from unittest.mock import patch
+
+import requests_mock
+from truss.truss_config import ExternalData
+from truss.util.download import download_external_data
+
+TEST_DOWNLOAD_URL = "http://example.com/some-download-url"
+
+
+def test_download(tmp_path):
+    mocked_download_content = b"mocked content"
+    with patch.dict(os.environ, {}), requests_mock.Mocker() as m:
+        m.get(TEST_DOWNLOAD_URL, content=mocked_download_content)
+        external_data = ExternalData.from_list(
+            [{"local_data_path": "foo", "url": TEST_DOWNLOAD_URL}]
+        )
+        download_external_data(external_data=external_data, data_dir=tmp_path)
+
+        with open(tmp_path / "foo", "rb") as f:
+            content = f.read()
+
+        assert content == mocked_download_content
+
+
+def test_download_into_nested_subdir(tmp_path):
+    mocked_download_content = b"mocked content"
+    with patch.dict(os.environ, {}), requests_mock.Mocker() as m:
+        m.get(TEST_DOWNLOAD_URL, content=mocked_download_content)
+        external_data = ExternalData.from_list(
+            [{"local_data_path": "foo/bar/baz", "url": TEST_DOWNLOAD_URL}]
+        )
+        download_external_data(external_data=external_data, data_dir=tmp_path)
+
+        with open(tmp_path / "foo" / "bar" / "baz", "rb") as f:
+            content = f.read()
+
+        assert content == mocked_download_content

truss/tests/test_util.py

@@ -25,7 +25,7 @@ def download_external_data(external_data: Optional[ExternalData], data_dir: Path
             raise ValueError(
                 "Local data path of external data cannot point to outside data directory"
             )
-        path.parent.mkdir(exist_ok=True)
+        path.parent.mkdir(exist_ok=True, parents=True)
 
     if b10cp_path is not None:
         print("b10cp found, using it to download external data")