Add provision support for SwinB model files

Author: Purg

.dockerignore

@@ -2,4 +2,5 @@
 /.workspace-shell-dev
 /.container_xauth
 /HoloLens2-ResearchMode-Unity
+/model_files
 /unity

.gitignore

@@ -3,6 +3,7 @@
 #
 /.workspace-shell-dev/
 /.container_xauth
+/model_files
 
 # This .gitignore file should be placed at the root of your Unity project directory
 #

README.md

@@ -73,9 +73,31 @@ report-back to the HoloLens2 platform.
 Workspace root: `./ros/`
 
 System requirements
+* ansible
 * docker
 * docker-compose
 
+Some files required from the `https://data.kitware.com` Girder service require
+authentication due to their protected nature.
+The environment variable `GIRDER_API_KEY` must be defined with a valid API key,
+otherwise an authentication token cannot be retrieved.
+
+## Provision Files
+External large files should be provisioned by running the ansible tool:
+
+    ansible-playbook -i ansible/hosts.yml ansible/provision_files.yml
+
+This may include large files for running the system, like ML model files, or
+other files required for building docker images.
+
+This provisioning may require additional configuration and variables set in
+your environment in order to satisfy some permissions:
+* `GIRDER_API_KEY` will need to be set in order to acquire protected files from
+  `data.kitware.com`.
+
+The configuration that controls what is staged and where is located
+in the `ansible/roles/provision-files/vars/main.yml` file.
+
 ## Docker-based Workflow
 **Intention**: Use containerization to standardize development and runtime
 environment and practice.
@@ -84,9 +106,9 @@ environment and practice.
   configurations.
 
 Docker functionality is located under the `./docker/` directory.
-* `ros2-base` provides a base environment that supports building and running
-  our workspace.
-* `ros2-workspace-build` provides a build of our workspace.
+* `workspace-base-dev` provides a base environment that supports building and
+  running our workspace.
+* `workspace-build` provides a build of our workspace.
 
 ### Building Docker Images
 Run `./angel-docker-build.sh`.
@@ -103,7 +125,7 @@ The definition of this service is found in the `docker/docker-compose.yml`
 configuration.
 
 This will mount the `./ros/` subtree on the host system into the
-`/angel_workspace/src` directory in the run container.
+`/angel_workspace/src/` directory in the run container.
 
 This "workspace" context additionally mounts build, install and log output
 directories to a spot on the host in order to:
@@ -115,6 +137,9 @@ The script `/angel_workspace/workspace_build.sh` is available to run.
 This script is used during the image build process, so using this script
 ensures that the same build method is performed.
 
+Other directories and files are mounted into the container environment for
+development purposes and external file sharing.
+
 This shell will **_NOT_** have the local installation sourced in order to
 facilitate further safe build actions.
 `ros2 run` actions should be performed in a separate shell from where builds

ansible/roles/provision-files/vars/main.yml

@@ -4,7 +4,7 @@
 stage_dirs:
   # Example
   #example: some/directory
-  {}  # REMOVE WHEN ADDING ENTRIES
+  model_files: "model_files"
 
 basic_downloads:
   # Example
@@ -21,7 +21,15 @@ girder_file_downloads:
     #- file_id: "111111111111111111111111"
     #  sha512: "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
     #  dest: "{{ stage_dirs.build_itt_base }}/some_file.txt"
-    []  # REMOVE WHEN ADDING LIST CONTENT
+
+    # SwinB label list text file.
+    - file_id: 62cc5fa7bddec9d0c4faa19f
+      sha512: 3fbf340b9cb87a71dd3df81880b38ee414c47b43fbfb78c37220020fbdb8d0d0ff93d76b079aecd94c10317ad6764590fd2a98849338f7fb46f943825820aaf6
+      dest: "{{ stage_dirs.model_files }}/swinb_coffee_task_labels.txt"
+    # SwinB model weights checkpoint
+    - file_id: 62cc5fbebddec9d0c4faa1b5
+      sha512: cf991079ea52fdab602d46052925a06a40f60176271e7a098ee34bb9262af72be5f1925014f30faf27a59c1b9b46f4f1d8401add4b6e669b982ca3f5e9f1026b
+      dest: "{{ stage_dirs.model_files }}/swinb_model_stage_base_ckpt_6.pth"
 
 # List of git repositories to check out at a specific ref and then archive.
 # Destination files will be written as GZipped TAR files, so please suffix

docker/docker-compose.yml

@@ -57,6 +57,7 @@ services:
       - ../ros:${ANGEL_WORKSPACE_DIR}/src
       - ../tmux:${ANGEL_WORKSPACE_DIR}/tmux
       - ../angel_system:${ANGEL_WORKSPACE_DIR}/angel_system
+      - ../model_files:${ANGEL_WORKSPACE_DIR}/model_files
       - ./workspace_build.sh:${ANGEL_WORKSPACE_DIR}/workspace_build.sh
       - ./workspace_entrypoint.sh:${ANGEL_WORKSPACE_DIR}/workspace_entrypoint.sh
       - ./workspace_setenv.sh:${ANGEL_WORKSPACE_DIR}/workspace_setenv.sh
@@ -65,6 +66,7 @@ services:
       - ${WORKSPACE_SHELL_HOST_DIR}/install:${ANGEL_WORKSPACE_DIR}/install
       - ${WORKSPACE_SHELL_HOST_DIR}/log:${ANGEL_WORKSPACE_DIR}/log
       - ${WORKSPACE_SHELL_HOST_DIR}/stuff:${ANGEL_WORKSPACE_DIR}/stuff
+      - ${WORKSPACE_SHELL_HOST_DIR}/cache/torch:/root/.cache/torch
       # X11 things
       - /tmp/.X11-unix:/tmp/.X11-unix
       # assume this file exists, should be created before running.

docker/workspace-build/Dockerfile

@@ -82,11 +82,9 @@ RUN apt-get -y update \
 # Install tracked python dependencies
 COPY poetry.lock pyproject.toml "${ANGEL_WORKSPACE_DIR}"/
 RUN cd "${ANGEL_WORKSPACE_DIR}" \
-    # TODO: Re-enable use of hashes once editable/git deps are removed. \
-    #       - Remove `--without-hashes` option. \
     # * Poetry extraneously finds matplotlib requiring setuptools-scm.
     #   theoretically being "fixed" in matplotlib 3.6
- && poetry export --without-hashes --dev -f requirements.txt | \
+ && poetry export --dev -f requirements.txt | \
       pip3 install --no-deps --user -r /dev/stdin \
  && pip uninstall -y setuptools-scm
 

model_files/README.rst

@@ -0,0 +1,2 @@
+This directory is a target for provisioned files.
+Files you may add here `SHOULD NOT` be checked into Git.

ros/angel_system_nodes/configs/swinb_activity_det_config.json

@@ -1,8 +1,8 @@
 {
     "type": "angel_system.impls.detect_activities.swinb.swinb_detect_activities.SwinBTransformer",
     "angel_system.impls.detect_activities.swinb.swinb_detect_activities.SwinBTransformer": {
-        "checkpoint_path": "angel_system/model_checkpoints/swinb_coffee_tasks.pth",
-        "labels_file": "angel_system/labels/swinb_coffee_task_labels.txt",
+        "checkpoint_path": "model_files/swinb_model_stage_base_ckpt_6.pth",
+        "labels_file": "model_files/swinb_coffee_task_labels.txt",
         "num_classes": 14,
         "num_frames": 32,
         "sampling_rate": 2,