Dev/new gsp metrics

angel_system/global_step_prediction/global_step_predictor.py

@@ -936,13 +936,74 @@ def plot_gt_vs_predicted_plus_activations(self, step_gts, fname_suffix=None):
 
         plt.legend()
         if not fname_suffix:
-            fname_suffix = f"vid{vid_id}"
+            fname_suffix = "vid_unknown"
+
+        # check for a long path in the fname_suffix
+        if "/" in fname_suffix:
+            # get just the end part of the path
+            fname_suffix = fname_suffix.split("/")[-1]
+
         recipe_type = self.determine_recipe_from_gt_first_step(step_gts)
         title = f"plot_pred_vs_gt_{recipe_type}_{fname_suffix}.png"
         plt.title(title)
         fig.savefig(f"./outputs/{title}")
         plt.close()
 
+    def save_TP_FP_FN_per_class(self, step_gts):
+        """
+        Given the ground truth and predicted steps, return TP/FP/FN frame counts
+        for each class
+
+        Output form: TP, FP, FN, where each is an N-dimensional vector, where
+        N = number of step classes.
+        """
+        assert len(self.trackers) == 1
+        for i, tracker in enumerate(self.trackers):
+            # import ipdb; ipdb.set_trace()
+            step_predictions = tracker["granular_step_prediction_history"]
+        TP = np.zeros(len(self.avg_probs))
+        FP = np.zeros(len(self.avg_probs))
+        FN = np.zeros(len(self.avg_probs))
+        for ind in range(len(self.avg_probs)):
+            # i = the index we'll get TP, FN, and FP for.
+            _TP = len(
+                [
+                    a
+                    for j, a in enumerate(step_predictions[: len(step_gts)])
+                    if a == ind and step_gts[j] == ind
+                ]
+            )
+            TP[ind] = _TP
+            _FP = len(
+                [
+                    a
+                    for j, a in enumerate(step_predictions[: len(step_gts)])
+                    if a == ind and step_gts[j] != ind
+                ]
+            )
+            FP[ind] = _FP
+            _FN = len(
+                [
+                    a
+                    for j, a in enumerate(step_predictions[: len(step_gts)])
+                    if a != ind and step_gts[j] == ind
+                ]
+            )
+            FN[ind] = _FN
+        return TP, FP, FN
+
+    def get_single_tracker_pred_history(self):
+        """
+        Get a single tracker's prediction history.
+        For now, this is only functional in the case of a single tracker instance
+        being initialized.
+
+        Output:
+        - list: prediction history. Length = number of frames processed.
+        """
+        assert len(self.trackers) == 1
+        return self.trackers[0]["granular_step_prediction_history"]
+
     def sanitize_str(self, str_: str):
         """
         Convert string to lowercase and emove trailing whitespace and period.

angel_system/global_step_prediction/run_expirement.py

@@ -9,6 +9,7 @@
 import matplotlib.pyplot as plt
 import seaborn as sns
 from sklearn.metrics import confusion_matrix
+import math
 
 from angel_system.global_step_prediction.global_step_predictor import (
     GlobalStepPredictor,
@@ -32,11 +33,19 @@ def run_inference_all_vids(
     all_vid_ids = np.unique(np.asarray(coco_test.images().lookup("video_id")))
     avg_probs = None
     preds, gt = [], []
+    avg_smd, avg_smd_normd = np.array([]), np.array([])
+    mean_F1s = np.array([])
+    tasks_completed = np.array([])
     for vid_id in all_vid_ids:
         print(f"vid_id {vid_id}===========================")
 
         act_path = code_dir / "config/activity_labels/medical" / f"{medical_task}.yaml"
 
+        # Get framerate
+        framerate = round(coco_test.videos(video_ids=[vid_id]).peek()["framerate"])
+        # Assuming FR = 15 or 30
+        assert framerate in [15, 30], f"framerate rounded to {framerate}"
+
         step_predictor = GlobalStepPredictor(
             recipe_types=[f"{medical_task}"],
             activity_config_fpath=act_path.as_posix(),
@@ -46,8 +55,8 @@ def run_inference_all_vids(
                 / "config/tasks/medical"
                 / f"{medical_task}.yaml"
             },
-            # threshold_multiplier=0.3,
-            # threshold_frame_count=2
+            threshold_multiplier=0.6,
+            threshold_frame_count=5,
         )
 
         if avg_probs is not None:
@@ -76,6 +85,14 @@ def run_inference_all_vids(
         # All N activity confs x each video frame
         activity_confs = test_video_dset.annots().get("prob")
         activity_gts = truth_video_dset.annots().get("category_id")
+        # If framerate is 30, take every other frame.
+        if framerate == 30:
+            # This must be a 30Hz video, not 15Hz. Take every other GT frame.
+            print(f"halve gt for {vid_id}. Len = {len(activity_gts)}")
+            activity_gts = [a for ind, a in enumerate(activity_gts) if ind % 2 == 0]
+            print(f"new len = {len(activity_gts)}")
+        # ...and cut out the first 25 frames.
+        activity_gts = activity_gts[25:]
 
         def get_unique(activity_ids):
             """
@@ -106,12 +123,104 @@ def get_unique(activity_ids):
 
         print(f"unique broad steps: {get_unique(broad_step_gts)}")
 
+        # activity_gt_maxes[i] set to max step so far at index i.
+        activity_gt_maxes = activity_gts.copy()
+        for ind in range(1, len(activity_gts)):
+            activity_gt_maxes[ind] = max(activity_gts[:ind])
+
+        _TP, _FP, _FN = step_predictor.save_TP_FP_FN_per_class(activity_gt_maxes)
+        if "TP" in locals():
+            TP += _TP
+            FP += _FP
+            FN += _FN
+        else:
+            TP = _TP
+            FP = _FP
+            FN = _FN
+        class_F1s, mean_F1 = compute_class_f1s_and_mean_f1(TP, FP, FN)
+        print(f"class-wise F1s: {class_F1s}\nmean F1: {mean_F1}")
+
+        mean_F1s = np.append(mean_F1s, mean_F1)
+
+        pred_history = step_predictor.get_single_tracker_pred_history()
+        smds, smds_normd, task_completed = get_start_moment_distances(
+            pred_history, activity_gt_maxes
+        )
+        tasks_completed = np.append(tasks_completed, task_completed)
+
+        if not math.isnan(compute_mean_smd(smds)):
+            avg_smd = np.append(avg_smd, compute_mean_smd(smds))
+            avg_smd_normd = np.append(avg_smd_normd, compute_mean_smd(smds_normd))
+
+        print(f"smds (# frames): {smds}, normalized:{smds_normd}")
+        try:
+            print(f"avg frame-wise smd:{avg_smd[-1]}, normalized: {avg_smd_normd[-1]}")
+        except:
+            import ipdb
+
+            ipdb.set_trace()
+
         _ = step_predictor.plot_gt_vs_predicted_one_recipe(
-            granular_step_gts,
+            activity_gts,
             recipe_type,
             fname_suffix=f"{str(vid_id)}_granular_{extra_output_suffix}",
             granular_or_broad="granular",
         )
+    print("########## OVERALL")
+    print(
+        f"Overall average smd: {np.mean(avg_smd)}. Normalized: {np.mean(avg_smd_normd)}"
+    )
+    print(f"tasks completed: {np.sum(tasks_completed)} / {len(tasks_completed)}")
+    print(f"overall mean F1: {np.mean(mean_F1s)}")
+
+
+def get_start_moment_distances(pred_history, activity_gt_maxes):
+    """
+    Get the distance between ground truth & predictions of the starting frame of
+    each step.
+
+    Outputs:
+    - smds: list of length equal to number
+    """
+    num_classes = max(activity_gt_maxes)
+    vid_length = len(activity_gt_maxes)
+    smds = np.zeros(num_classes)
+    smds_normd = np.zeros(num_classes)
+    task_completed = 1
+    for i in range(num_classes):
+        if i + 1 in pred_history:
+            smds[i] = abs(
+                np.where(pred_history == i + 1)[0][0] - activity_gt_maxes.index(i + 1)
+            )
+            smds_normd[i] = smds[i] / vid_length
+        else:
+            smds[i] = -1
+            smds_normd[i] = -1
+            if i + 1 == num_classes:
+                task_completed = 0
+    return smds, smds_normd, task_completed
+
+
+def compute_mean_smd(smd_array):
+    """
+    Compute mean starting moment difference. If a task was not completed,
+    the incomplete steps are not factored in here.
+    """
+
+    mask = smd_array >= 0
+    return np.mean(smd_array[mask])
+
+
+def compute_class_f1s_and_mean_f1(TP, FP, FN):
+    F1s = np.zeros(len(TP))
+    # class-wise F1s:
+    for i in range(len(TP)):
+        F1s[i] = 2 * TP[i] / (2 * TP[i] + FP[i] + FN[i])
+    # mean F1
+    mean_F1 = (
+        2 * np.sum(TP[:-1]) / (2 * np.sum(TP[:-1]) + np.sum(FP[:-1]) + np.sum(FN[:-1]))
+    )
+    return F1s, mean_F1
 
 
 @click.command(context_settings={"help_option_names": ["-h", "--help"]})

ansible/roles/provision-files/vars/main.yml

@@ -71,15 +71,15 @@ girder_file_downloads:
       sha512: e6b382f5ba3d4d7f17d29caa3f6ea06dd5db24fee3bf59b12ff748d58b1b064c374741c805ee7c07a2a6950761a38e5398f3707ed3e83d7d53218972c57d12a2
       dest: "{{ stage_dirs.object_detector }}/m2_det.pt"
     # Activity classifier
-    - file_id: 671bff2983e1acbfd87e723f
-      sha512: 5165219f60e3a160a8ec51f0446db4ec23ed70c399df66f482a233566e177d1f8cf044fc6ccc5227f5c748cbe6d2f17143109e791a139dfc2225d75de67829b2
+    - file_id: 673f69bef4ba555b2ab8ae15
+      sha512: 91d69908ef49b4a43dd344ae36a1c11eda6afb9b15dd748292435343087e08a3b1c22347efe592d7f078669d52a3dc496ca605ba4463997878151973e4885e77
       dest: "{{ stage_dirs.activity_classifier }}/m2_tcn.ckpt"
-    - file_id: 672543fcca1110761bd32f7d
-      sha512: aef3ab3f8927f83b7ecad3cc28ea7b42e662d5c11740df1015542df6740db3633a7829187e9f5a35b4a66ebfeb48756c6598e4b7e6f68cc1ae90912fd71e31dd
+    - file_id: 673f69b3f4ba555b2ab8ae12
+      sha512: 68bea2bd8748cc2ad55ca8784d45b9b0c9f21bd3cbca9d8983d158281adf15197d52a2582477ff1ac79ece0ce9187eb466056d407cb3e54894c3f9fd47dc5e41
       dest: "{{ stage_dirs.activity_classifier }}/m2_config.yaml"
     # Global Step predictor model
-    - file_id: 6723f3d283e1acbfd87e7269
-      sha512: fb676aae6f768498df860bbf4a1621ee8403cba96a1b9acc088f5f23671c57ca1f06f6c535fa8b0eb5074c7fed56e8ff5a7f990e2893f3759662c76137db1746
+    - file_id: 673f74eff4ba555b2ab8ae3e
+      sha512: 437f5557ff3670c75118d7699964ece42d3480502a71888de62c53c6998b42a297fb7b21ea0bc6bfc4da59e64f9730b69d61d55b5f2c9dd73d93eeee682d60de
       dest: "{{ stage_dirs.task_monitor }}/global_step_predictor_act_avgs_m2.npy"
 
     # ---- M3 ----
@@ -122,15 +122,15 @@ girder_file_downloads:
       sha512: 7183385f8eaca85997725a107a76034de2bd4a59c1434b4bdb7c1ac8931cf4b68a53f6e736734643386364b9f0856de795a14965b6a02bc5eb5891252e6a73c9
       dest: "{{ stage_dirs.object_detector }}/r18_det.pt"
     # Activity classifier
-    - file_id: 672cf299ca1110761bd32ffd
-      sha512: 0bfd4c5a1f254120758b9802eefef8933bfebf80837bd61445c3c26228edaefa34f0ed81178670538426983a93d60198f1be82a767e350b26d4fb3d081972b0d
+    - file_id: 673f6e62f4ba555b2ab8ae35
+      sha512: e731dad0d0688ede4fc1b04b7c6c67478971ab97da062813ec04571daca07f413aa4775c385c3e8b28a8f5f50f8157e68547d45d033ab4bb0926935ad39e7f26
       dest: "{{ stage_dirs.activity_classifier }}/r18_tcn.ckpt"
-    - file_id: 672cf437ca1110761bd33009
-      sha512: d99c484071f22661425a9673ff351db90ce2fba40f2faa56f0b388deb2d0cdcc96931a2627e5aec45ab5a4624c9ab04da23b63364dd6eb1d5641a90701737006
+    - file_id: 673f6e58f4ba555b2ab8ae32
+      sha512: 09a4c97e79c34ac95c95a4c63370b928527a98b9bab233aa12e4b14cfa95e0bb5a09bad90d19715ab656d17281ba5fbaa719647a5545bf2447ae584da5bd8d6a
       dest: "{{ stage_dirs.activity_classifier }}/r18_config.yaml"
     # Global Step predictor model
-    - file_id: 66464bf9687336214e7cdeae
-      sha512: bc7884c258cb8704372dd69a3e7d999eaf70d716d271982077c7216ef27ab01219ef1e488f6f9669e11a7e8aa6ffb9d8e07d74edc47294f90cc9e6a723f5a822
+    - file_id: 673f8cbcf4ba555b2ab8ae42
+      sha512: 4198d861961886b05b85dd178004ea2ca54ce4625bd9ea37a18f000f1090e8646e5838c1577090b799662e0cae779558f857b6a568c115f79bbca6d021a446f7
       dest: "{{ stage_dirs.task_monitor }}/global_step_predictor_act_avgs_R18.npy"
 
 # List of git repositories to check out at a specific ref and then archive.

config/activity_labels/medical/a8.yaml

@@ -0,0 +1,25 @@
+version: "1"
+title: "NPA"
+labels:
+  # Item:
+  # - id: Integer identification number of the label.
+  # - label: Short human-readable semantic string to succinctly represent the
+  #          label.
+  - id: 0
+    label: "background"
+    full_str: "background"
+  - id: 1
+    label: "select-tube"
+    full_str: "Select NPA tube that best fits casualty."
+  - id: 2
+    label: "position-head"
+    full_str: "Place casualty into 'sniffing position' with head tilted back and nostrils exposed."
+  - id: 3
+    label: "cover-with-lube"
+    full_str: "Cover NPA with lube."
+  - id: 4
+    label: "insert-npa"
+    full_str: "Insert NPA perpendicular to casualty nostril until flange meets tip of nose, rotating along the way."
+  - id: 5
+    label: "look-listen-feel"
+    full_str: "Look, listen, and feel for airway compliance."

config/activity_labels/medical/m4.yaml

@@ -10,10 +10,10 @@ labels:
     full_str: "background"
   - id: 1
     label: "apply-pressure"
-    full_str: "Apply direct hand pressure."
+    full_str: "Apply direct hand pressure to the wound."
   - id: 2
     label: "open-pack"
-    full_str: "Open dressing packaging."
+    full_str: "Open gauze packaging."
   - id: 3
     label: "apply-dress-w-pressure"
-    full_str: "Apply dressing with pressure point directly over wound site."
+    full_str: "Pack the wound."

config/activity_labels/medical/r16.yaml

@@ -0,0 +1,25 @@
+version: "1"
+title: "Ventilate (BVM)"
+labels:
+  # Item:
+  # - id: Integer identification number of the label.
+  # - label: Short human-readable semantic string to succinctly represent the
+  #          label.
+  - id: 0
+    label: "background"
+    full_str: "background"
+  - id: 1
+    label: "position-head"
+    full_str: "Place casualty into 'sniffing position' with head tilted back and nostrils exposed."
+  - id: 2
+    label: "open-packaging"
+    full_str: "Open the BVM packaging."
+  - id: 3
+    label: "attach-mask"
+    full_str: "Attach mask to BVM and expand BVM to full size."
+  - id: 4
+    label: "place-mask"
+    full_str: "Place mask over patient’s mouth in proper orientation."
+  - id: 5
+    label: "squeeze-bvm"
+    full_str: "Squeeze BVM while holding mask to patient’s mouth."

config/activity_labels/medical/r19.yaml

@@ -0,0 +1,28 @@
+version: "1"
+title: "Needle_Chest_Decomp"
+labels:
+  # Item:
+  # - id: Integer identification number of the label.
+  # - label: Short human-readable semantic string to succinctly represent the
+  #          label.
+  - id: 0
+    label: "background"
+    full_str: "background"
+  - id: 1
+    label: "locate-insertion-site"
+    full_str: "Locate insertion site at the second intercostal space at the midclavicular line."
+  - id: 2
+    label: "wipe-insertion-site"
+    full_str: "Wipe the insertion site with an alcohol wipe."
+  - id: 3
+    label: "prepare-catheter-and-needle"
+    full_str: "Prepare catheter and needle."
+  - id: 4
+    label: "insert-needle"
+    full_str: "Insert needle into insertion site."
+  - id: 5
+    label: "remove-needle"
+    full_str: "Remove needle, keeping catheter inside the patient."
+  - id: 6
+    label: "apply-tape"
+    full_str: "Apply tape around catheter to secure it in place."