Skip to content

Commit d831944

Browse files
cameron-a-johnsonPurg
authored andcommitted
Ongoing non-HMM step tracking code addition
1 parent 5bc78e2 commit d831944

File tree

5 files changed

+822
-0
lines changed

5 files changed

+822
-0
lines changed

.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
/.container_xauth
66
/model_files
77
/ros_bags
8+
/outputs
89

910
### Python template
1011
# Byte-compiled / optimized / DLL files

angel_system/global_step_prediction/__init__.py

Whitespace-only changes.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,339 @@
1+
import yaml
2+
import os
3+
import seaborn as sn
4+
import numpy as np
5+
import kwcoco
6+
import matplotlib.pyplot as plt
7+
from sklearn.metrics import confusion_matrix
8+
import scipy.ndimage as ndi
9+
10+
def sanitize_str(str_: str):
11+
"""
12+
Convert string to lowercase and emove trailing whitespace and period.
13+
14+
:param str_: Input text
15+
16+
:return: ``str_`` converted to lowercase and stripped of trailing whitespace and period.
17+
:rtype: str
18+
"""
19+
return str_.lower().strip(" .")
20+
21+
def plot_positive_GT_conf_distributions(activity_confs, activity_gt):
22+
"""
23+
plot_TP_conf_distributions:
24+
For each activity, plot the distribution of confidences when ground
25+
truth indicates that activity is happening.
26+
27+
i.e.: for activity x, for frames in which ground truth = x, plot
28+
the distribution of confidences.
29+
30+
Inputs:
31+
activity_confs: frames x class-wise-confidences. Given a kwcoco
32+
dataset called "coco":
33+
```
34+
activity_confs = torch.asarray(coco.images().lookup("activity_conf"))
35+
```
36+
(49K x 25 for coffee val set.)
37+
activity_gt: frames x ground truth activity_id.
38+
Given a kwcoco dataset called "coco":
39+
```
40+
activity_gt = torch.asarray(coco.images().lookup("activity_gt"))
41+
```
42+
"""
43+
44+
sns.set_theme(style="white", rc={"axes.facecolor": (0, 0, 0, 0)})
45+
46+
# Get data together
47+
true_confs = [float(activity_confs[i,truth_ind]) for i, truth_ind in enumerate(activity_gt)]
48+
data = {"true_conf":true_confs, "gt":activity_gt}
49+
df = pd.DataFrame(data)
50+
51+
false_confs = np.array([[a for i, a in enumerate(act_conf) if i != gt] for act_conf, gt in zip(activity_confs, activity_gt)]).flatten()
52+
false_gt = np.array([[gt for i, a in enumerate(act_conf) if i != gt] for act_conf, gt in zip(activity_confs, activity_gt)]).flatten()
53+
data_opposite = {"true_conf":false_confs, "gt":false_gt}
54+
df_opposite = pd.DataFrame(data_opposite)
55+
56+
def plot(df):
57+
# Initialize the FacetGrid object
58+
pal = sns.cubehelix_palette(10, rot=-.25, light=.7)
59+
g = sns.FacetGrid(df, row="gt", hue="gt", aspect=15, height=.5, palette=pal)
60+
61+
# Draw the densities in a few steps
62+
g.map(sns.kdeplot, "true_conf",
63+
bw_adjust=.5, clip_on=False,
64+
fill=True, alpha=1, linewidth=1.5)
65+
g.map(sns.kdeplot, "true_conf", clip_on=False, color="w", lw=2, bw_adjust=.5)
66+
67+
# passing color=None to refline() uses the hue mapping
68+
g.refline(y=0, linewidth=2, linestyle="-", color=None, clip_on=False)
69+
70+
# Define and use a simple function to label the plot in axes coordinates
71+
def label(x, color, label):
72+
ax = plt.gca()
73+
ax.text(0, .2, label, fontweight="bold", color=color,
74+
ha="left", va="center", transform=ax.transAxes)
75+
g.map(label, "true_conf")
76+
77+
# Set the subplots to overlap
78+
g.figure.subplots_adjust(hspace=-.25)
79+
80+
# Remove axes details that don't play well with overlap
81+
g.set_titles("")
82+
g.set(yticks=[], ylabel="")
83+
g.despine(bottom=True, left=True)
84+
85+
# save
86+
plt.savefig("./outputs/plot_positive_GT_conf_distributions.png")
87+
88+
89+
def bilateralFtr1D(y, sSpatial = 5, sIntensity = 1):
90+
'''
91+
The equation of the bilateral filter is
92+
93+
( dx ^ 2 ) ( dI ^2 )
94+
F = exp (- ----------------- ) * exp (- ------------------- )
95+
( sigma_spatial ^ 2 ) ( sigma_Intensity ^ 2 )
96+
~~~~~~~~~~~~~~~~~~~~~~~~~~
97+
This is a guassian filter!
98+
dx - The 'geometric' distance between the 'center pixel' and the pixel
99+
to sample
100+
dI - The difference between the intensity of the 'center pixel' and
101+
the pixel to sample
102+
sigma_spatial and sigma_Intesity are constants. Higher values mean
103+
that we 'tolerate more' higher value of the distances dx and dI.
104+
105+
Dependencies: numpy, scipy.ndimage.gaussian_filter1d
106+
107+
calc gaussian kernel size as: filterSize = (2 * radius) + 1; radius = floor (2 * sigma_spatial)
108+
y - input data
109+
'''
110+
111+
# gaussian filter and parameters
112+
radius = np.floor (2 * sSpatial)
113+
filterSize = ((2 * radius) + 1)
114+
ftrArray = np.zeros(int(filterSize))
115+
ftrArray[int(radius)] = 1
116+
117+
# Compute the Gaussian filter part of the Bilateral filter
118+
gauss = ndi.gaussian_filter1d(ftrArray, sSpatial)
119+
120+
# 1d data dimensions
121+
width = y.size
122+
123+
# 1d resulting data
124+
ret = np.zeros (width)
125+
126+
for i in range(width):
127+
128+
## To prevent accessing values outside of the array
129+
# The left part of the lookup area, clamped to the boundary
130+
xmin = max(i - radius, 1);
131+
# How many columns were outside the image, on the left?
132+
dxmin = xmin - (i - radius);
133+
134+
# The right part of the lookup area, clamped to the boundary
135+
xmax = min(i + radius, width);
136+
# How many columns were outside the image, on the right?
137+
dxmax = (i + radius) - xmax;
138+
139+
# The actual range of the array we will look at
140+
area = y [int(xmin):int(xmax)]
141+
142+
# The center position
143+
center = y[i]
144+
145+
# The left expression in the bilateral filter equation
146+
# We take only the relevant parts of the matrix of the
147+
# Gaussian weights - we use dxmin, dxmax, dymin, dymax to
148+
# ignore the parts that are outside the image
149+
expS = gauss[int((1+dxmin)):int((filterSize-dxmax))]
150+
151+
# The right expression in the bilateral filter equation
152+
dy = y [int(xmin):int(xmax)] - y[i]
153+
dIsquare = (dy * dy)
154+
expI = np.exp (- dIsquare / (sIntensity * sIntensity))
155+
156+
# The bilater filter (weights matrix)
157+
F = expI * expS
158+
159+
# Normalized bilateral filter
160+
Fnormalized = F / sum(F)
161+
162+
# Multiply the area by the filter
163+
tempY = y [int(xmin):int(xmax)] * Fnormalized
164+
165+
# The resulting pixel is the sum of all the pixels in
166+
# the area, according to the weights of the filter
167+
# ret(i,j,R) = sum (tempR(:))
168+
ret[i] = sum (tempY)
169+
170+
return ret
171+
172+
173+
def get_average_TP_activations(coco):
174+
# For each activity, given the Ground Truth-specified
175+
# frame subset where that activity is happening, get the
176+
# average activation of that class.
177+
178+
all_activity_ids = np.unique(np.asarray(coco.images().lookup('activity_gt')))
179+
all_vid_ids = np.unique(np.asarray(coco.images().lookup('video_id')))
180+
181+
avg_probs = np.zeros(max(all_activity_ids) + 1)
182+
183+
for activity_id in all_activity_ids:
184+
#image_ids = coco.index.vidid_to_gids[vid_id]
185+
image_ids = [img['id'] for img in coco.videos(video_ids=all_vid_ids).images[0].objs if img['activity_gt'] == activity_id]
186+
sub_dset = coco.subset(gids=image_ids, copy=True)
187+
probs_for_true_inds = np.asarray(
188+
sub_dset.images().lookup("activity_conf"))[:,activity_id]
189+
avg_prob = np.mean(probs_for_true_inds)
190+
avg_probs[activity_id] = avg_prob
191+
192+
return avg_probs
193+
194+
config_fn = "config/tasks/task_steps_cofig-recipe-coffee-shortstrings.yaml"
195+
with open(config_fn, "r") as stream:
196+
config = yaml.safe_load(stream)
197+
labels = [sanitize_str(l["description"]) for l in config["steps"]]
198+
steps = config['steps']
199+
if steps[0]['id'] == 1:
200+
config['steps'].insert(0, {'id':0,
201+
'activity_id':0,
202+
'description':'background',
203+
'median_duration_seconds':0.5,
204+
'mean_conf':0.5,
205+
'std_conf':0.2,
206+
})
207+
208+
coco_val = kwcoco.CocoDataset("model_files/val_activity_preds_epoch40.mscoco.json")
209+
coco_test = kwcoco.CocoDataset("model_files/test_activity_preds.mscoco.json")
210+
211+
image_ids = coco_test.index.vidid_to_gids[3]
212+
video_dset = coco_test.subset(gids=image_ids, copy=True)
213+
214+
# "Training": for each activity class, see what the average "true positive"
215+
# activation was.
216+
avg_probs = get_average_TP_activations(coco_test)
217+
print(f"average_probs = {avg_probs}")
218+
219+
all_vid_ids = np.unique(np.asarray(coco_val.images().lookup('video_id')))
220+
221+
for vid_id in all_vid_ids:
222+
print(f"vid_id {vid_id}")
223+
224+
image_ids = coco_test.index.vidid_to_gids[vid_id]
225+
video_dset = coco_test.subset(gids=image_ids, copy=True)
226+
227+
# All N activity confs x each video frame
228+
activity_confs = video_dset.images().lookup("activity_conf")
229+
230+
next_step = 1
231+
step_predictions = []
232+
num_frames_activated = 0
233+
234+
# Predicted step: confidence has been above threshold for 5 frames.
235+
threshold_frame_count = 8
236+
for i, activity_conf in enumerate(activity_confs):
237+
238+
# Check if we're done: if so, append last step & continue
239+
if next_step == len(steps):
240+
step_predictions.append(next_step-1)
241+
continue
242+
# Next step
243+
next_activity_id = steps[next_step]['activity_id']
244+
next_next_activity_id = steps[min(len(steps)-1,next_step + 1)][
245+
'activity_id']
246+
247+
next_activity_conf = activity_conf[next_activity_id]
248+
next_next_activity_conf = activity_conf[next_next_activity_id]
249+
250+
avg_prob_next_activity = avg_probs[next_activity_id]
251+
avg_prob_next_next_activity = avg_probs[next_next_activity_id]
252+
'''
253+
if next_activity_id == 16 and vid_id == 2:
254+
print(f"next_activity_id = {next_activity_id}")
255+
print(f"avg_prob_next_activity = {avg_prob_next_activity}")
256+
'''
257+
if i > 15:
258+
threshold_frame_count = 16
259+
260+
if next_activity_conf > 0.8 * avg_prob_next_activity:
261+
num_frames_activated += 1
262+
'''
263+
if next_activity_id == 16 and vid_id == 2:
264+
print(f"num_frames_activated = {num_frames_activated}. prob = {next_activity_conf}")
265+
'''
266+
else:
267+
num_frames_activated = 0
268+
if next_next_activity_conf > 0.8 * avg_prob_next_activity:
269+
num_skip2_frames_activated += 1
270+
else:
271+
num_skip2_frames_activated = 0
272+
273+
if num_frames_activated >= threshold_frame_count:
274+
#if next_step < 23:
275+
#next_step += 1
276+
next_step += 1
277+
num_frames_activated = 0
278+
num_skip2_frames_activated = 0
279+
elif num_skip2_frames_activated >= threshold_frame_count:
280+
next_step = min(next_step + 2, len(steps))
281+
num_frames_activated = 0
282+
num_skip2_frames_activated = 0
283+
print("hit a skip-step!!")
284+
285+
step_predictions.append(next_step-1)
286+
287+
# Ground truth step:
288+
activity_gts = video_dset.images().lookup("activity_gt")
289+
step_gts = []
290+
step_gts_no_background = []
291+
current_step = 0
292+
for activity_gt in activity_gts:
293+
# convert activity id to step id
294+
step_id = next(int(item['id']) for item in steps if item['activity_id'] == activity_gt)
295+
step_gts.append(step_id)
296+
297+
# A version of GT that never jumps back to 0
298+
if step_id > 0:
299+
current_step = step_id
300+
step_gts_no_background.append(current_step)
301+
302+
303+
# Plot confusion matrix
304+
fig, ax = plt.subplots(figsize=(100, 100))
305+
cm = confusion_matrix(step_gts_no_background, step_predictions, normalize="true")
306+
sn.heatmap(cm, annot=True, fmt="0.0%", ax=ax, linewidth=.5)
307+
sn.set(font_scale=4)
308+
ax.set(
309+
title="Confusion Matrix",
310+
xlabel="Predicted Label",
311+
ylabel="True Label",)
312+
fig.savefig(f"./outputs/plot_confusion_mat_vid{vid_id}.png")
313+
314+
# Plot gt vs predicted class across all vid frames
315+
fig = plt.figure()
316+
sn.set(font_scale=1)
317+
step_gts = [float(i) for i in step_gts]
318+
plt.plot(step_gts, label = 'gt')
319+
plt.plot(step_predictions, label = 'estimated')
320+
#plt.plot(inliers-0.5, label = 'inliers')
321+
plt.plot(10*np.asarray(activity_confs)[:,17]-5, label = 'act_preds[17]')
322+
plt.plot(10*np.asarray(activity_confs)[:,18]-5, label = 'act_preds[18]')
323+
plt.plot(10*np.asarray(activity_confs)[:,19]-5, label = 'act_preds[19]')
324+
325+
plt.plot(bilateralFtr1D(10*np.asarray(activity_confs)[:,17])-10, label = 'act_preds_bilateral[17]')
326+
plt.plot(bilateralFtr1D(10*np.asarray(activity_confs)[:,18])-10, label = 'act_pred_bilateral[18]')
327+
plt.plot(bilateralFtr1D(10*np.asarray(activity_confs)[:,19])-10, label = 'act_preds_bilateral[19]')
328+
#plt.plot(10*X_conf_incremental, label = 'confidence')
329+
#plt.plot(10*vid_acts[:,10], label = act_labels[10])
330+
#plt.plot(10*vid_acts[:,11], label = act_labels[11])
331+
#plt.plot(10*vid_acts[:,12], label = act_labels[12])
332+
plt.legend()
333+
fig.savefig(f"./outputs/plot_pred_vs_gt_vid{vid_id}.png")
334+
335+
if False:
336+
plot_positive_GT_conf_distributions(activity_confs, activity_gt)
337+
338+
339+

0 commit comments

Comments
 (0)