feat: add complete implementation of CLEAR dataset (#3466)

Author: suhana13

src/helm/benchmark/presentation/run_entries_medhelm.conf

@@ -12,12 +12,109 @@ entries: [
   {description: "medcalc_bench:model=qwen/qwen2.5-7b-instruct,model_deployment=huggingface/qwen2.5-7b-instruct-4bit", priority: 1},
   {description: "medcalc_bench:model=microsoft/phi-3.5-mini-instruct,model_deployment=huggingface/phi-3.5-mini-instruct-4bit", priority: 1},
 
-  {description: "clear:model=google/gemini-1.5-pro-001,model_deployment=stanfordhealthcare/gemini-1.5-pro-001", priority: 1},
-  {description: "clear:model=openai/gpt-4o-2024-05-13,model_deployment=stanfordhealthcare/gpt-4o-2024-05-13", priority: 1},
-  {description: "clear:model=openai/gpt-4o-mini-2024-07-18,model_deployment=stanfordhealthcare/gpt-4o-mini-2024-07-18", priority: 1},
-  {description: "clear:model=meta/llama-3.3-70b-instruct,model_deployment=stanfordhealthcare/llama-3.3-70b-instruct", priority: 1},
-  {description: "clear:model=qwen/qwen2.5-7b-instruct,model_deployment=huggingface/qwen2.5-7b-instruct-4bit", priority: 1},
-  {description: "clear:model=microsoft/phi-3.5-mini-instruct,model_deployment=huggingface/phi-3.5-mini-instruct-4bit", priority: 1},
+  #Alcohol Dependence
+  {description: "clear:condition=alcohol_dependence,max_eval_instances=100,model=google/gemini-1.5-pro-001,model_deployment=stanfordhealthcare/gemini-1.5-pro-001", priority: 1},
+  {description: "clear:condition=alcohol_dependence,max_eval_instances=100,model=openai/gpt-4o-2024-05-13,model_deployment=stanfordhealthcare/gpt-4o-2024-05-13", priority: 1},
+  {description: "clear:condition=alcohol_dependence,max_eval_instances=100,model=openai/gpt-4o-mini-2024-07-18,model_deployment=stanfordhealthcare/gpt-4o-mini-2024-07-18", priority: 1},
+  {description: "clear:condition=alcohol_dependence,max_eval_instances=100,model=meta/llama-3.3-70b-instruct,model_deployment=stanfordhealthcare/llama-3.3-70b-instruct", priority: 1},
+  {description: "clear:condition=alcohol_dependence,max_eval_instances=100,model=qwen/qwen2.5-7b-instruct,model_deployment=huggingface/qwen2.5-7b-instruct-4bit", priority: 1},
+  {description: "clear:condition=alcohol_dependence,max_eval_instances=100,model=microsoft/phi-3.5-mini-instruct,model_deployment=huggingface/phi-3.5-mini-instruct-4bit", priority: 1},
+  
+  #Attention Deficit Hyperactivity Disorder
+  {description: "clear:condition=attention_deficit_hyperactivity_disorder,max_eval_instances=100,model=google/gemini-1.5-pro-001,model_deployment=stanfordhealthcare/gemini-1.5-pro-001", priority: 1},
+  {description: "clear:condition=attention_deficit_hyperactivity_disorder,max_eval_instances=100,model=openai/gpt-4o-2024-05-13,model_deployment=stanfordhealthcare/gpt-4o-2024-05-13", priority: 1},
+  {description: "clear:condition=attention_deficit_hyperactivity_disorder,max_eval_instances=100,model=openai/gpt-4o-mini-2024-07-18,model_deployment=stanfordhealthcare/gpt-4o-mini-2024-07-18", priority: 1},
+  {description: "clear:condition=attention_deficit_hyperactivity_disorder,max_eval_instances=100,model=meta/llama-3.3-70b-instruct,model_deployment=stanfordhealthcare/llama-3.3-70b-instruct", priority: 1},
+  {description: "clear:condition=attention_deficit_hyperactivity_disorder,max_eval_instances=100,model=qwen/qwen2.5-7b-instruct,model_deployment=huggingface/qwen2.5-7b-instruct-4bit", priority: 1},
+  {description: "clear:condition=attention_deficit_hyperactivity_disorder,max_eval_instances=100,model=microsoft/phi-3.5-mini-instruct,model_deployment=huggingface/phi-3.5-mini-instruct-4bit", priority: 1},
+
+  #Bipolar Disorder
+  {description: "clear:condition=bipolar_disorder,max_eval_instances=100,model=google/gemini-1.5-pro-001,model_deployment=stanfordhealthcare/gemini-1.5-pro-001", priority: 1},
+  {description: "clear:condition=bipolar_disorder,max_eval_instances=100,model=openai/gpt-4o-2024-05-13,model_deployment=stanfordhealthcare/gpt-4o-2024-05-13", priority: 1},
+  {description: "clear:condition=bipolar_disorder,max_eval_instances=100,model=openai/gpt-4o-mini-2024-07-18,model_deployment=stanfordhealthcare/gpt-4o-mini-2024-07-18", priority: 1},
+  {description: "clear:condition=bipolar_disorder,max_eval_instances=100,model=meta/llama-3.3-70b-instruct,model_deployment=stanfordhealthcare/llama-3.3-70b-instruct", priority: 1},
+  {description: "clear:condition=bipolar_disorder,max_eval_instances=100,model=qwen/qwen2.5-7b-instruct,model_deployment=huggingface/qwen2.5-7b-instruct-4bit", priority: 1},
+  {description: "clear:condition=bipolar_disorder,max_eval_instances=100,model=microsoft/phi-3.5-mini-instruct,model_deployment=huggingface/phi-3.5-mini-instruct-4bit", priority: 1},
+
+  #Chronic Pain
+  {description: "clear:condition=chronic_pain,max_eval_instances=100,model=google/gemini-1.5-pro-001,model_deployment=stanfordhealthcare/gemini-1.5-pro-001", priority: 1},
+  {description: "clear:condition=chronic_pain,max_eval_instances=100,model=openai/gpt-4o-2024-05-13,model_deployment=stanfordhealthcare/gpt-4o-2024-05-13", priority: 1},
+  {description: "clear:condition=chronic_pain,max_eval_instances=100,model=openai/gpt-4o-mini-2024-07-18,model_deployment=stanfordhealthcare/gpt-4o-mini-2024-07-18", priority: 1},
+  {description: "clear:condition=chronic_pain,max_eval_instances=100,model=meta/llama-3.3-70b-instruct,model_deployment=stanfordhealthcare/llama-3.3-70b-instruct", priority: 1},
+  {description: "clear:condition=chronic_pain,max_eval_instances=100,model=qwen/qwen2.5-7b-instruct,model_deployment=huggingface/qwen2.5-7b-instruct-4bit", priority: 1},
+  {description: "clear:condition=chronic_pain,max_eval_instances=100,model=microsoft/phi-3.5-mini-instruct,model_deployment=huggingface/phi-3.5-mini-instruct-4bit", priority: 1},
+
+  #Homelessness
+  {description: "clear:condition=homelessness,max_eval_instances=100,model=google/gemini-1.5-pro-001,model_deployment=stanfordhealthcare/gemini-1.5-pro-001", priority: 1},
+  {description: "clear:condition=homelessness,max_eval_instances=100,model=openai/gpt-4o-2024-05-13,model_deployment=stanfordhealthcare/gpt-4o-2024-05-13", priority: 1},
+  {description: "clear:condition=homelessness,max_eval_instances=100,model=openai/gpt-4o-mini-2024-07-18,model_deployment=stanfordhealthcare/gpt-4o-mini-2024-07-18", priority: 1},
+  {description: "clear:condition=homelessness,max_eval_instances=100,model=meta/llama-3.3-70b-instruct,model_deployment=stanfordhealthcare/llama-3.3-70b-instruct", priority: 1},
+  {description: "clear:condition=homelessness,max_eval_instances=100,model=qwen/qwen2.5-7b-instruct,model_deployment=huggingface/qwen2.5-7b-instruct-4bit", priority: 1},
+  {description: "clear:condition=homelessness,max_eval_instances=100,model=microsoft/phi-3.5-mini-instruct,model_deployment=huggingface/phi-3.5-mini-instruct-4bit", priority: 1},
+
+  #Liver Disease
+  {description: "clear:condition=liver_disease,max_eval_instances=100,model=google/gemini-1.5-pro-001,model_deployment=stanfordhealthcare/gemini-1.5-pro-001", priority: 1},
+  {description: "clear:condition=liver_disease,max_eval_instances=100,model=openai/gpt-4o-2024-05-13,model_deployment=stanfordhealthcare/gpt-4o-2024-05-13", priority: 1},
+  {description: "clear:condition=liver_disease,max_eval_instances=100,model=openai/gpt-4o-mini-2024-07-18,model_deployment=stanfordhealthcare/gpt-4o-mini-2024-07-18", priority: 1},
+  {description: "clear:condition=liver_disease,max_eval_instances=100,model=meta/llama-3.3-70b-instruct,model_deployment=stanfordhealthcare/llama-3.3-70b-instruct", priority: 1},
+  {description: "clear:condition=liver_disease,max_eval_instances=100,model=qwen/qwen2.5-7b-instruct,model_deployment=huggingface/qwen2.5-7b-instruct-4bit", priority: 1},
+  {description: "clear:condition=liver_disease,max_eval_instances=100,model=microsoft/phi-3.5-mini-instruct,model_deployment=huggingface/phi-3.5-mini-instruct-4bit", priority: 1},
+  
+  #Major Depression
+  {description: "clear:condition=major_depression,max_eval_instances=100,model=google/gemini-1.5-pro-001,model_deployment=stanfordhealthcare/gemini-1.5-pro-001", priority: 1},
+  {description: "clear:condition=major_depression,max_eval_instances=100,model=openai/gpt-4o-2024-05-13,model_deployment=stanfordhealthcare/gpt-4o-2024-05-13", priority: 1},
+  {description: "clear:condition=major_depression,max_eval_instances=100,model=openai/gpt-4o-mini-2024-07-18,model_deployment=stanfordhealthcare/gpt-4o-mini-2024-07-18", priority: 1},
+  {description: "clear:condition=major_depression,max_eval_instances=100,model=meta/llama-3.3-70b-instruct,model_deployment=stanfordhealthcare/llama-3.3-70b-instruct", priority: 1},
+  {description: "clear:condition=major_depression,max_eval_instances=100,model=qwen/qwen2.5-7b-instruct,model_deployment=huggingface/qwen2.5-7b-instruct-4bit", priority: 1},
+  {description: "clear:condition=major_depression,max_eval_instances=100,model=microsoft/phi-3.5-mini-instruct,model_deployment=huggingface/phi-3.5-mini-instruct-4bit", priority: 1},
+
+  #Personality Disorder
+  {description: "clear:condition=personality_disorder,max_eval_instances=100,model=google/gemini-1.5-pro-001,model_deployment=stanfordhealthcare/gemini-1.5-pro-001", priority: 1},
+  {description: "clear:condition=personality_disorder,max_eval_instances=100,model=openai/gpt-4o-2024-05-13,model_deployment=stanfordhealthcare/gpt-4o-2024-05-13", priority: 1},
+  {description: "clear:condition=personality_disorder,max_eval_instances=100,model=openai/gpt-4o-mini-2024-07-18,model_deployment=stanfordhealthcare/gpt-4o-mini-2024-07-18", priority: 1},
+  {description: "clear:condition=personality_disordern,max_eval_instances=100,model=meta/llama-3.3-70b-instruct,model_deployment=stanfordhealthcare/llama-3.3-70b-instruct", priority: 1},
+  {description: "clear:condition=personality_disorder,max_eval_instances=100,model=qwen/qwen2.5-7b-instruct,model_deployment=huggingface/qwen2.5-7b-instruct-4bit", priority: 1},
+  {description: "clear:condition=personality_disorder,max_eval_instances=100,model=microsoft/phi-3.5-mini-instruct,model_deployment=huggingface/phi-3.5-mini-instruct-4bit", priority: 1},
+
+  #Post-Traumatic Stress Disorder
+  {description: "clear:condition=post_traumatic_stress_disorder,max_eval_instances=100,model=google/gemini-1.5-pro-001,model_deployment=stanfordhealthcare/gemini-1.5-pro-001", priority: 1},
+  {description: "clear:condition=post_traumatic_stress_disorder,max_eval_instances=100,model=openai/gpt-4o-2024-05-13,model_deployment=stanfordhealthcare/gpt-4o-2024-05-13", priority: 1},
+  {description: "clear:condition=post_traumatic_stress_disorder,max_eval_instances=100,model=openai/gpt-4o-mini-2024-07-18,model_deployment=stanfordhealthcare/gpt-4o-mini-2024-07-18", priority: 1},
+  {description: "clear:condition=post_traumatic_stress_disorder,max_eval_instances=100,model=meta/llama-3.3-70b-instruct,model_deployment=stanfordhealthcare/llama-3.3-70b-instruct", priority: 1},
+  {description: "clear:condition=post_traumatic_stress_disorder,max_eval_instances=100,model=qwen/qwen2.5-7b-instruct,model_deployment=huggingface/qwen2.5-7b-instruct-4bit", priority: 1},
+  {description: "clear:condition=post_traumatic_stress_disorder,max_eval_instances=100,model=microsoft/phi-3.5-mini-instruct,model_deployment=huggingface/phi-3.5-mini-instruct-4bit", priority: 1},
+
+  #Substance Use Disorder
+  {description: "clear:condition=substance_use_disorder,max_eval_instances=100,model=google/gemini-1.5-pro-001,model_deployment=stanfordhealthcare/gemini-1.5-pro-001", priority: 1},
+  {description: "clear:condition=substance_use_disorder,max_eval_instances=100,model=openai/gpt-4o-2024-05-13,model_deployment=stanfordhealthcare/gpt-4o-2024-05-13", priority: 1},
+  {description: "clear:condition=substance_use_disorder,max_eval_instances=100,model=openai/gpt-4o-mini-2024-07-18,model_deployment=stanfordhealthcare/gpt-4o-mini-2024-07-18", priority: 1},
+  {description: "clear:condition=substance_use_disorder,max_eval_instances=100,model=meta/llama-3.3-70b-instruct,model_deployment=stanfordhealthcare/llama-3.3-70b-instruct", priority: 1},
+  {description: "clear:condition=substance_use_disorder,max_eval_instances=100,model=qwen/qwen2.5-7b-instruct,model_deployment=huggingface/qwen2.5-7b-instruct-4bit", priority: 1},
+  {description: "clear:condition=substance_use_disorder,max_eval_instances=100,model=microsoft/phi-3.5-mini-instruct,model_deployment=huggingface/phi-3.5-mini-instruct-4bit", priority: 1},
+
+  #Suicidal Behavior
+  {description: "clear:condition=suicidal_behavior,max_eval_instances=100,model=google/gemini-1.5-pro-001,model_deployment=stanfordhealthcare/gemini-1.5-pro-001", priority: 1},
+  {description: "clear:condition=suicidal_behavior,max_eval_instances=100,model=openai/gpt-4o-2024-05-13,model_deployment=stanfordhealthcare/gpt-4o-2024-05-13", priority: 1},
+  {description: "clear:condition=suicidal_behavior,max_eval_instances=100,model=openai/gpt-4o-mini-2024-07-18,model_deployment=stanfordhealthcare/gpt-4o-mini-2024-07-18", priority: 1},
+  {description: "clear:condition=suicidal_behavior,max_eval_instances=100,model=meta/llama-3.3-70b-instruct,model_deployment=stanfordhealthcare/llama-3.3-70b-instruct", priority: 1},
+  {description: "clear:condition=suicidal_behavior,max_eval_instances=100,model=qwen/qwen2.5-7b-instruct,model_deployment=huggingface/qwen2.5-7b-instruct-4bit", priority: 1},
+  {description: "clear:condition=suicidal_behavior,max_eval_instances=100,model=microsoft/phi-3.5-mini-instruct,model_deployment=huggingface/phi-3.5-mini-instruct-4bit", priority: 1},
+
+  #Tobacco Dependence
+  {description: "clear:condition=tobacco_dependence,max_eval_instances=100,model=google/gemini-1.5-pro-001,model_deployment=stanfordhealthcare/gemini-1.5-pro-001", priority: 1},
+  {description: "clear:condition=tobacco_dependence,max_eval_instances=100,model=openai/gpt-4o-2024-05-13,model_deployment=stanfordhealthcare/gpt-4o-2024-05-13", priority: 1},
+  {description: "clear:condition=tobacco_dependence,max_eval_instances=100,model=openai/gpt-4o-mini-2024-07-18,model_deployment=stanfordhealthcare/gpt-4o-mini-2024-07-18", priority: 1},
+  {description: "clear:condition=tobacco_dependence,max_eval_instances=100,model=meta/llama-3.3-70b-instruct,model_deployment=stanfordhealthcare/llama-3.3-70b-instruct", priority: 1},
+  {description: "clear:condition=tobacco_dependence,max_eval_instances=100,model=qwen/qwen2.5-7b-instruct,model_deployment=huggingface/qwen2.5-7b-instruct-4bit", priority: 1},
+  {description: "clear:condition=tobacco_dependence,max_eval_instances=100,model=microsoft/phi-3.5-mini-instruct,model_deployment=huggingface/phi-3.5-mini-instruct-4bit", priority: 1},
+ 
+  #Unemployment
+  {description: "clear:condition=unemployment,max_eval_instances=100,model=google/gemini-1.5-pro-001,model_deployment=stanfordhealthcare/gemini-1.5-pro-001", priority: 1},
+  {description: "clear:condition=unemployment,max_eval_instances=100,model=openai/gpt-4o-2024-05-13,model_deployment=stanfordhealthcare/gpt-4o-2024-05-13", priority: 1},
+  {description: "clear:condition=unemployment,max_eval_instances=100,model=openai/gpt-4o-mini-2024-07-18,model_deployment=stanfordhealthcare/gpt-4o-mini-2024-07-18", priority: 1},
+  {description: "clear:condition=unemployment,max_eval_instances=100,model=meta/llama-3.3-70b-instruct,model_deployment=stanfordhealthcare/llama-3.3-70b-instruct", priority: 1},
+  {description: "clear:condition=unemployment,max_eval_instances=100,model=qwen/qwen2.5-7b-instruct,model_deployment=huggingface/qwen2.5-7b-instruct-4bit", priority: 1},
+  {description: "clear:condition=unemployment,max_eval_instances=100,model=microsoft/phi-3.5-mini-instruct,model_deployment=huggingface/phi-3.5-mini-instruct-4bit", priority: 1},
 
   ### Planning Treatments ###
   {description: "mtsamples_replicate:model=google/gemini-1.5-pro-001,model_deployment=stanfordhealthcare/gemini-1.5-pro-001", priority: 1},

src/helm/benchmark/run_specs/medhelm_run_specs.py

@@ -56,14 +56,19 @@ def get_medcalc_bench_spec() -> RunSpec:
 
 
 @run_spec_function("clear")
-def get_clear_spec() -> RunSpec:
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.clear_scenario.CLEARScenario", args={})
+def get_clear_spec(condition: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.clear_scenario.CLEARScenario", args={"condition": condition}
+    )
+
+    condition_display = condition.replace("_", " ")
 
     adapter_spec = get_multiple_choice_adapter_spec(
         method=ADAPT_MULTIPLE_CHOICE_JOINT,
         instructions=(
-            "Answer 'A' for 'Has a history of alcohol dependence', "
-            "'B' for 'Does not have a history of alcohol dependence;, or 'C' for 'Uncertain'"
+            f"Answer 'A' for 'Has a history of {condition_display}', "
+            f"'B' for 'Does not have a history of {condition_display}', or "
+            f"'C' for 'Uncertain'"
         ),
         input_noun=None,
         output_noun="Respond only with 'A', 'B', or 'C'. Do not add any other text, punctuation, or symbols",
@@ -72,7 +77,7 @@ def get_clear_spec() -> RunSpec:
     )
 
     return RunSpec(
-        name="clear",
+        name=f"clear:condition={condition}",
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs(),