Speech specs + fix FLEURS + add additional audio dependencies (#3151)

Author: teetone

setup.cfg

@@ -275,6 +275,7 @@ heim =
 
 audiolm =
     crfm-helm[openai]
+    crfm-helm[google]
 
     # For HuggingFace audio datasets
     soundfile~=0.12

src/helm/benchmark/presentation/run_entries_speech.conf

@@ -1,16 +1,71 @@
 entries: [
+    ####################################################################################################################
+    # Auditory Perception
+    ####################################################################################################################
+    {description: "meld_audio:model=audiolm", priority: 1}
+    {description: "vocal_sound:model=audiolm", priority: 1}
+    {description: "audiocaps:model=audiolm", priority: 1}
+
+    ####################################################################################################################
+    # Fairness
+    ####################################################################################################################
+
+    ####################################################################################################################
+    # Robustness
+    ####################################################################################################################
+
     {description: "audio_mnist:model=audiolm", priority: 1}
 
-    # TODO: populate with the rest of the languages
-    {description: "covost2:source_language=English,target_language=Chinese,model=audiolm", priority: 1}
+    {description: "speech_robust_bench:subject=accented_cv,model=audiolm", priority: 1}
+    {description: "speech_robust_bench:subject=accented_cv_es,model=audiolm", priority: 1}
+    {description: "speech_robust_bench:subject=chime_far,model=audiolm", priority: 1}
+    {description: "speech_robust_bench:subject=chime_near,model=audiolm", priority: 1}
+    {description: "speech_robust_bench:subject=ami_far,model=audiolm", priority: 1}
+    {description: "speech_robust_bench:subject=ami_near,model=audiolm", priority: 1}
+
+
+    ####################################################################################################################
+    # Bias
+    ####################################################################################################################
 
-    {description: "vocal_sound:model=audiolm", priority: 1}
+
+    ####################################################################################################################
+    # Multilinguality
+    ####################################################################################################################
 
     # TODO: populate with the rest of the languages
+    {description: "covost2:source_language=English,target_language=Chinese,model=audiolm", priority: 1}
+
+    # There are other languages in the multilinguality aspect
     {description: "multilingual_librispeech:language=Dutch,model=audiolm", priority: 1}
+    {description: "multilingual_librispeech:language=Spanish,model=audiolm", priority: 1}
+    {description: "multilingual_librispeech:language=Italian,model=audiolm", priority: 1}
+    {description: "multilingual_librispeech:language=German,model=audiolm", priority: 1}
+    {description: "multilingual_librispeech:language=French,model=audiolm", priority: 1}
+    {description: "multilingual_librispeech:language=Portuguese,model=audiolm", priority: 1}
+    {description: "multilingual_librispeech:language=Polish,model=audiolm", priority: 1}
 
-    # TODO: populate with the rest of the languages
     {description: "fleurs:language=Finnish,model=audiolm", priority: 1}
+    {description: "fleurs:language=Bulgarian,model=audiolm", priority: 1}
+    {description: "fleurs:language=Hebrew,model=audiolm", priority: 1}
+    {description: "fleurs:language=Zulu,model=audiolm", priority: 1}
+    {description: "fleurs:language=Bengali,model=audiolm", priority: 1}
+    {description: "fleurs:language=Thai,model=audiolm", priority: 1}
+    {description: "fleurs:language=Mandarin_Chinese,model=audiolm", priority: 1}
+
+    {description: "common_voice_15:language=English,model=audiolm", priority: 1}
+    {description: "common_voice_15:language=Chinese_hk,model=audiolm", priority: 1}
+    {description: "common_voice_15:language=German,model=audiolm", priority: 1}
+    {description: "common_voice_15:language=French,model=audiolm", priority: 1}
+
+    ####################################################################################################################
+    # Toxicity
+    ####################################################################################################################
+
+
+    ####################################################################################################################
+    # Safety
+    ####################################################################################################################
+
 
-    {description: "audiocaps:model=audiolm", priority: 1}
 ]

src/helm/benchmark/scenarios/audio_language/common_voice_15_scenario.py

@@ -86,6 +86,7 @@ def get_instances(self, output_path: str) -> List[Instance]:
                 name=language_category,
                 cache_dir=output_path,
                 split=TEST_SPLIT,
+                trust_remote_code=True,
             )
         ):
             local_audio_path = row["path"]

src/helm/benchmark/scenarios/audio_language/fleurs_scenario.py

@@ -58,8 +58,8 @@ class FLEURSScenario(Scenario):
             ("Burmese", "my"),
             ("Catalan", "ca"),
             ("Cebuano", "ceb"),
-            ("Mandarin Chinese", "cmn_hans"),
-            ("Cantonese Chinese", "yue_hant"),
+            ("Mandarin_Chinese", "cmn_hans"),
+            ("Cantonese_Chinese", "yue_hant"),
             ("Croatian", "hr"),
             ("Czech", "cs"),
             ("Danish", "da"),
@@ -278,9 +278,10 @@ class FLEURSScenario(Scenario):
     def __init__(self, language: str) -> None:
         super().__init__()
 
-        language = language.capitalize()
         if language not in FLEURSScenario._FLEURS_TEST_LANG_TO_ID.keys():
-            raise ValueError(f"Invalid language. Valid languages are: {FLEURSScenario._FLEURS_TEST_LANG_TO_ID.keys()}")
+            raise ValueError(
+                f"Invalid language: {language}. Valid languages are: {FLEURSScenario._FLEURS_TEST_LANG_TO_ID.keys()}"
+            )
 
         self._fleurs_lang_short_to_long = {v: k for k, v in FLEURSScenario._FLEURS_LANG_TO_ID.items()}
         self._fleurs_long_to_lang = {
@@ -298,6 +299,7 @@ def get_instances(self, output_path: str) -> List[Instance]:
                 name=f"fleurs.{language_category}",
                 cache_dir=output_path,
                 split=TEST_SPLIT,
+                trust_remote_code=True,
             )
         ):
             local_audio_path = row["path"]

src/helm/benchmark/scenarios/audio_language/multilingual_librispeech_scenario.py

@@ -46,7 +46,7 @@ class MultilingualLibriSpeechScenario(Scenario):
     description = (
         "Speech recognition in 7 different languages ([Pratap et al, 2022](https://arxiv.org/abs/2012.03411))."
     )
-    tags: List[str] = ["audio", "multilinguality", "regocnition"]
+    tags: List[str] = ["audio", "multilinguality", "recognition"]
 
     def __init__(self, language: str) -> None:
         super().__init__()