Use Anthropic tokenizer from Hugging Face (#3467)

Author: yifanmai

setup.cfg

@@ -139,7 +139,7 @@ amazon =
     botocore~=1.34.131
 
 anthropic =
-    anthropic~=0.17,<0.39  # TODO(#3212): Limit anthropic to >=0.39 after resolving #3212.
+    anthropic~=0.39
     websocket-client~=1.3.2  # For legacy stanford-online-all-v4-s3
     httpx<0.28.0  # TODO(#3324): Remove this tepmorary workaround
 

src/helm/config/tokenizer_configs.yaml

@@ -86,7 +86,9 @@ tokenizer_configs:
   # Anthropic
   - name: anthropic/claude
     tokenizer_spec:
-      class_name: "helm.tokenizers.anthropic_tokenizer.AnthropicTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: Xenova/claude-tokenizer
     end_of_text_token: "<|endoftext|>"
     prefix_token: "<|endoftext|>"
 

src/helm/tokenizers/anthropic_tokenizer.py

@@ -1,6 +1,6 @@
 from abc import abstractmethod
 from dataclasses import asdict
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List
 
 from helm.common.cache import Cache, CacheConfig
 from helm.common.request import wrap_request_time
@@ -153,31 +153,3 @@ def decode(self, request: DecodeRequest) -> DecodeRequestResult:
             )
         except Exception as error:
             raise ValueError(f"Failed to decode tokens with {self.__class__.__name__} tokenizer: {error}") from error
-
-
-def cleanup_str(token: str, tokenizer_name: Optional[str] = None) -> str:
-    """
-    Certain tokenizers introduce special characters to represent spaces, such as
-    "Ġ" or "▁". This function removes those characters.
-    """
-    if tokenizer_name in [
-        "TsinghuaKEG/ice",
-        "bigscience/T0pp",
-        "google/t5-11b",
-        "google/flan-t5-xxl",
-        "google/ul2",
-        "Yandex/yalm",
-        "ai21/j1",
-        "together",
-    ]:
-        return token.replace("▁", " ")
-    elif tokenizer_name is not None and tokenizer_name.startswith("huggingface"):
-        return token.replace("Ġ", " ")
-    return token
-
-
-def cleanup_tokens(tokens: List[str], tokenizer_name: Optional[str] = None) -> List[str]:
-    """
-    Applies `cleanup_str` to each token in `tokens`.
-    """
-    return [cleanup_str(token, tokenizer_name) for token in tokens]

src/helm/tokenizers/caching_tokenizer.py

@@ -10,7 +10,7 @@
     TokenizationRequest,
     TokenizationRequestResult,
 )
-from helm.tokenizers.anthropic_tokenizer import AnthropicTokenizer
+from helm.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer
 
 
 class TestAnthropicTokenizer:
@@ -21,7 +21,11 @@ class TestAnthropicTokenizer:
     def setup_method(self, method):
         cache_file = tempfile.NamedTemporaryFile(delete=False)
         self.cache_path: str = cache_file.name
-        self.tokenizer = AnthropicTokenizer(SqliteCacheConfig(self.cache_path))
+        self.tokenizer = HuggingFaceTokenizer(
+            SqliteCacheConfig(self.cache_path),
+            tokenizer_name="anthropic/claude",
+            pretrained_model_name_or_path="Xenova/claude-tokenizer",
+        )
 
     def teardown_method(self, method):
         os.remove(self.cache_path)

src/helm/tokenizers/test_anthropic_tokenizer.py

@@ -41,7 +41,9 @@ def cleanup_str(token: str, tokenizer_name: Optional[str] = None) -> str:
         "together",
     ]:
         return token.replace("▁", " ")
-    elif tokenizer_name is not None and tokenizer_name.startswith("huggingface"):
+    elif tokenizer_name is not None and (
+        tokenizer_name.startswith("huggingface") or tokenizer_name == "anthropic/claude"
+    ):
         return token.replace("Ġ", " ")
     return token