fix: add tokenizer customization back

bigcode-project · Nov 12, 2024 · 8645863 · 8645863
1 parent fc3034a
commit 8645863
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/bigcodebench/provider/__init__.py b/bigcodebench/provider/__init__.py
@@ -41,6 +41,8 @@ def make_model(
             instruction_prefix=instruction_prefix,
             response_prefix=response_prefix,
             trust_remote_code=trust_remote_code,
+            tokenizer_name=tokenizer_name,
+            tokenizer_legacy=tokenizer_legacy,
         )
     elif backend == "hf":
         from bigcodebench.provider.hf import HuggingFaceDecoder
@@ -58,6 +60,8 @@ def make_model(
             response_prefix=response_prefix,
             attn_implementation=attn_implementation,
             trust_remote_code=trust_remote_code,
+            tokenizer_name=tokenizer_name,
+            tokenizer_legacy=tokenizer_legacy,
         )
     elif backend == "openai":
         from bigcodebench.provider.openai import OpenAIChatDecoder