need additional_special_tokens argument for HFLM initializer

Stability-AI · Oct 28, 2023 · 3e3aaa6 · 3e3aaa6
1 parent 9b42d41
commit 3e3aaa6
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/lm_eval/models/gpt2.py b/lm_eval/models/gpt2.py
@@ -19,6 +19,7 @@ def __init__(
         load_in_8bit: Optional[bool] = False,
         trust_remote_code: Optional[bool] = False,
         use_fast: Optional[bool] = True,
+        additional_special_tokens: Optional[str] = None
     ):
         super().__init__()
 
@@ -64,6 +65,7 @@ def __init__(
             revision=revision,
             trust_remote_code=trust_remote_code,
             use_fast=use_fast,
+            additional_special_tokens=additional_special_tokens
         )
         self.vocab_size = self.tokenizer.vocab_size