fix: Fixed AttributeError: 'ChatGLMTokenizer' object has no attribute…

… 'tokenizer'. Tested on transformers==4.38.1 (DLLXW#45, DLLXW#63)
noahc1510 · Mar 30, 2024 · ca37e1f · ca37e1f
1 parent 444155b
commit ca37e1f
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/chatglm_tokenizer/tokenization_chatglm.py b/chatglm_tokenizer/tokenization_chatglm.py
@@ -66,11 +66,11 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
     model_input_names = ["input_ids", "attention_mask", "position_ids"]
 
     def __init__(self, vocab_file, padding_side="left", clean_up_tokenization_spaces=False, **kwargs):
+        self.tokenizer = SPTokenizer(vocab_file)
         super().__init__(padding_side=padding_side, clean_up_tokenization_spaces=clean_up_tokenization_spaces, **kwargs)
         self.name = "GLMTokenizer"
 
         self.vocab_file = vocab_file
-        self.tokenizer = SPTokenizer(vocab_file)
         self.special_tokens = {
             "<bos>": self.tokenizer.bos_id,
             "<eos>": self.tokenizer.eos_id,