Skip to content

Commit

Permalink
remove debug message - update version 0.1.4
Browse files Browse the repository at this point in the history
  • Loading branch information
SamuelCahyawijaya committed Jun 17, 2022
1 parent 6f5d530 commit 9be04f5
Show file tree
Hide file tree
Showing 2 changed files with 1 addition and 6 deletions.
5 changes: 0 additions & 5 deletions src/indobenchmark/tokenization_indonlg.py
Original file line number Diff line number Diff line change
Expand Up @@ -309,23 +309,18 @@ def convert_ids_to_tokens(
"""
if isinstance(ids, int):
if ids not in self.added_tokens_decoder or ids in self.special_tokens_to_ids:
print('is int', ids, skip_special_tokens)
return self._convert_id_to_token(ids, skip_special_tokens=skip_special_tokens)
else:
print('add token decoder', ids, skip_special_tokens)
return self.added_tokens_decoder[ids]
tokens = []
for index in ids:
index = int(index)
if skip_special_tokens and index in (self.all_special_ids + list(self.special_tokens_to_ids.values())):
continue
if index not in self.added_tokens_decoder or index in self.special_tokens_to_ids:
print('is list', index, skip_special_tokens)
tokens.append(self._convert_id_to_token(index, skip_special_tokens=skip_special_tokens))
else:
print('list add token decoder', index, skip_special_tokens)
tokens.append(self.added_tokens_decoder[index])
print()
return tokens

def _convert_token_to_id(self, token):
Expand Down
2 changes: 1 addition & 1 deletion src/indobenchmark/version.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@
# 1) we don't load dependencies by storing it in __init__.py
# 2) we can import it in setup.py for the same reason
# 3) we can import it into your module module
__version__ = '0.1.3'
__version__ = '0.1.4'

0 comments on commit 9be04f5

Please sign in to comment.