From d8858680f7ea89c5c274aef746004a7ba0d00317 Mon Sep 17 00:00:00 2001 From: Barbara Suslova Date: Sat, 27 Apr 2024 09:27:57 +0000 Subject: [PATCH 1/3] remove leading whitespace in checks --- lm_eval/models/octoai_llms.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lm_eval/models/octoai_llms.py b/lm_eval/models/octoai_llms.py index ecfebb33c9..63d8ac8963 100644 --- a/lm_eval/models/octoai_llms.py +++ b/lm_eval/models/octoai_llms.py @@ -248,7 +248,7 @@ def get_llama_token(self, token: str): sym = bytes.fromhex("e29681").decode("utf-8") # workaround for case sym + "_" if token.startswith("_" + sym): - res = token.replace("_" + sym, " ", 1) + res = token.replace("_" + sym, " ", 1) elif token.startswith(sym): res = token.replace(sym, " ") return res @@ -271,18 +271,18 @@ def get_result(self, response, context, continuation): prob_cont = "" while prob_ctx.endswith(token): prob_cont = token + prob_cont - if continuation == prob_cont: + if continuation in prob_cont: break prob_ctx = prob_ctx[:-len(token)] cont_len += 1 token = self.get_llama_token(tokens[-cont_len]) try: - assert continuation.startswith(token), f"Tokenization issue, wrong token: \"{token}\"" + assert continuation.startswith(token.strip()), f"Tokenization issue, wrong token: \"{token}\"" except: - print("CONTEXT:", context) - print("CONTINUATION:", continuation) - print("TOKENS:", tokens) - print("TOKEN:", f"\"{token}\"") + print("CONTEXT:", context, sep="\n") + print("CONTINUATION:", continuation, sep="\n") + print("TOKENS:", tokens[-cont_len:], sep="\n") + print("TOKEN:", f"\"{token}\"", sep="\n") return self.dummy_result() res_logprob = sum(logprobs[-cont_len:]) From b3956e599520a73e0fd05c6e861654fd7138bad7 Mon Sep 17 00:00:00 2001 From: Barbara Suslova Date: Tue, 14 May 2024 08:39:28 +0000 Subject: [PATCH 2/3] fix bigbench task --- lm_eval/models/octoai_llms.py | 6 +++--- lm_eval/tasks/bigbench.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lm_eval/models/octoai_llms.py b/lm_eval/models/octoai_llms.py index 63d8ac8963..330f39c4c4 100644 --- a/lm_eval/models/octoai_llms.py +++ b/lm_eval/models/octoai_llms.py @@ -248,7 +248,7 @@ def get_llama_token(self, token: str): sym = bytes.fromhex("e29681").decode("utf-8") # workaround for case sym + "_" if token.startswith("_" + sym): - res = token.replace("_" + sym, " ", 1) + res = token.replace("_" + sym, " ", 1) elif token.startswith(sym): res = token.replace(sym, " ") return res @@ -271,13 +271,13 @@ def get_result(self, response, context, continuation): prob_cont = "" while prob_ctx.endswith(token): prob_cont = token + prob_cont - if continuation in prob_cont: + if continuation == prob_cont: break prob_ctx = prob_ctx[:-len(token)] cont_len += 1 token = self.get_llama_token(tokens[-cont_len]) try: - assert continuation.startswith(token.strip()), f"Tokenization issue, wrong token: \"{token}\"" + assert continuation.startswith(token), f"Tokenization issue, wrong token: \"{token}\"" except: print("CONTEXT:", context, sep="\n") print("CONTINUATION:", continuation, sep="\n") diff --git a/lm_eval/tasks/bigbench.py b/lm_eval/tasks/bigbench.py index 0d1a7daffb..61576fd0f1 100644 --- a/lm_eval/tasks/bigbench.py +++ b/lm_eval/tasks/bigbench.py @@ -70,7 +70,7 @@ def doc_to_text(self, doc): res = f"{res}{choice_prefix}{choice_prefix.join(permuted_choices)}" example_output_prefix = self._task_json.get("example_output_prefix", "\nA: ") - res = f"{res}{example_output_prefix}" + res = f"{res}{example_output_prefix.rstrip()}" return res def doc_to_target(self, doc): @@ -86,7 +86,7 @@ def construct_requests(self, doc, ctx): if self._has_multi_choice: queries = self._doc_to_queries(doc) requests += [ - rf.loglikelihood(ctx, continuation)[0] for continuation in queries + rf.loglikelihood(ctx, " " + continuation)[0] for continuation in queries ] if self._has_generative: requests.append( From e54b2f1a8b264210a1ebe57f77a94cd24988e3a1 Mon Sep 17 00:00:00 2001 From: Barbara Suslova Date: Wed, 15 May 2024 08:33:26 +0000 Subject: [PATCH 3/3] fix position --- lm_eval/tasks/bigbench.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lm_eval/tasks/bigbench.py b/lm_eval/tasks/bigbench.py index 61576fd0f1..4e88686948 100644 --- a/lm_eval/tasks/bigbench.py +++ b/lm_eval/tasks/bigbench.py @@ -70,8 +70,8 @@ def doc_to_text(self, doc): res = f"{res}{choice_prefix}{choice_prefix.join(permuted_choices)}" example_output_prefix = self._task_json.get("example_output_prefix", "\nA: ") - res = f"{res}{example_output_prefix.rstrip()}" - return res + res = f"{res}{example_output_prefix}" + return res.rstrip() def doc_to_target(self, doc): return max(doc["target_scores"].items(), key=lambda x: x[1])[0]