From 8819b91b6465ddfe3ff104c95acec5c8edbc9684 Mon Sep 17 00:00:00 2001 From: ljvmiranda921 Date: Fri, 2 Aug 2024 21:59:35 -0700 Subject: [PATCH 1/2] Update namespace --- scripts/run_generative.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run_generative.py b/scripts/run_generative.py index 0571e86..fa50bc3 100644 --- a/scripts/run_generative.py +++ b/scripts/run_generative.py @@ -266,7 +266,7 @@ def format_judgements(batch, optional_chat_template=None): answer_b, multi_turn=mult_turn, model_modifier=model_modifier, - include_langs=args.include_langs, + include_langs=args.include_languages, ) if optional_chat_template is not None: From 10215cb8c1b2a2e8d87e5eb9999f41827025d6c2 Mon Sep 17 00:00:00 2001 From: Lj Miranda Date: Fri, 2 Aug 2024 22:21:51 -0700 Subject: [PATCH 2/2] Fix shuffle assignment --- scripts/run_generative.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/run_generative.py b/scripts/run_generative.py index fa50bc3..f1d36f0 100644 --- a/scripts/run_generative.py +++ b/scripts/run_generative.py @@ -302,7 +302,12 @@ def format_judgements(batch, optional_chat_template=None): winners = [process_judgement(a, is_prometheus=is_prometheus) for a in answers] def process_shuffled(win, shuffle): - winner_text, loser_text = "B", "A" if shuffle else "A", "B" + if shuffle: + winner_text = "B" + loser_text = "A" + else: + winner_text = "A" + loser_text = "B" if win == winner_text: return 1 elif win == loser_text: