Merge branch 'hf-cache' of https://github.com/IBM/unitxt into hf-cache

Signed-off-by: elronbandel <[email protected]>

prepare/cards/fin_qa.py

@@ -50,7 +50,8 @@
                 ["table-average", "table header", "number", "the average of one table row"],
                 ["table-max", "table header", "number", "the maximum number of one table row"],
                 ["table-min", "table header", "number", "the minimum number of one table row"]]
-                Answer with only the program, without any additional explanation.
+                \nAnswer with only the program, without any additional explanation or introductory text.
+                \nHere are some input-output examples. Read the examples carefully to figure out the mapping. The output of the last example is not given, and your job is to figure out what it is.
                 """,
             input_format="""Pre-table text: {pre_text}
                 Table: {table}
@@ -59,7 +60,9 @@
                 Program:
                     """,
             output_format="{program_re}",
-            postprocessors=[],
+            postprocessors=[
+                "processors.take_first_non_empty_line",
+            ],
         ),
     ],
     __description__=(

prepare/cards/quality.py

@@ -0,0 +1,72 @@
+import unitxt
+from unitxt.card import TaskCard
+from unitxt.catalog import add_to_catalog
+from unitxt.collections_operators import Explode
+from unitxt.loaders import LoadCSV
+from unitxt.operators import (
+    Copy,
+    MapInstanceValues,
+    Set,
+)
+from unitxt.splitters import SplitRandomMix
+from unitxt.test_utils.card import test_card
+
+file_path = "https://raw.githubusercontent.com/nyu-mll/quality/05e85750d4c5444d2a0a4ad299f6df5f4df06068/data/v1.0.1/QuALITY.v1.0.1.htmlstripped."
+
+with unitxt.settings.context(allow_unverified_code=True):
+    card = TaskCard(
+        loader=LoadCSV(
+            files={"train": file_path + "train", "validation": file_path + "dev"},
+            file_type="json",
+            lines=True,
+            data_classification_policy=["public"],
+        ),
+        preprocess_steps=[
+            SplitRandomMix(
+                {
+                    "train": "train[80%]",
+                    "validation": "train[20%]",
+                    "test": "validation",
+                }
+            ),
+            Copy(field="article", to_field="context"),
+            Explode(field="questions", to_field="data"),
+            Copy(field="data/question", to_field="question"),
+            Copy(field="data/options", to_field="choices"),
+            Copy(field="data/gold_label", to_field="answer"),
+            MapInstanceValues(
+                mappers={
+                    "answer": {
+                        "1": 0,
+                        "2": 1,
+                        "3": 2,
+                        "4": 3,
+                        "5": 4,
+                    }
+                }
+            ),
+            Set(fields={"context_type": "document"}),
+        ],
+        task="tasks.qa.multiple_choice.with_context",
+        templates="templates.qa.multiple_choice.with_context.all",
+        __description__=(
+            """QuALITY (Question Answering with Long Input Texts, Yes!) is a multiple-choice reading comprehension dataset with long documents. The dataset comprises of documents from Project Gutenberg and questions written by human annotators. Each question has 4-5 answer choices, and requires understanding of the entire document to answer correctly. Questions are designed to test comprehensive understanding of the entire document, with various difficulty levels."""
+        ),
+        __tags__={
+            "annotations_creators": "expert-generated",
+            "language": ["en"],
+            "license": "cc-by-4.0",
+            "size_categories": ["10K<n<100K"],
+            "task_categories": [
+                "question-answering",
+                "multiple-choice",
+                "reading-comprehension",
+            ],
+            "multilinguality": "monolingual",
+            "task_ids": ["extractive-qa", "reading-comprehension"],
+        },
+    )
+
+    # Test and add the card to the catalog
+    test_card(card, strict=False)
+    add_to_catalog(card, "cards.quality", overwrite=True)

prepare/cards/scigen.py

@@ -28,7 +28,7 @@
                 }
             ),
         ],
-        task="tasks.generation.from_pair[metrics=[metrics.llm_as_judge.rating.llama_3_1_70b_instruct_cross_provider_template_table2text_single_turn_with_reference]]",
+        task="tasks.generation.from_pair[metrics=[metrics.rouge,metrics.bert_score.bert_base_uncased,metrics.bleu,metrics.meteor,metrics.llm_as_judge.rating.llama_3_1_70b_instruct_cross_provider_template_table2text_single_turn_with_reference]]",
         templates=[
             "templates.generation.from_pair.default[postprocessors=[processors.lower_case]]"
         ],

prepare/cards/tab_fact.py

@@ -32,7 +32,9 @@
         task="tasks.classification.multi_class.relation",
         templates=[
             InputOutputTemplate(
-                instruction="Given a {text_a_type} and {text_b_type} classify the {type_of_relation} of the {text_b_type} to one of {classes}. You should only output the result. Do not add any explanation or other information.",
+                instruction="Given a {text_a_type} and {text_b_type} classify the {type_of_relation} of the {text_b_type} to one of {classes}."
+                + "\nOutput only the final answer without any explanations, extra information, or introductory text."
+                + "\nHere are some input-output examples. Read the examples carefully to figure out the mapping. The output of the last example is not given, and your job is to figure out what it is.",
                 input_format="{text_a_type}: {text_a}\n{text_b_type}: {text_b} ",
                 output_format="{label}",
                 postprocessors=[
@@ -52,5 +54,5 @@
         ),
     )
 
-    test_card(card)
+    test_card(card, num_demos=2, demos_pool_size=20)
     add_to_catalog(card, "cards.tab_fact", overwrite=True)

prepare/cards/tablebench_data_analysis.py

@@ -51,11 +51,14 @@
     ),
     templates=[
         InputOutputTemplate(
-            instruction="You are a table analyst. Your task is to answer questions based on the table content. {answer_formatter}",
+            instruction="You are a table analyst. Your task is to answer questions based on the table content. {answer_formatter}"
+            + "\nOutput only the final answer without any explanations, extra information, or introductory text."
+            + "\nHere are some input-output examples. Read the examples carefully to figure out the mapping. The output of the last example is not given, and your job is to figure out what it is.",
             input_format="{context_type}: {context} \nQuestion: {question}",
             target_prefix="Final Answer: ",
             output_format="{answers}",
             postprocessors=[
+                "processors.take_first_non_empty_line",
                 "processors.lower_case",
                 "processors.remove_punctuations",
                 "processors.remove_articles",

prepare/cards/tablebench_fact_checking.py

@@ -56,11 +56,14 @@
     ),
     templates=[
         InputOutputTemplate(
-            instruction="You are a table analyst. Your task is to answer questions based on the table content. {answer_formatter}",
+            instruction="You are a table analyst. Your task is to answer questions based on the table content. {answer_formatter}"
+            + "\nOutput only the final answer without any explanations, extra information, or introductory text."
+            + "\nHere are some input-output examples. Read the examples carefully to figure out the mapping. The output of the last example is not given, and your job is to figure out what it is.",
             input_format="{context_type}: {context} \nQuestion: {question}",
             target_prefix="Final Answer: ",
             output_format="{answers}",
             postprocessors=[
+                "processors.take_first_non_empty_line",
                 "processors.lower_case",
                 "processors.remove_punctuations",
                 "processors.remove_articles",

prepare/cards/tablebench_numerical_reasoning.py

@@ -51,11 +51,14 @@
     ),
     templates=[
         InputOutputTemplate(
-            instruction="You are a table analyst. Your task is to answer questions based on the table content. {answer_formatter}",
+            instruction="You are a table analyst. Your task is to answer questions based on the table content. {answer_formatter}"
+            + "\nOutput only the final answer without any explanations, extra information, or introductory text."
+            + "\nHere are some input-output examples. Read the examples carefully to figure out the mapping. The output of the last example is not given, and your job is to figure out what it is.",
             input_format="{context_type}: {context} \nQuestion: {question}",
             target_prefix="Final Answer: ",
             output_format="{answers}",
             postprocessors=[
+                "processors.take_first_non_empty_line",
                 "processors.lower_case",
                 "processors.remove_punctuations",
                 "processors.remove_articles",

prepare/cards/turl_col_type.py

@@ -44,7 +44,9 @@
         InputOutputTemplate(
             instruction="""
                     This is a column type annotation task. The goal of this task is to choose the correct types for one selected column of the given input table from the given candidate types. The Wikipedia page, section and table caption (if any) provide important information for choosing the correct column types.
-                    Candidate Types: {vocab} \nOutput only the correct column types for the mentioned from the candidate types.
+                    Candidate Types: {vocab}
+                    \nOutput only the correct column types from the candidate list for the mentioned columns. Do not include any explanations, extra information, or introductory text—only the final answer.
+                    \nHere are some input-output examples. Read the examples carefully to figure out the mapping. The output of the last example is not given, and your job is to figure out what it is.
                 """.strip(),
             input_format="\nColumn name: {colname}"
             "\nPage Title: {page_title} "

prepare/cards/wikitq.py

@@ -25,10 +25,12 @@
     task="tasks.qa.extractive[metrics=[metrics.f1_strings, metrics.unsorted_list_exact_match]]",
     templates=[
         MultiReferenceTemplate(
-            instruction="Answer the question based on the provided table. You should only output the final answer. Do not add any explanation or other information.",
-            input_format="\nQuestion: {question}\nTable: {context}\nAnswer: ",
+            instruction="Answer the question based on the provided table. Extract and output only the final answer—the exact phrase or data from the table that directly answers the question. Do not include any alterations, explanations, or introductory text"
+            + "\nHere are some input-output examples. Read the examples carefully to figure out the mapping. The output of the last example is not given, and your job is to figure out what it is.",
+            input_format="\nQuestion: {question}" "\nTable: {context}" "\nAnswer: ",
             references_field="answers",
             postprocessors=[
+                "processors.take_first_non_empty_line",
                 "processors.to_list_by_comma_space",
                 "processors.str_to_float_format",
             ],

prepare/tasks/generation.py

@@ -29,9 +29,9 @@
         reference_fields={"output": str},
         prediction_type=str,
         metrics=[
-            "metrics.bleu",
             "metrics.rouge",
             "metrics.bert_score.bert_base_uncased",
+            "metrics.bleu",
             "metrics.meteor",
         ],
         augmentable_inputs=["input_a", "input_b"],

prepare/templates/generation/generation.py

@@ -33,7 +33,8 @@
 
 add_to_catalog(
     InputOutputTemplate(
-        instruction="Given the following {type_of_input_a} and {type_of_input_b}, generate the corresponding {type_of_output}.",
+        instruction="Given the following {type_of_input_a} and {type_of_input_b}, generate the corresponding {type_of_output}."
+        + "\nHere are some input-output examples. Read the examples carefully to figure out the mapping. The output of the last example is not given, and your job is to figure out what it is.",
         input_format="{type_of_input_a}: \n{input_a} \n{type_of_input_b}: \n{input_b} \n{type_of_output}:",
         output_format="{output}",
         postprocessors=[

prepare/templates/qa/with_context.py

@@ -92,7 +92,8 @@
 
 add_to_catalog(
     MultiReferenceTemplate(
-        instruction="Using the information from the {context_type} given below, summarize a paragraph-long response to the following user query.",
+        instruction="Using the information from the {context_type} given below, summarize a paragraph-long response to the following user query."
+        + "\nHere are some input-output examples. Read the examples carefully to figure out the mapping. The output of the last example is not given, and your job is to figure out what it is.",
         input_format="{context_type}:\n{context}\nQuery:\n{question}",
         output_format="{answers}",
         target_prefix="Answer:\n",

src/unitxt/catalog/cards/fin_qa.json

@@ -59,10 +59,12 @@
     "templates": [
         {
             "__type__": "input_output_template",
-            "instruction": "Presented with a financial report consisting of textual contents and a structured table, given a question, generate the reasoning program in the domain specific language (DSL) that will be executed to get the answer. \nThe DSL consists of mathematical operations and table operations as executable programs. The program consists of a sequence of operations. Each operation takes a list of arguments. \nThere are 6 mathematical operations: add, subtract, multiply, divide, greater, exp, and 4 table aggregation operations table-max, table-min, table-sum, table-average, that apply aggregation operations on table rows. The mathematical operations take arguments of either numbers from the given reports, or a numerical result from a previous step.\nThe table operations take arguments of table row names. We use the special token #n to denote the result from the nth step. \nFor example, in the example \"divide(9413, 20.01), divide(8249, 9.48), subtract(#0, #1)\", the program consists of 3 steps; The first and the second division steps take arguments from the table and the text, respectively, then the third step subtracts the results from the two previous steps.\n                Definitions of all operations:\n                [[\"Name\", \"Arguments\", \"Output\", \"Description\"],\n                [\"add\", \"number1, number2\", \"number\", \"add two numbers: number1 + number2\"],\n                [\"subtract\", \"number1, number2\", \"number\", \"subtract two numbers: number1 - number2\"],\n                [\"multiply\", \"number1, number2\", \"number\", \"multiply two numbers: number1 * number2\"],\n                [\"divide\", \"number1, number2\", \"number\", \"multiply two numbers: number1 / number2\"],\n                [\"exp\", \"number1, number2\", \"number\", \"exponential: number1 ^ number2\"],\n                [\"greater\", \"number1, number2\", \"bool\", \"comparison: number1 > number2\"],\n                [\"table-sum\", \"table header\", \"number\", \"the summation of one table row\"],\n                [\"table-average\", \"table header\", \"number\", \"the average of one table row\"],\n                [\"table-max\", \"table header\", \"number\", \"the maximum number of one table row\"],\n                [\"table-min\", \"table header\", \"number\", \"the minimum number of one table row\"]]\n                Answer with only the program, without any additional explanation.\n                ",
+            "instruction": "Presented with a financial report consisting of textual contents and a structured table, given a question, generate the reasoning program in the domain specific language (DSL) that will be executed to get the answer. \nThe DSL consists of mathematical operations and table operations as executable programs. The program consists of a sequence of operations. Each operation takes a list of arguments. \nThere are 6 mathematical operations: add, subtract, multiply, divide, greater, exp, and 4 table aggregation operations table-max, table-min, table-sum, table-average, that apply aggregation operations on table rows. The mathematical operations take arguments of either numbers from the given reports, or a numerical result from a previous step.\nThe table operations take arguments of table row names. We use the special token #n to denote the result from the nth step. \nFor example, in the example \"divide(9413, 20.01), divide(8249, 9.48), subtract(#0, #1)\", the program consists of 3 steps; The first and the second division steps take arguments from the table and the text, respectively, then the third step subtracts the results from the two previous steps.\n                Definitions of all operations:\n                [[\"Name\", \"Arguments\", \"Output\", \"Description\"],\n                [\"add\", \"number1, number2\", \"number\", \"add two numbers: number1 + number2\"],\n                [\"subtract\", \"number1, number2\", \"number\", \"subtract two numbers: number1 - number2\"],\n                [\"multiply\", \"number1, number2\", \"number\", \"multiply two numbers: number1 * number2\"],\n                [\"divide\", \"number1, number2\", \"number\", \"multiply two numbers: number1 / number2\"],\n                [\"exp\", \"number1, number2\", \"number\", \"exponential: number1 ^ number2\"],\n                [\"greater\", \"number1, number2\", \"bool\", \"comparison: number1 > number2\"],\n                [\"table-sum\", \"table header\", \"number\", \"the summation of one table row\"],\n                [\"table-average\", \"table header\", \"number\", \"the average of one table row\"],\n                [\"table-max\", \"table header\", \"number\", \"the maximum number of one table row\"],\n                [\"table-min\", \"table header\", \"number\", \"the minimum number of one table row\"]]\n                \nAnswer with only the program, without any additional explanation or introductory text.\n                \nHere are some input-output examples. Read the examples carefully to figure out the mapping. The output of the last example is not given, and your job is to figure out what it is.\n                ",
             "input_format": "Pre-table text: {pre_text}\n                Table: {table}\n                Post-table text: {post_text}\n                Question: {question}\n                Program:\n                    ",
             "output_format": "{program_re}",
-            "postprocessors": []
+            "postprocessors": [
+                "processors.take_first_non_empty_line"
+            ]
         }
     ],
     "__description__": "FINQA is an expert-annotated QA dataset that aims to tackle numerical reasoning over real-world financial data.",