From bce168cd50e60e27f3461a09c78b0f8baaffb9ea Mon Sep 17 00:00:00 2001 From: Rui Xing Date: Wed, 15 May 2024 14:52:30 +0400 Subject: [PATCH] Add Multi-Language Support (Chinese) (#21) * Add multi-language support (Chinese) * Update minimal_test.py --- factcheck/utils/logger.py | 2 +- factcheck/utils/prompt/__init__.py | 2 + factcheck/utils/prompt/chatgpt_prompt_zh.py | 114 ++++++++++++++++++ script/minimal_test.py | 9 +- ...minimal_test.json => minimal_test_en.json} | 7 ++ script/minimal_test_zh.json | 44 +++++++ 6 files changed, 174 insertions(+), 4 deletions(-) create mode 100644 factcheck/utils/prompt/chatgpt_prompt_zh.py rename script/{minimal_test.json => minimal_test_en.json} (94%) create mode 100644 script/minimal_test_zh.json diff --git a/factcheck/utils/logger.py b/factcheck/utils/logger.py index 8ba437d..a2ffaaa 100644 --- a/factcheck/utils/logger.py +++ b/factcheck/utils/logger.py @@ -20,7 +20,7 @@ def __init__(self, name: str, loglevel=logging.INFO): # If the directory does not exist, create it os.makedirs("./log") env = os.environ.get("env", "dev") - fh = TimedRotatingFileHandler(filename="./log/factcheck_{}.log".format(env), when="D") + fh = TimedRotatingFileHandler(filename="./log/factcheck_{}.log".format(env), when="D", encoding="utf-8") fh.setLevel(loglevel) if not self.logger.handlers: # Create another handler for output to the console diff --git a/factcheck/utils/prompt/__init__.py b/factcheck/utils/prompt/__init__.py index 4f10919..c1d63ad 100644 --- a/factcheck/utils/prompt/__init__.py +++ b/factcheck/utils/prompt/__init__.py @@ -1,9 +1,11 @@ from .chatgpt_prompt import ChatGPTPrompt +from .chatgpt_prompt_zh import ChatGPTPromptZH from .claude_prompt import ClaudePrompt from .customized_prompt import CustomizedPrompt prompt_map = { "chatgpt_prompt": ChatGPTPrompt, + "chatgpt_prompt_zh": ChatGPTPromptZH, "claude_prompt": ClaudePrompt, } diff --git a/factcheck/utils/prompt/chatgpt_prompt_zh.py b/factcheck/utils/prompt/chatgpt_prompt_zh.py new file mode 100644 index 0000000..e00783d --- /dev/null +++ b/factcheck/utils/prompt/chatgpt_prompt_zh.py @@ -0,0 +1,114 @@ +from .base import BasePrompt +from dataclasses import dataclass + +decompose_prompt_zh = """你的任务是将文本分解为原子命题(atomic claim)。 +答案应该是一个带有单个key为"claims"的JSON，其value是一个字符串列表，其中每个字符串应该是一个独立于上下文的命题，代表一个事实。 +注意: +1. 每一个原子命题应简明扼要（少于15个字），并且是独立完整的。 +2. 避免使用"他"、"她"、"它"、"这个"、"此公司"、"此人"等模糊的称呼，请使用全名。 +3. 为文本每个句子至少生成一个命题。 + +例如， +玛丽是一个五岁的女孩，她喜欢弹钢琴，她不喜欢饼干。 +输出: +{{"claim": ["玛丽是一个五岁的女孩。", "玛丽是一个五岁的女孩。", "玛丽不喜欢饼干。"]}} + +文本: {doc} +输出: +""" + +checkworthy_prompt_zh = """你的任务是评估每个陈述，以确定它提供的信息的真实性是否可以被人类客观验证，而不考虑当前陈述的准确性。请遵循以下规则: +1. 观点与事实: 区分主观的、无法证实的观点和表达事实信息的陈述，即使表述较为宽泛和笼统。请关注是否有可以验证其事实性的命题。 +2. 清晰和专一性: 陈述必须有清晰、具体且可验证的参考(例如，"他是教授"这个陈述在不知道"他"是谁的情况下是不可验证的)。 +3. 存在事实信息: 如果一个陈述包含了可以根据证据或可靠来源进行验证的事实元素，那么就认为它是可验证的，即使整个陈述可能较为笼统或不正确。 +你的输出是JSON格式，key是每个陈述，value是"Yes"或"No"并简述原因。 + +例如，给定这些语句: +1. 加里·史密斯是一位杰出的经济学教授。 +2. 他是麻省理工学院的教授。 +3. 奥巴马是英国总统。 +4. 上海大学是中国江苏的大学。 + +预期输出为: +{{ + "加里·史密斯是一位杰出的经济学教授": "Yes(该陈述包含有关Gary Smith的专业头衔和领域的可验证的事实信息。)", + "他是麻省理工学院的教授。": "No(该陈述没有明确提到‘他’是谁，因此无法核实。)", + "奥巴马是英国总统。": "Yes(该陈述包含了关于一个国家政治领导的可核实的信息。)", + "上海大学是中国江苏的大学。": "Yes(该陈述包含了关于一个大学地理位置的可验证的信息。)" +｝｝ + +对于这些陈述: +{texts} + +输出应是: +""" + +qgen_prompt_zh = """给定一个命题，你的任务是创建最少数量的问题，以验证命题的正确性。输出为JSON格式，其有单个key"Questions"，value是问题列表。例如: + +命题:你的鼻子呼吸在两个鼻孔之间来回切换。当你睡觉时，大约每45分钟换一次。这是为了防止粘液积聚。这个现象叫做鼻腔循环。 +输出:{{"Questions": ["你的鼻子呼吸会在鼻孔之间交换吗?","你的鼻子呼吸多久交换一次?","你的鼻孔呼吸为什么会交换?","什么是鼻循环?"]}} + +命题:斯坦福监狱实验是在斯坦福大学心理学大楼恩西纳大厅的地下室进行的。 +输出: +{{"Questions": ["斯坦福监狱实验是在哪里进行的?"]}} + +命题:Havel-Hakimi算法是一种将图的邻接矩阵转换为其邻接表的算法。它以瓦茨拉夫·哈维尔和萨米·哈基米的名字命名。 +输出: +{{"Questions": ["Havel-Hakimi算法是做什么的?","Havel-Hakimi算法是以谁命名的?"]}} + +命题:社会工作是一种基于人文主义哲学传统的职业。这是一门起源于19世纪的知识学科。 +输出: +{{"Questions": ["社会工作是基于什么哲学传统?","社会工作起源于哪一年?"]}} + +命题:{claim} +输出: +""" + +verify_prompt_zh = """你的任务是使用陈述附带的证据(evidence)来评估其陈述的准确性。仔细审查这些证据，注意它可能在细节上有所不同，有时会呈现相互矛盾的信息。你的判断应该根据这些证据，并考虑其相关性和可靠性。 + +请记住，证据中缺乏细节并不一定表明陈述是不准确的。在评估陈述的真实性时，要区分错误和证据支持陈述的地方。 + +请以JSON格式构建您的回答，包含以下四个key: +- "reasoning": 解释你的判断的推理过程。 +- "error": 如果文本事实正确，该项为"none";否则，请指出命题中任何具体的不准确之处。 +- "correction": 如果文本是事实，该项为"none";否则，对任何发现的不准确之处进行纠正，并在纠正时候使用证据来支持。 +- "factuality": 如果给定的文本是事实，该项为"true"，表明该陈述是事实。否则为"false"，说明根据证据，该陈述并非事实。 + +例如: +输入: +[文本]: MBZUAI位于阿拉伯联合酋长国的阿布扎比。 +[证据]: MBZUAI位于哪里?\n答案:马斯达尔城-阿布扎比-阿拉伯联合酋长国 + +输出: +{{ + "reasoning": "证据证实MBZUAI位于阿拉伯联合酋长国阿布扎比的马斯达尔城，因此该陈述在事实上是正确的", + "error": none, + "correction": none, + "factuality": true +｝｝ + + +输入: +[文本]: 铜会与硫酸亚铁(FeSO4)发生反应。 +[证据]: 铜是较不活泼的金属。它具有正的标准还原电位。具有高标准还原电位的金属不能取代具有低标准还原电位的金属。因此，铜不能从硫酸亚铁溶液中取代铁。所以不会发生反应。 + +输出: +{{ + "reasoning": "所提供的证据证实，铜不能从硫酸亚铁溶液中取代铁，也不会发生变化。", + "error": "铜不能与文中所述的硫酸亚铁反应。", + "correction": "铜不能与硫酸亚铁反应，因为它不能取代硫酸亚铁溶液中的铁。", + "factuality": false +｝｝ + +输入 +[文本]:{claim} +[证据]:{evidence} + +输出: +""" + +class ChatGPTPromptZH: + decompose_prompt = decompose_prompt_zh + checkworthy_prompt = checkworthy_prompt_zh + qgen_prompt = qgen_prompt_zh + verify_prompt = verify_prompt_zh diff --git a/script/minimal_test.py b/script/minimal_test.py index 9c2833d..f1c9007 100644 --- a/script/minimal_test.py +++ b/script/minimal_test.py @@ -12,9 +12,12 @@ reset = "\033[0m" -def minimal_test(): +def minimal_test(lang="en"): # Initialize the FactCheck class - factcheck = FactCheck() + prompt = "chatgpt_prompt" + if lang == "zh": + prompt = "chatgpt_prompt_zh" + factcheck = FactCheck(prompt=prompt) def atom_test(instance): response = instance["response"] @@ -27,7 +30,7 @@ def atom_test(instance): except: # noqa E722 return False - with open("minimal_test.json") as f: + with open(f"minimal_test_{lang}.json", encoding="utf-8") as f: test_data = json.load(f) num_tests = len(test_data) diff --git a/script/minimal_test.json b/script/minimal_test_en.json similarity index 94% rename from script/minimal_test.json rename to script/minimal_test_en.json index 3fec4db..f070312 100644 --- a/script/minimal_test.json +++ b/script/minimal_test_en.json @@ -1,6 +1,7 @@ [ { "id": 1, + "language": "en", "response": "I am a famous scientist.", "attributes": { "factuality": "Nothing to check." @@ -8,6 +9,7 @@ }, { "id": 2, + "language": "en", "response": "Steve Jobs is the founder of Apple.", "attributes": { "factuality": true @@ -15,6 +17,7 @@ }, { "id": 3, + "language": "en", "response": "Elon Musk bought Twitter in 2023 and renamed it to X.", "attributes": { "factuality": false @@ -22,6 +25,7 @@ }, { "id": 4, + "language": "en", "response": "Blockchain technology is only used for cryptocurrencies.", "attributes": { "factuality": false @@ -29,6 +33,7 @@ }, { "id": 5, + "language": "en", "response": "Facial recognition technology is infallible and cannot be tricked.", "attributes": { "factuality": false @@ -36,6 +41,7 @@ }, { "id": 6, + "language": "en", "response": "Shanghai Jiao Tong University is one of the top public universities in Guangdong, China", "attributes": { "factuality": false @@ -43,6 +49,7 @@ }, { "id": 7, + "language": "en", "response": "William Yarnel Slack (August 1, 1816 - March 21, 1862) was an American lawyer, politician, and military officer who fought for the Confederate States of America during the American Civil War. Born in Kentucky, Slack moved to Missouri as a child and later entered the legal profession. After serving in the Missouri General Assembly from 1842 to 1843, he fought as a captain in the United States Army for fourteen months during the Mexican–American War, beginning in 1846. He saw action at the Battle of Embudo Pass and the Siege of Pueblo de Taos. Returning to a legal career, Slack became influential in his local area. After the outbreak of the American Civil War in April 1861, Slack, who held pro-slavery views, supported the Confederate cause. When the Missouri State Guard (MSG) was formed the next month to oppose the Union Army, he was appointed as a brigadier general in the MSG's 4th Division. After participating in the Battle of Carthage in July, he fought in the Battle of Wilson's Creek on August 10. After a surprise Union attack, Slack's deployment of his division gave time for further Confederate States Army and MSG troops to deploy. Suffering a bad hip wound at Wilson's Creek, he was unable to rejoin his command until October. Along with other Missouri State Guard officers, Slack transferred to the Confederate States Army in late 1861 where he commanded a brigade with the rank of colonel. On March 7, 1862, during the Battle of Pea Ridge, Slack suffered another wound that was close to the injury he had received at Wilson's Creek. Infection set in, and he died on March 21. He was posthumously promoted to brigadier general in the Confederate army on April 17; the Confederate States Senate might not have known that he was dead at the time of the promotion. (1786)", "attributes": { "factuality": false diff --git a/script/minimal_test_zh.json b/script/minimal_test_zh.json new file mode 100644 index 0000000..92e2b1a --- /dev/null +++ b/script/minimal_test_zh.json @@ -0,0 +1,44 @@ +[ + { + "id": 1, + "response": "我是一个NLP科学家。", + "attributes": { + "factuality": "Nothing to check." + } + }, + { + "id": 2, + "response": "马云创立了阿里巴巴公司。", + "attributes": { + "factuality": true + } + }, + { + "id": 3, + "response": "埃隆·马斯克(Elon Musk)在2023年收购了Twitter，并将其更名为X。", + "attributes": { + "factuality": false + } + }, + { + "id": 4, + "response": "马化腾创办了京东和腾讯。", + "attributes": { + "factuality": false + } + }, + { + "id": 5, + "response": "人脸识别系统研究始于20世纪20年代。", + "attributes": { + "factuality": false + } + }, + { + "id": 6, + "response": "天津大学是中国江苏的大学。", + "attributes": { + "factuality": false + } + } +]