添加 evaluation.py
This commit is contained in:
parent
ba9c62ca80
commit
061cd74fca
|
|
@ -0,0 +1,393 @@
|
|||
import random
|
||||
import openai
|
||||
import time
|
||||
import json
|
||||
import argparse
|
||||
import tiktoken
|
||||
from openai import OpenAI
|
||||
from openai import OpenAIError
|
||||
|
||||
|
||||
client = OpenAI(
|
||||
api_key="sk-5f06261529bb44df86d9b2fdbae1a6b5", # 在这里将 MOONSHOT_API_KEY 替换为你从 Kimi 开放平台申请的 API Key
|
||||
base_url="https://api.deepseek.com/v1",
|
||||
)
|
||||
|
||||
def get_qa_response(model, question, answer):
|
||||
message = [
|
||||
{"role": "system", "content":"You are a huallucination detector. You MUST determine if the provided answer contains hallucination or not for the question based on the world knowledge. The answer you provided MUST be \"Yes\" or \"No\""},
|
||||
{"role": "user", "content":
|
||||
"\n\n#Question#: " + question +
|
||||
"\n#Answer#: " + answer +
|
||||
"\n#Your Judgement#: "}
|
||||
]
|
||||
prompt = "\n#Question#: " + question + "\n#Answer#: " + answer + "\n#Your Judgement#:"
|
||||
while True:
|
||||
try:
|
||||
if model == "gpt-3.5-turbo":
|
||||
res = openai.ChatCompletion.create(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=message,
|
||||
temperature=0.0,
|
||||
)
|
||||
response = res['choices'][0]['message']['content']
|
||||
else:
|
||||
res = client.chat.completions.create(
|
||||
model="deepseek-chat",
|
||||
messages=message,
|
||||
temperature=1,
|
||||
max_tokens=256,
|
||||
top_p=1
|
||||
)
|
||||
response = res.choices[0].message.content
|
||||
break
|
||||
except OpenAIError:
|
||||
print('openai.error.RateLimitError\nRetrying...')
|
||||
time.sleep(60)
|
||||
except openai.error.ServiceUnavailableError:
|
||||
print('openai.error.ServiceUnavailableError\nRetrying...')
|
||||
time.sleep(20)
|
||||
except openai.error.Timeout:
|
||||
print('openai.error.Timeout\nRetrying...')
|
||||
time.sleep(20)
|
||||
except openai.error.APIError:
|
||||
print('openai.error.APIError\nRetrying...')
|
||||
time.sleep(20)
|
||||
except openai.error.APIConnectionError:
|
||||
print('openai.error.APIConnectionError\nRetrying...')
|
||||
time.sleep(20)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def get_dialogue_response(model, dialog, response, instruction):
|
||||
message = [
|
||||
{"role": "system", "content": "You are a response judge. You MUST determine if the provided response contains non-factual or hallucinated information. The answer you give MUST be \"Yes\" or \"No\""},
|
||||
{"role": "user", "content": instruction +
|
||||
"\n\n#Dialogue History#: " + dialog +
|
||||
"\n#Response#: " + response +
|
||||
"\n#Your Judgement#: "}
|
||||
]
|
||||
prompt = instruction + "\n\n#Dialogue History#: " + dialog + "\n#Response#: " + response + "\n#Your Judgement#:"
|
||||
while True:
|
||||
try:
|
||||
if model == "gpt-3.5-turbo":
|
||||
res = openai.ChatCompletion.create(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=message,
|
||||
temperature=0.0,
|
||||
)
|
||||
response = res['choices'][0]['message']['content']
|
||||
else:
|
||||
res = openai.Completion.create(
|
||||
model=model,
|
||||
prompt=prompt,
|
||||
temperature=0.0
|
||||
)
|
||||
response = res["choices"][0]['text'].strip()
|
||||
break
|
||||
except openai.error.RateLimitError:
|
||||
print('openai.error.RateLimitError\nRetrying...')
|
||||
time.sleep(60)
|
||||
except openai.error.ServiceUnavailableError:
|
||||
print('openai.error.ServiceUnavailableError\nRetrying...')
|
||||
time.sleep(20)
|
||||
except openai.error.Timeout:
|
||||
print('openai.error.Timeout\nRetrying...')
|
||||
time.sleep(20)
|
||||
except openai.error.APIError:
|
||||
print('openai.error.APIError\nRetrying...')
|
||||
time.sleep(20)
|
||||
except openai.error.APIConnectionError:
|
||||
print('openai.error.APIConnectionError\nRetrying...')
|
||||
time.sleep(20)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def num_tokens_from_message(message, model="davinci"):
|
||||
encoding = tiktoken.encoding_for_model(model)
|
||||
num_tokens = len(encoding.encode(message))
|
||||
return num_tokens
|
||||
|
||||
|
||||
def truncate_message(prompt1, prompt2, model="davinci"):
|
||||
if num_tokens_from_message(prompt1 + prompt2, model) > 2033:
|
||||
truncation_length = 2033 - num_tokens_from_message(prompt2)
|
||||
while num_tokens_from_message(prompt1) > truncation_length:
|
||||
prompt1 = " ".join(prompt1.split()[:-1])
|
||||
prompt = prompt1 + prompt2
|
||||
return prompt
|
||||
|
||||
|
||||
def get_summarization_response(model, document, summary, instruction):
|
||||
message = [
|
||||
{"role": "system", "content": "You are a summary judge. You MUST determine if the provided summary contains non-factual or hallucinated information. The answer you give MUST be \"Yes\" or \"No\""},
|
||||
{"role": "user", "content": instruction +
|
||||
"\n\n#Document#: " + document +
|
||||
"\n#Summary#: " + summary +
|
||||
"\n#Your Judgement#: "}
|
||||
]
|
||||
prompt1 = instruction + "\n\n#Document#: " + document
|
||||
prompt2 = "\n#Summary#: " + summary + "\n#Your Judgement#:"
|
||||
if model == "davinci":
|
||||
prompt = truncate_message(prompt1, prompt2)
|
||||
else:
|
||||
prompt = prompt1 + prompt2
|
||||
while True:
|
||||
try:
|
||||
if model == "gpt-3.5-turbo":
|
||||
res = openai.ChatCompletion.create(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=message,
|
||||
temperature=0.0,
|
||||
)
|
||||
response = res['choices'][0]['message']['content']
|
||||
else:
|
||||
res = openai.Completion.create(
|
||||
model=model,
|
||||
prompt=prompt,
|
||||
temperature=0.0
|
||||
)
|
||||
response = res["choices"][0]['text'].strip()
|
||||
break
|
||||
except openai.error.RateLimitError:
|
||||
print('openai.error.RateLimitError\nRetrying...')
|
||||
time.sleep(60)
|
||||
except openai.error.ServiceUnavailableError:
|
||||
print('openai.error.ServiceUnavailableError\nRetrying...')
|
||||
time.sleep(20)
|
||||
except openai.error.Timeout:
|
||||
print('openai.error.Timeout\nRetrying...')
|
||||
time.sleep(20)
|
||||
except openai.error.APIError:
|
||||
print('openai.error.APIError\nRetrying...')
|
||||
time.sleep(20)
|
||||
except openai.error.APIConnectionError:
|
||||
print('openai.error.APIConnectionError\nRetrying...')
|
||||
time.sleep(20)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def evaluation_qa_dataset(model, file, output_path):
|
||||
result=[]
|
||||
TP = 0
|
||||
FP=0
|
||||
FN=0
|
||||
# test_file=json.loads(file)
|
||||
with open(file, 'r', encoding="utf-8") as f:
|
||||
# print(f"File content: {file}")
|
||||
test_file=json.load(f)
|
||||
data = []
|
||||
for i in range(len(test_file)):
|
||||
data.append(test_file[i])
|
||||
for i in range(len(data)):
|
||||
question= data[i]["Question"]
|
||||
answer=data[i]["Answer"]
|
||||
ground_truth = data[i]["Hallucination"]
|
||||
|
||||
output_samples = get_qa_response(model, question, answer)
|
||||
print('sample {} success......'.format(i))
|
||||
if ("Yes" in output_samples and "YES" in ground_truth):
|
||||
TP=TP+1
|
||||
elif ("Yes" in output_samples and "NO" in ground_truth):
|
||||
FP=FP+1
|
||||
else:
|
||||
FN=FN+1
|
||||
result.append({"Question":question,"Answer":answer,"Hallucination":ground_truth, "res":output_samples})
|
||||
Precision=TP/(TP+FP)
|
||||
Recall=TP/(TP+FN)
|
||||
F1score=2*(Precision*Recall)/(Precision+Recall)
|
||||
print(' F1score: {}'.format(F1score))
|
||||
dump_jsonl(result, output_path, append=True)
|
||||
# correct = 0
|
||||
# incorrect = 0
|
||||
# for i in range(len(data)):
|
||||
# knowledge = data[i]["knowledge"]
|
||||
# question = data[i]["question"]
|
||||
# hallucinated_answer = data[i]["hallucinated_answer"]
|
||||
# right_answer = data[i]["right_answer"]
|
||||
|
||||
# if random.random() > 0.5:
|
||||
# answer = hallucinated_answer
|
||||
# ground_truth = "Yes"
|
||||
# else:
|
||||
# answer = right_answer
|
||||
# ground_truth = "No"
|
||||
|
||||
# ans = get_qa_response(model, question, answer, instruction)
|
||||
# ans = ans.replace(".", "")
|
||||
|
||||
# if ("Yes" in ans and "No" in ans) or ("Yes" not in ans and "No" not in ans):
|
||||
# gen = {"knowledge": knowledge, "question": question, "answer": answer, "ground_truth": ground_truth, "judgement": "failed!"}
|
||||
# dump_jsonl(gen, output_path, append=True)
|
||||
# incorrect += 1
|
||||
# print('sample {} fails......'.format(i))
|
||||
# continue
|
||||
# elif "Yes" in ans:
|
||||
# if ans != "Yes":
|
||||
# ans = "Yes"
|
||||
# gen = {"knowledge": knowledge, "question": question, "answer": answer, "ground_truth": ground_truth, "judgement": ans}
|
||||
# elif "No" in ans:
|
||||
# if ans != "No":
|
||||
# ans = "No"
|
||||
# gen = {"knowledge": knowledge, "question": question, "answer": answer, "ground_truth": ground_truth, "judgement": ans}
|
||||
# else:
|
||||
# gen = None
|
||||
# incorrect += 1
|
||||
|
||||
# assert(gen is not None)
|
||||
|
||||
# if ground_truth == ans:
|
||||
# correct += 1
|
||||
# else:
|
||||
# incorrect += 1
|
||||
|
||||
# print('sample {} success......'.format(i))
|
||||
|
||||
|
||||
# print('{} correct samples, {} incorrect samples, Accuracy: {}'.format(correct, incorrect, correct/len(data)))
|
||||
|
||||
|
||||
def evaluation_dialogue_dataset(model, file, instruction, output_path):
|
||||
with open(file, 'r', encoding="utf-8") as f:
|
||||
data = []
|
||||
for line in f:
|
||||
data.append(json.loads(line))
|
||||
|
||||
correct = 0
|
||||
incorrect = 0
|
||||
for i in range(len(data)):
|
||||
knowledge = data[i]["knowledge"]
|
||||
dialog = data[i]["dialogue_history"]
|
||||
hallucinated_response = data[i]["hallucinated_response"]
|
||||
right_response = data[i]["right_response"]
|
||||
|
||||
if random.random() > 0.5:
|
||||
response = hallucinated_response
|
||||
ground_truth = "Yes"
|
||||
else:
|
||||
response = right_response
|
||||
ground_truth = "No"
|
||||
|
||||
ans = get_dialogue_response(model, dialog, response, instruction)
|
||||
ans = ans.replace(".", "")
|
||||
|
||||
if ("Yes" in ans and "No" in ans) or ("Yes" not in ans and "No" not in ans):
|
||||
gen = {"knowledge": knowledge, "dialogue_history": dialog, "response": response, "ground_truth": ground_truth, "judgement": "failed!"}
|
||||
dump_jsonl(gen, output_path, append=True)
|
||||
incorrect += 1
|
||||
print('sample {} fails......'.format(i))
|
||||
continue
|
||||
elif "Yes" in ans:
|
||||
if ans != "Yes":
|
||||
ans = "Yes"
|
||||
gen = {"knowledge": knowledge, "dialogue_history": dialog, "response": response, "ground_truth": ground_truth, "judgement": ans}
|
||||
elif "No" in ans:
|
||||
if ans != "No":
|
||||
ans = "No"
|
||||
gen = {"knowledge": knowledge, "dialogue_history": dialog, "response": response, "ground_truth": ground_truth, "judgement": ans}
|
||||
else:
|
||||
gen = None
|
||||
assert (gen is not None)
|
||||
|
||||
if ground_truth == ans:
|
||||
correct += 1
|
||||
else:
|
||||
incorrect += 1
|
||||
|
||||
print('sample {} success......'.format(i))
|
||||
dump_jsonl(gen, output_path, append=True)
|
||||
|
||||
print('{} correct samples, {} incorrect samples, Accuracy: {}'.format(correct, incorrect, correct / len(data)))
|
||||
|
||||
|
||||
def evaluation_summarization_dataset(model, file, instruction, output_path):
|
||||
with open(file, 'r', encoding="utf-8") as f:
|
||||
data = []
|
||||
for line in f:
|
||||
data.append(json.loads(line))
|
||||
|
||||
correct = 0
|
||||
incorrect = 0
|
||||
for i in range(len(data)):
|
||||
|
||||
document = data[i]["document"]
|
||||
hallucinated_summary = data[i]["hallucinated_summary"]
|
||||
right_summary = data[i]["right_summary"]
|
||||
|
||||
if random.random() > 0.5:
|
||||
summary = hallucinated_summary
|
||||
ground_truth = "Yes"
|
||||
else:
|
||||
summary = right_summary
|
||||
ground_truth = "No"
|
||||
|
||||
ans = get_summarization_response(model, document, summary, instruction)
|
||||
ans = ans.replace(".", "")
|
||||
|
||||
if ("Yes" in ans and "No" in ans) or ("Yes" not in ans and "No" not in ans):
|
||||
gen = {"document": document, "summary": summary, "ground_truth": ground_truth, "judgement": "failed!"}
|
||||
dump_jsonl(gen, output_path, append=True)
|
||||
incorrect += 1
|
||||
print('sample {} fails......'.format(i))
|
||||
continue
|
||||
elif "Yes" in ans:
|
||||
if ans != "Yes":
|
||||
ans = "Yes"
|
||||
gen = {"document": document, "summary": summary, "ground_truth": ground_truth, "judgement": ans}
|
||||
elif "No" in ans:
|
||||
if ans != "No":
|
||||
ans = "No"
|
||||
gen = {"document": document, "summary": summary, "ground_truth": ground_truth, "judgement": ans}
|
||||
else:
|
||||
gen = None
|
||||
assert (gen is not None)
|
||||
|
||||
if ground_truth == ans:
|
||||
correct += 1
|
||||
else:
|
||||
incorrect += 1
|
||||
|
||||
print('sample {} success......'.format(i))
|
||||
dump_jsonl(gen, output_path, append=True)
|
||||
|
||||
print('{} correct samples, {} incorrect samples, Accuracy: {}'.format(correct, incorrect, correct / len(data)))
|
||||
|
||||
|
||||
def dump_jsonl(data, output_path, append=False):
|
||||
"""
|
||||
Write list of objects to a JSON lines file.
|
||||
"""
|
||||
mode = 'a+' if append else 'w'
|
||||
with open(output_path, mode, encoding='utf-8') as f:
|
||||
json_record = json.dumps(data, ensure_ascii=False)
|
||||
f.write(json_record + '\n')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description="Hallucination Generation")
|
||||
|
||||
parser.add_argument("--task", default="qa", help="qa, dialogue, or summarization")
|
||||
parser.add_argument("--model", default="qwen", help="model name")
|
||||
args = parser.parse_args()
|
||||
|
||||
instruction_file = "{}/{}_evaluation_instruction.txt".format(args.task, args.task)
|
||||
f = open(instruction_file, 'r', encoding="utf-8")
|
||||
instruction = f.read()
|
||||
|
||||
model = args.model
|
||||
output_path = "{}/{}_{}_results.json".format(args.task, args.task, args.model)
|
||||
|
||||
# data = "../data/{}_data.json".format(args.task)
|
||||
data="/home/lee/code/HaluEval/evaluation/factuality_train.json"
|
||||
|
||||
if args.task == "qa":
|
||||
evaluation_qa_dataset(model, data, output_path)
|
||||
elif args.task == "dialogue":
|
||||
evaluation_dialogue_dataset(model, data, instruction, output_path)
|
||||
elif args.task == "summarization":
|
||||
evaluation_summarization_dataset(model, data, instruction, output_path)
|
||||
else:
|
||||
raise ValueError("The task must be qa, dialogue, or summarization!")
|
||||
Loading…
Reference in New Issue