From 24923b65f1b557cc4a51f4c78ffa0386b620899d Mon Sep 17 00:00:00 2001 From: weixin_43297441 Date: Mon, 3 Mar 2025 11:48:07 +0800 Subject: [PATCH] new 6 --- hal_det_llama.py | 45 ++++++++++++++++++++++++--------------------- hal_generate.py | 28 +++++++++------------------- hal_gt.py | 23 ++++++++++++----------- 3 files changed, 45 insertions(+), 51 deletions(-) diff --git a/hal_det_llama.py b/hal_det_llama.py index 7c70c56..dad8b96 100644 --- a/hal_det_llama.py +++ b/hal_det_llama.py @@ -52,11 +52,10 @@ def main(): parser = argparse.ArgumentParser() - parser.add_argument('--model_name', type=str, default='llama2_chat_7B') - parser.add_argument('--dataset_name', type=str, default='triviaqa') + parser.add_argument('--model', type=str, default='llama2_chat_7B') + parser.add_argument('--model_name', type=str, default='step-1-8k') + parser.add_argument('--dataset_name', type=str, default='tqa') parser.add_argument('--num_gene', type=int, default=1) - # parser.add_argument('--gene', type=int, default=0) - # parser.add_argument('--generate_gt', type=int, default=0) parser.add_argument('--use_rouge', type=bool, default= False) parser.add_argument('--weighted_svd', type=int, default=0) parser.add_argument('--feat_loc_svd', type=int, default=0) @@ -67,7 +66,7 @@ def main(): parser.add_argument("--model_dir", type=str, default=None, help='local directory with model data') args = parser.parse_args() - MODEL = HF_NAMES[args.model_name] if not args.model_dir else args.model_dir + MODEL = HF_NAMES[args.model] if not args.model_dir else args.model_dir @@ -180,12 +179,16 @@ def main(): question = dataset[int(used_indices[i])]['question'] else: question = dataset[i]['question'] + if args.most_likely: + info = 'most_likely_' + else: + info = 'batch_generations_' answers = np.load( - f'save_for_eval/{args.dataset_name}_hal_det/answers/most_likely_hal_det_{args.model_name}_{args.dataset_name}_answers_index_{i}.npy') + f'save_for_eval/{args.dataset_name}/{args.model_name}_hal_det/answers/' + info + f'hal_det_{args.model_name}_{args.dataset_name}_answers_index_{i}.npy') truths= np.load( - f'save_for_eval/{args.dataset_name}_hal_det/answers/most_likely_hal_det_{args.model_name}_{args.dataset_name}_answers_index_{i}.npy') + f'save_for_eval/{args.dataset_name}/{args.model_name}_hal_det/truths/' + info + f'hal_det_{args.model_name}_{args.dataset_name}_truths_index_{i}.npy') hallucinations= np.load( - f'save_for_eval/{args.dataset_name}_hal_det/hallucinations/most_likely_hal_det_{args.model_name}_{args.dataset_name}_hallucinations_index_{i}.npy') + f'save_for_eval/{args.dataset_name}/{args.model_name}_hal_det/hallucinations/' + info + f'hal_det_{args.model_name}_{args.dataset_name}_hallucinations_index_{i}.npy') for anw in answers: if args.dataset_name == 'tydiqa': @@ -205,7 +208,7 @@ def main(): hidden_states = hidden_states.detach().cpu().numpy()[:, -1, :] embed_generated.append(hidden_states) embed_generated = np.asarray(np.stack(embed_generated), dtype=np.float32) - np.save(f'save_for_eval/{args.dataset_name}_hal_det/most_likely_{args.model_name}_gene_embeddings_layer_wise.npy', embed_generated) + np.save(f'save_for_eval/{args.dataset_name}/{args.model_name}_hal_det/' + info + f'{args.model_name}_gene_embeddings_layer_wise.npy', embed_generated) for tru in truths: @@ -226,7 +229,7 @@ def main(): hidden_states = hidden_states.detach().cpu().numpy()[:, -1, :] embed_generated_t.append(hidden_states) embed_generated_t = np.asarray(np.stack(embed_generated_t), dtype=np.float32) - np.save(f'save_for_eval/{args.dataset_name}_hal_det/most_likely_{args.model_name}_gene_embeddings_t_layer_wise.npy', embed_generated_t) + np.save(f'save_for_eval/{args.dataset_name}/{args.model_name}_hal_det/' + info + f'{args.model_name}_gene_embeddings_t_layer_wise.npy', embed_generated_t) for hal in hallucinations: @@ -247,7 +250,7 @@ def main(): hidden_states = hidden_states.detach().cpu().numpy()[:, -1, :] embed_generated_h.append(hidden_states) embed_generated_h = np.asarray(np.stack(embed_generated_h), dtype=np.float32) - np.save(f'save_for_eval/{args.dataset_name}_hal_det/most_likely_{args.model_name}_gene_embeddings_h_layer_wise.npy', embed_generated_h) + np.save(f'save_for_eval/{args.dataset_name}/{args.model_name}_hal_det/' + info + f'{args.model_name}_gene_embeddings_h_layer_wise.npy', embed_generated_h) HEADS = [f"model.layers.{i}.self_attn.head_out" for i in range(model.config.num_hidden_layers)] MLPS = [f"model.layers.{i}.mlp" for i in range(model.config.num_hidden_layers)] @@ -261,7 +264,7 @@ def main(): answers = np.load( - f'save_for_eval/{args.dataset_name}_hal_det/answers/most_likely_hal_det_{args.model_name}_{args.dataset_name}_answers_index_{i}.npy') + f'save_for_eval/{args.dataset_name}/{args.model_name}_hal_det/answers/' + info + f'hal_det_{args.model_name}_{args.dataset_name}_answers_index_{i}.npy') for anw in answers: if args.dataset_name == 'tydiqa': prompt = tokenizer( @@ -288,18 +291,18 @@ def main(): embed_generated_loc2 = np.asarray(np.stack(embed_generated_loc2), dtype=np.float32) embed_generated_loc1 = np.asarray(np.stack(embed_generated_loc1), dtype=np.float32) - np.save(f'save_for_eval/{args.dataset_name}_hal_det/most_likely_{args.model_name}_gene_embeddings_head_wise.npy', embed_generated_loc1) - np.save(f'save_for_eval/{args.dataset_name}_hal_det/most_likely_{args.model_name}_embeddings_mlp_wise.npy', embed_generated_loc2) + np.save(f'save_for_eval/{args.dataset_name}/{args.model_name}_hal_det/' + info + f'{args.model_name}_gene_embeddings_head_wise.npy', embed_generated_loc1) + np.save(f'save_for_eval/{args.dataset_name}/{args.model_name}_hal_det/' + info + f'{args.model_name}_embeddings_mlp_wise.npy', embed_generated_loc2) # get the split and label (true or false) of the unlabeled data and the test data. if args.use_rouge: - gts = np.load(f'./ml_{args.dataset_name}_rouge_score.npy') - gts_bg = np.load(f'./bg_{args.dataset_name}_rouge_score.npy') + gts = np.load(f'./ml_{args.dataset_name}_{args.model_name}_rouge_score.npy') + gts_bg = np.load(f'./bg_{args.dataset_name}_{args.model_name}_rouge_score.npy') else: - gts = np.load(f'./ml_{args.dataset_name}_bleurt_score.npy') - gts_bg = np.load(f'./bg_{args.dataset_name}_bleurt_score.npy') + gts = np.load(f'./ml_{args.dataset_name}_{args.model_name}_bleurt_score.npy') + gts_bg = np.load(f'./bg_{args.dataset_name}_{args.model_name}_bleurt_score.npy') thres = args.thres_gt gt_label = np.asarray(gts> thres, dtype=np.int32) gt_label_bg = np.asarray(gts_bg > thres, dtype=np.int32) @@ -430,15 +433,15 @@ def main(): if args.most_likely: if feat_loc == 3: - embed_generated = np.load(f'save_for_eval/{args.dataset_name}_hal_det/most_likely_{args.model_name}_gene_embeddings_layer_wise.npy', + embed_generated = np.load(f'save_for_eval/{args.dataset_name}/{args.model_name}_hal_det/' + info + f'{args.model_name}_gene_embeddings_layer_wise.npy', allow_pickle=True) elif feat_loc == 2: embed_generated = np.load( - f'save_for_eval/{args.dataset_name}_hal_det/most_likely_{args.model_name}_gene_embeddings_mlp_wise.npy', + f'save_for_eval/{args.dataset_name}/{args.model_name}_hal_det/' + info + f'{args.model_name}_gene_embeddings_mlp_wise.npy', allow_pickle=True) else: embed_generated = np.load( - f'save_for_eval/{args.dataset_name}_hal_det/most_likely_{args.model_name}_gene_embeddings_head_wise.npy', + f'save_for_eval/{args.dataset_name}/{args.model_name}_hal_det/' + info + f'{args.model_name}_gene_embeddings_head_wise.npy', allow_pickle=True) feat_indices_wild = [] feat_indices_eval = [] diff --git a/hal_generate.py b/hal_generate.py index 4ee4809..2bdecba 100644 --- a/hal_generate.py +++ b/hal_generate.py @@ -62,7 +62,7 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('--model_name', type=str, default='step-1-8k') - parser.add_argument('--dataset_name', type=str, default='triviaqa') + parser.add_argument('--dataset_name', type=str, default='tqa') parser.add_argument('--num_gene', type=int, default=1) parser.add_argument('--use_api', type=bool, default=True) parser.add_argument('--most_likely', type=bool, default=True) @@ -226,14 +226,14 @@ def main(): response = client.chat.completions.create( model = args.model_name, messages = prompt, - # max_tokens=256, + max_tokens=256, top_p=1, temperature = 1, ) hallucination_response = client.chat.completions.create( model = args.model_name, messages = hallucination_prompt, - # max_tokens=256, + max_tokens=256, top_p=1, temperature = 1, ) @@ -241,7 +241,7 @@ def main(): truth_response=client.chat.completions.create( model = args.model_name, messages = truth_prompt, - # max_tokens=256, + max_tokens=256, top_p=1, temperature=1 ) @@ -252,7 +252,7 @@ def main(): response = client.chat.completions.create( model = args.model_name, messages = prompt, - # max_tokens=256, + max_tokens=256, n=1, # best_of=1, top_p=0.5, @@ -300,22 +300,12 @@ def main(): truths[gen_iter]=truth_decoded - # if args.dataset_name == 'tydiqa': - # pass - # elif args.dataset_name == 'triviaqa': - # pass if args.dataset_name == 'coqa': - truths[0]=dataset[i]['answer'] - if args.num_gene >1 and dataset[i]['additional_answers']>= args.num_gene-1: - left_truth=dataset[i]['additional_answers'][:args.num_gene-1] - truths=truths+left_truth + truths=[dataset[i]['answer']]+dataset[i]['additional_answers'] + truths=truths[:args.num_gene] elif args.dataset_name == 'tqa': - truths[0]=dataset[i]['Best Answer'] - if args.num_gene >1: - correct=dataset[i]['Correct Answers'].split(";") - if len(correct) >= args.num_gene-1: - left_truth=correct[:args.num_gene-1] - truths=truths+left_truth + truths=[dataset[i]['best_answer']]+dataset[i]['correct_answers'] + truths=truths[:args.num_gene] else: assert 'Not supported dataset!' diff --git a/hal_gt.py b/hal_gt.py index edf2a23..46a1fd7 100644 --- a/hal_gt.py +++ b/hal_gt.py @@ -49,14 +49,15 @@ def main(): parser = argparse.ArgumentParser() - parser.add_argument('--model_name', type=str, default='llama2_chat_7B') - parser.add_argument('--dataset_name', type=str, default='triviaqa') + parser.add_argument('--model', type=str, default='llama2_chat_7B') + parser.add_argument('--model_name', type=str, default='step-1-8k') + parser.add_argument('--dataset_name', type=str, default='tqa') parser.add_argument('--num_gene', type=int, default=1) parser.add_argument('--use_api', type=bool, default=False) - parser.add_argument('--most_likely', type=bool, default=False) + parser.add_argument('--most_likely', type=bool, default=True) parser.add_argument("--model_dir", type=str, default=None, help='local directory with model data') parser.add_argument("--instruction", type=str, default=None, help='local directory of instruction file.') - parser.add_argument('--use_rouge', type=bool, default=True) + parser.add_argument('--use_rouge', type=bool, default=False) parser.add_argument('--thres_gt', type=float, default=0.5) # parser.add_argument('--model_name', type=str, default='llama2_chat_7B') @@ -74,7 +75,7 @@ def main(): # parser.add_argument("--model_dir", type=str, default=None, help='local directory with model data') args = parser.parse_args() - MODEL = HF_NAMES[args.model_name] if not args.model_dir else args.model_dir + MODEL = HF_NAMES[args.model] if not args.model_dir else args.model_dir @@ -198,10 +199,10 @@ def main(): if args.most_likely: answers = np.load( - f'./save_for_eval/{args.dataset_name}_hal_det/answers/most_likely_hal_det_{args.model_name}_{args.dataset_name}_answers_index_{i}.npy') + f'./save_for_eval/{args.dataset_name}/{args.model_name}_hal_det/answers/most_likely_hal_det_{args.model_name}_{args.dataset_name}_answers_index_{i}.npy') else: answers = np.load( - f'./save_for_eval/{args.dataset_name}_hal_det/answers/batch_generations_hal_det_{args.model_name}_{args.dataset_name}_answers_index_{i}.npy') + f'./save_for_eval/{args.dataset_name}/{args.model_name}_hal_det/answers/batch_generations_hal_det_{args.model_name}_{args.dataset_name}_answers_index_{i}.npy') # get the gt. if args.use_rouge: @@ -240,14 +241,14 @@ def main(): # breakpoint() if args.most_likely: if args.use_rouge: - np.save(f'./ml_{args.dataset_name}_rouge_score.npy', gts) + np.save(f'./ml_{args.dataset_name}_{args.model_name}_rouge_score.npy', gts) else: - np.save(f'./ml_{args.dataset_name}_bleurt_score.npy', gts) + np.save(f'./ml_{args.dataset_name}_{args.model_name}_bleurt_score.npy', gts) else: if args.use_rouge: - np.save(f'./bg_{args.dataset_name}_rouge_score.npy', gts) + np.save(f'./bg_{args.dataset_name}_{args.model_name}_rouge_score.npy', gts) else: - np.save(f'./bg_{args.dataset_name}_bleurt_score.npy', gts) + np.save(f'./bg_{args.dataset_name}_{args.model_name}_bleurt_score.npy', gts)