Note: evaluation simply move model to single GPU. Therefor, to evaluate large model such as Llama-2-70B on single A100-80GB, please activate '--real_quant'.
(model, tokenizer, args, logger)
| 21 | |
| 22 | @torch.no_grad() |
| 23 | def evaluate(model, tokenizer, args, logger): |
| 24 | ''' |
| 25 | Note: evaluation simply move model to single GPU. |
| 26 | Therefor, to evaluate large model such as Llama-2-70B on single A100-80GB, |
| 27 | please activate '--real_quant'. |
| 28 | ''' |
| 29 | # import pdb;pdb.set_trace() |
| 30 | block_class_name = model.model.layers[0].__class__.__name__ |
| 31 | device_map = infer_auto_device_map(model, max_memory={i: args.max_memory for i in range(torch.cuda.device_count())}, no_split_module_classes=[block_class_name]) |
| 32 | model = dispatch_model(model, device_map=device_map) |
| 33 | results = {} |
| 34 | |
| 35 | if args.eval_ppl: |
| 36 | datasets = ["wikitext2", "c4"] |
| 37 | ppl_results = test_ppl(model, tokenizer, datasets, args.ppl_seqlen) |
| 38 | for dataset in ppl_results: |
| 39 | logger.info(f'{dataset} perplexity: {ppl_results[dataset]:.2f}') |
| 40 | |
| 41 | if args.eval_tasks != "": |
| 42 | import lm_eval |
| 43 | from lm_eval.models.huggingface import HFLM |
| 44 | from lm_eval.utils import make_table |
| 45 | task_list = args.eval_tasks.split(',') |
| 46 | model = HFLM(pretrained=model, batch_size=args.eval_batch_size) |
| 47 | task_manager = lm_eval.tasks.TaskManager() |
| 48 | results = lm_eval.simple_evaluate( |
| 49 | model=model, |
| 50 | tasks=task_list, |
| 51 | num_fewshot=0, |
| 52 | task_manager=task_manager, |
| 53 | ) |
| 54 | logger.info(make_table(results)) |
| 55 | total_acc = 0 |
| 56 | for task in task_list: |
| 57 | total_acc += results['results'][task]['acc,none'] |
| 58 | logger.info(f'Average Acc: {total_acc/len(task_list)*100:.2f}%') |
| 59 | return results |
| 60 | |
| 61 | |
| 62 | def main(): |