(
path: str,
dataset: str = 'squad',
max_seq_length: int = 4096,
tokenizer_model: str = 'gpt-4',
template:
str = 'Answer the question based on the given documents. Only give me the answer and do not output any other words.\n\nThe following are given documents.\n\n{context}\n\nAnswer the question based on the given documents. Only give me the answer and do not output any other words.\n\nQuestion: {query} Answer:',
tokens_to_generate: int = 32,
num_samples: int = 500,
pre_samples: int = 0,
random_seed: int = 42,
remove_newline_tab: str = '',
)
| 20 | |
| 21 | @staticmethod |
| 22 | def load( |
| 23 | path: str, |
| 24 | dataset: str = 'squad', |
| 25 | max_seq_length: int = 4096, |
| 26 | tokenizer_model: str = 'gpt-4', |
| 27 | template: |
| 28 | str = 'Answer the question based on the given documents. Only give me the answer and do not output any other words.\n\nThe following are given documents.\n\n{context}\n\nAnswer the question based on the given documents. Only give me the answer and do not output any other words.\n\nQuestion: {query} Answer:', |
| 29 | tokens_to_generate: int = 32, |
| 30 | num_samples: int = 500, |
| 31 | pre_samples: int = 0, |
| 32 | random_seed: int = 42, |
| 33 | remove_newline_tab: str = '', |
| 34 | ) -> Dataset: |
| 35 | |
| 36 | if tokenizer_model == 'gpt-4': |
| 37 | tokenizer = tiktoken.encoding_for_model(tokenizer_model) |
| 38 | else: |
| 39 | tokenizer = AutoTokenizer.from_pretrained(tokenizer_model, |
| 40 | trust_remote_code=True) |
| 41 | |
| 42 | random.seed(random_seed) |
| 43 | np.random.seed(random_seed) |
| 44 | |
| 45 | # Read SQuAD QA dataset |
| 46 | def _read_squad(file): |
| 47 | file = get_data_path(file, local_mode=True) |
| 48 | with open(file) as f: |
| 49 | data = json.load(f) |
| 50 | |
| 51 | total_docs = [ |
| 52 | p['context'] for d in data['data'] for p in d['paragraphs'] |
| 53 | ] |
| 54 | total_docs = sorted(list(set(total_docs))) |
| 55 | total_docs_dict = {c: idx for idx, c in enumerate(total_docs)} |
| 56 | |
| 57 | total_qas = [] |
| 58 | for d in data['data']: |
| 59 | more_docs = [ |
| 60 | total_docs_dict[p['context']] for p in d['paragraphs'] |
| 61 | ] |
| 62 | for p in d['paragraphs']: |
| 63 | for qas in p['qas']: |
| 64 | if not qas['is_impossible']: |
| 65 | total_qas.append({ |
| 66 | 'query': |
| 67 | qas['question'], |
| 68 | 'outputs': [a['text'] for a in qas['answers']], |
| 69 | 'context': [total_docs_dict[p['context']]], |
| 70 | 'more_context': [ |
| 71 | idx for idx in more_docs |
| 72 | if idx != total_docs_dict[p['context']] |
| 73 | ], |
| 74 | }) |
| 75 | |
| 76 | return total_qas, total_docs |
| 77 | |
| 78 | # Read Hotpot QA dataset |
| 79 | def _read_hotpotqa(file_path): |
no outgoing calls
no test coverage detected