| 16 | |
| 17 | @staticmethod |
| 18 | def load(path: str, tokenizer_path: str, tokenizer_kwargs: Optional[Dict] = dict(), num_gram: int=5, num_replica: int=5): |
| 19 | import numpy as np |
| 20 | from transformers import AutoTokenizer |
| 21 | tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, trust_remote_code=True, **tokenizer_kwargs) |
| 22 | data = [] |
| 23 | with open(path, encoding='utf-8') as f: |
| 24 | for index, line in enumerate(f): |
| 25 | line = json.loads(line) |
| 26 | if 'rewritten' in path: |
| 27 | text = line['rewritten_question'] + ' ' + line['rewritten_answer'] |
| 28 | elif 'origin' in path: |
| 29 | text = line['question'] + ' ' + line['answer'] |
| 30 | else: |
| 31 | raise ValueError(f'Unknown file type: {path}') |
| 32 | tokens = tokenizer.encode(text, add_special_tokens=False) |
| 33 | if len(tokens) >= num_gram + max(num_replica, 2): |
| 34 | starting_points = np.linspace(2, len(tokens) - num_gram, num=num_replica, endpoint=True, dtype=int).tolist() |
| 35 | else: |
| 36 | starting_points = np.linspace(2, max(2, len(tokens)), num=num_replica, endpoint=True, dtype=int).tolist() |
| 37 | for s in starting_points: |
| 38 | data.append({ |
| 39 | 'index': index, |
| 40 | 'prompt': tokenizer.decode(tokens[:s]), |
| 41 | 'reference': tokenizer.decode(tokens[s:s+num_gram]) |
| 42 | }) |
| 43 | dataset = Dataset.from_list(data) |
| 44 | return dataset |
| 45 | |
| 46 | def exact_match_score(predicted_text, original_text): |
| 47 | return predicted_text == original_text |