| 8 | |
| 9 | |
| 10 | def build_dataset(config): |
| 11 | |
| 12 | def load_dataset(path, pad_size=32): |
| 13 | contents = [] |
| 14 | with open(path, 'r', encoding='UTF-8') as f: |
| 15 | for line in tqdm(f): |
| 16 | lin = line.strip() |
| 17 | if not lin: |
| 18 | continue |
| 19 | content, label = lin.split('\t') |
| 20 | token = config.tokenizer.tokenize(content) |
| 21 | token = [CLS] + token |
| 22 | seq_len = len(token) |
| 23 | mask = [] |
| 24 | token_ids = config.tokenizer.convert_tokens_to_ids(token) |
| 25 | |
| 26 | if pad_size: |
| 27 | if len(token) < pad_size: |
| 28 | mask = [1] * len(token_ids) + [0] * (pad_size - len(token)) |
| 29 | token_ids += ([0] * (pad_size - len(token))) |
| 30 | else: |
| 31 | mask = [1] * pad_size |
| 32 | token_ids = token_ids[:pad_size] |
| 33 | seq_len = pad_size |
| 34 | contents.append((token_ids, int(label), seq_len, mask)) |
| 35 | return contents |
| 36 | train = load_dataset(config.train_path, config.pad_size) |
| 37 | dev = load_dataset(config.dev_path, config.pad_size) |
| 38 | test = load_dataset(config.test_path, config.pad_size) |
| 39 | return train, dev, test |
| 40 | |
| 41 | |
| 42 | class DatasetIterater(object): |