| 41 | print(f"Vocab size: {len(vocab)}") |
| 42 | |
| 43 | def load_dataset(path, pad_size=32): |
| 44 | contents = [] |
| 45 | with open(path, 'r', encoding='UTF-8') as f: |
| 46 | for line in tqdm(f): |
| 47 | lin = line.strip() |
| 48 | if not lin: |
| 49 | continue |
| 50 | content, label = lin.split('\t') |
| 51 | words_line = [] |
| 52 | token = tokenizer(content) |
| 53 | seq_len = len(token) |
| 54 | if pad_size: |
| 55 | if len(token) < pad_size: |
| 56 | token.extend([PAD] * (pad_size - len(token))) |
| 57 | else: |
| 58 | token = token[:pad_size] |
| 59 | seq_len = pad_size |
| 60 | # word to id |
| 61 | for word in token: |
| 62 | words_line.append(vocab.get(word, vocab.get(UNK))) |
| 63 | contents.append((words_line, int(label), seq_len)) |
| 64 | return contents # [([...], 0), ([...], 1), ...] |
| 65 | train = load_dataset(config.train_path, config.pad_size) |
| 66 | dev = load_dataset(config.dev_path, config.pad_size) |
| 67 | test = load_dataset(config.test_path, config.pad_size) |