(config, ues_word)
| 29 | |
| 30 | |
| 31 | def build_dataset(config, ues_word): |
| 32 | if ues_word: |
| 33 | tokenizer = lambda x: x.split(' ') # 以空格隔开，word-level |
| 34 | else: |
| 35 | tokenizer = lambda x: [y for y in x] # char-level |
| 36 | if os.path.exists(config.vocab_path): |
| 37 | vocab = pkl.load(open(config.vocab_path, 'rb')) |
| 38 | else: |
| 39 | vocab = build_vocab(config.train_path, tokenizer=tokenizer, max_size=MAX_VOCAB_SIZE, min_freq=1) |
| 40 | pkl.dump(vocab, open(config.vocab_path, 'wb')) |
| 41 | print(f"Vocab size: {len(vocab)}") |
| 42 | |
| 43 | def load_dataset(path, pad_size=32): |
| 44 | contents = [] |
| 45 | with open(path, 'r', encoding='UTF-8') as f: |
| 46 | for line in tqdm(f): |
| 47 | lin = line.strip() |
| 48 | if not lin: |
| 49 | continue |
| 50 | content, label = lin.split('\t') |
| 51 | words_line = [] |
| 52 | token = tokenizer(content) |
| 53 | seq_len = len(token) |
| 54 | if pad_size: |
| 55 | if len(token) < pad_size: |
| 56 | token.extend([PAD] * (pad_size - len(token))) |
| 57 | else: |
| 58 | token = token[:pad_size] |
| 59 | seq_len = pad_size |
| 60 | # word to id |
| 61 | for word in token: |
| 62 | words_line.append(vocab.get(word, vocab.get(UNK))) |
| 63 | contents.append((words_line, int(label), seq_len)) |
| 64 | return contents # [([...], 0), ([...], 1), ...] |
| 65 | train = load_dataset(config.train_path, config.pad_size) |
| 66 | dev = load_dataset(config.dev_path, config.pad_size) |
| 67 | test = load_dataset(config.test_path, config.pad_size) |
| 68 | return vocab, train, dev, test |
| 69 | |
| 70 | |
| 71 | class DatasetIterater(object): |
no test coverage detected