Create vocabulary file (if it does not exist yet) from data file. Data file is assumed to contain one sentence per line. Each sentence is tokenized and digits are normalized (if normalize_digits is set). Vocabulary contains the most-frequent tokens up to max_vocabulary_size. We write it to
(vocabulary_path, data_path, max_vocabulary_size,
tokenizer=None, normalize_digits=True)
| 76 | |
| 77 | |
| 78 | def create_vocabulary(vocabulary_path, data_path, max_vocabulary_size, |
| 79 | tokenizer=None, normalize_digits=True): |
| 80 | """Create vocabulary file (if it does not exist yet) from data file. |
| 81 | |
| 82 | Data file is assumed to contain one sentence per line. Each sentence is |
| 83 | tokenized and digits are normalized (if normalize_digits is set). |
| 84 | Vocabulary contains the most-frequent tokens up to max_vocabulary_size. |
| 85 | We write it to vocabulary_path in a one-token-per-line format, so that later |
| 86 | token in the first line gets id=0, second line gets id=1, and so on. |
| 87 | |
| 88 | Args: |
| 89 | vocabulary_path: path where the vocabulary will be created. |
| 90 | data_path: data file that will be used to create vocabulary. |
| 91 | max_vocabulary_size: limit on the size of the created vocabulary. |
| 92 | tokenizer: a function to use to tokenize each data sentence; |
| 93 | if None, basic_tokenizer will be used. |
| 94 | normalize_digits: Boolean; if true, all digits are replaced by 0s. |
| 95 | """ |
| 96 | if not gfile.Exists(vocabulary_path): |
| 97 | print("Creating vocabulary %s from data %s" % (vocabulary_path, data_path)) |
| 98 | vocab = {} |
| 99 | with gfile.GFile(data_path, mode="rb") as f: |
| 100 | counter = 0 |
| 101 | for line in f: |
| 102 | counter += 1 |
| 103 | if counter % 100000 == 0: |
| 104 | print(" processing line %d" % counter) |
| 105 | line = tf.compat.as_bytes(line) |
| 106 | tokens = tokenizer(line) if tokenizer else basic_tokenizer(line) |
| 107 | for w in tokens: |
| 108 | word = _DIGIT_RE.sub(b"0", w) if normalize_digits else w |
| 109 | if word in vocab: |
| 110 | vocab[word] += 1 |
| 111 | else: |
| 112 | vocab[word] = 1 |
| 113 | vocab_list = _START_VOCAB + sorted(vocab, key=vocab.get, reverse=True) |
| 114 | if len(vocab_list) > max_vocabulary_size: |
| 115 | vocab_list = vocab_list[:max_vocabulary_size] |
| 116 | with gfile.GFile(vocabulary_path, mode="wb") as vocab_file: |
| 117 | for w in vocab_list: |
| 118 | vocab_file.write(w + b"\n") |
| 119 | |
| 120 | |
| 121 | def initialize_vocabulary(vocabulary_path): |
no test coverage detected