MCPcopy Create free account
hub / github.com/google-research/language / input_function

Function input_function

language/labs/exemplar_decoding/utils/data.py:103–162  ·  view source on GitHub ↗

Input function.

(is_train, vocab, hps)

Source from the content-addressed store, hash-verified

101
102
103def input_function(is_train, vocab, hps):
104 """Input function."""
105 if is_train:
106 data_path = hps.train_path
107 else:
108 data_path = hps.dev_path
109
110 dataset = tf.data.Dataset.from_generator(
111 partial(generate_tensors,
112 path=data_path,
113 vocab=vocab,
114 hps=hps,
115 is_train=is_train),
116 INPUT_DTYPES, INPUT_SHAPES)
117
118 if is_train:
119 dataset = dataset.shuffle(hps.batch_size * 1000,
120 reshuffle_each_iteration=True)
121 dataset = dataset.repeat()
122 if hps.dataset == "giga":
123 if hps.encode_neighbor or hps.model == "nn2seq":
124 bucket_boundaries = [50, 100, 150, 200, 250, 300]
125 else:
126 bucket_boundaries = [25, 50, 75, 100, 125, 150]
127 elif hps.dataset == "nyt":
128 bucket_boundaries = [100, 300, 500, 700, 900]
129 elif hps.dataset == "cnnd":
130 bucket_boundaries = [100, 200, 300, 400, 500, 600]
131
132 padding_values = {
133 "src_inputs": vocab.word2id(PAD_TOKEN),
134 "src_len": vocab.word2id(PAD_TOKEN),
135 "neighbor_ids": vocab.word2id(PAD_TOKEN),
136 "neighbor_inputs": vocab.word2id(PAD_TOKEN),
137 "neighbor_binary": vocab.word2id(PAD_TOKEN),
138 "neighbor_len": vocab.word2id(PAD_TOKEN),
139 "decoder_inputs": vocab.word2id(PAD_TOKEN),
140 "decoder_len": vocab.word2id(PAD_TOKEN),
141 "targets": vocab.word2id(PAD_TOKEN),
142 "reference": PAD_TOKEN,
143 "reference_len": vocab.word2id(PAD_TOKEN)
144 }
145 if is_train:
146 dataset = dataset.apply(tf.contrib.data.bucket_by_sequence_length(
147 element_length_func=lambda x: x["src_len"],
148 bucket_boundaries=bucket_boundaries,
149 bucket_batch_sizes=[hps.batch_size]*(len(bucket_boundaries)+1),
150 padded_shapes=INPUT_SHAPES,
151 padding_values=padding_values))
152 else:
153 dataset = dataset.apply(tf.contrib.data.bucket_by_sequence_length(
154 element_length_func=lambda x: x["src_len"],
155 bucket_boundaries=[100000],
156 bucket_batch_sizes=[hps.batch_size]*2,
157 padded_shapes=INPUT_SHAPES,
158 padding_values=padding_values))
159
160 dataset = dataset.prefetch(hps.batch_size)

Callers

nothing calls this directly

Calls 1

word2idMethod · 0.80

Tested by

no test coverage detected

Used in the wild real call sites across dependent graphs

searching dependent graphs…