| 306 | # generate dataset pipeline |
| 307 | def build_model_input(filename, batch_size, num_epochs): |
| 308 | def parse_csv(value): |
| 309 | tf.logging.info('Parsing {}'.format(filename)) |
| 310 | HASH_defaults = [[" "] for i in range(0, len(HASH_INPUTS))] |
| 311 | label_defaults = [[0] for i in range (0, len(LABEL_COLUMNS))] |
| 312 | column_headers = LABEL_COLUMNS + HASH_INPUTS |
| 313 | record_defaults = label_defaults + HASH_defaults |
| 314 | columns = tf.io.decode_csv(value, record_defaults=record_defaults) |
| 315 | all_columns = collections.OrderedDict(zip(column_headers, columns)) |
| 316 | labels = [] |
| 317 | for i in range(0, len(LABEL_COLUMNS)): |
| 318 | labels.append(all_columns.pop(LABEL_COLUMNS[i])) |
| 319 | label = tf.stack(labels, axis=1) |
| 320 | features = all_columns |
| 321 | return features, label |
| 322 | |
| 323 | def parse_parquet(value): |
| 324 | tf.logging.info('Parsing {}'.format(filename)) |