(tf_config=None, server=None)
| 463 | |
| 464 | |
| 465 | def main(tf_config=None, server=None): |
| 466 | |
| 467 | if args.incremental_ckpt and not args.tf: |
| 468 | print("Incremental_Checkpoint is not really enabled.") |
| 469 | print("Please see the comments in the code.") |
| 470 | sys.exit() |
| 471 | |
| 472 | # check dataset |
| 473 | print('Checking dataset') |
| 474 | train_file = args.data_location |
| 475 | test_file = args.data_location |
| 476 | if args.parquet_dataset and not args.tf: |
| 477 | train_file += '/train.parquet' |
| 478 | test_file += '/eval.parquet' |
| 479 | else: |
| 480 | train_file += '/train.csv' |
| 481 | test_file += '/eval.csv' |
| 482 | if (not os.path.exists(train_file)) or (not os.path.exists(test_file)): |
| 483 | print("Dataset does not exist in the given data_location.") |
| 484 | sys.exit() |
| 485 | no_of_training_examples = 0 |
| 486 | no_of_test_examples = 0 |
| 487 | if args.parquet_dataset and not args.tf: |
| 488 | import pyarrow.parquet as pq |
| 489 | no_of_training_examples = pq.read_table(train_file).num_rows |
| 490 | no_of_test_examples = pq.read_table(test_file).num_rows |
| 491 | else: |
| 492 | no_of_training_examples = sum(1 for line in open(train_file)) |
| 493 | no_of_test_examples = sum(1 for line in open(test_file)) |
| 494 | print("Numbers of training dataset is {}".format(no_of_training_examples)) |
| 495 | print("Numbers of test dataset is {}".format(no_of_test_examples)) |
| 496 | |
| 497 | # set batch size, eporch & steps |
| 498 | batch_size = math.ceil( |
| 499 | args.batch_size / args.micro_batch |
| 500 | ) if args.micro_batch and not args.tf else args.batch_size |
| 501 | |
| 502 | if args.steps == 0: |
| 503 | no_of_epochs = 1 |
| 504 | train_steps = math.ceil( |
| 505 | (float(no_of_epochs) * no_of_training_examples) / batch_size) |
| 506 | else: |
| 507 | no_of_epochs = math.ceil( |
| 508 | (float(batch_size) * args.steps) / no_of_training_examples) |
| 509 | train_steps = args.steps |
| 510 | test_steps = math.ceil(float(no_of_test_examples) / batch_size) |
| 511 | print("The training steps is {}".format(train_steps)) |
| 512 | print("The testing steps is {}".format(test_steps)) |
| 513 | |
| 514 | # set fixed random seed |
| 515 | tf.set_random_seed(args.seed) |
| 516 | |
| 517 | # set directory path |
| 518 | model_dir = os.path.join(args.output_dir, |
| 519 | 'model_DeepFM_' + str(int(time.time()))) |
| 520 | checkpoint_dir = args.checkpoint if args.checkpoint else model_dir |
| 521 | print("Saving model checkpoints to " + checkpoint_dir) |
| 522 |
no test coverage detected