(tf_config=None, server=None)
| 666 | |
| 667 | |
| 668 | def main(tf_config=None, server=None): |
| 669 | # check dataset and count data set size |
| 670 | print("Checking dataset...") |
| 671 | train_file = args.data_location |
| 672 | test_file = args.data_location |
| 673 | if args.parquet_dataset and not args.tf: |
| 674 | train_file += '/train.parquet' |
| 675 | test_file += '/eval.parquet' |
| 676 | else: |
| 677 | train_file += '/train.csv' |
| 678 | test_file += '/eval.csv' |
| 679 | if (not os.path.exists(train_file)) or (not os.path.exists(test_file)): |
| 680 | print("Dataset does not exist in the given data_location.") |
| 681 | sys.exit() |
| 682 | no_of_training_examples = 0 |
| 683 | no_of_test_examples = 0 |
| 684 | if args.parquet_dataset and not args.tf: |
| 685 | import pyarrow.parquet as pq |
| 686 | no_of_training_examples = pq.read_table(train_file).num_rows |
| 687 | no_of_test_examples = pq.read_table(test_file).num_rows |
| 688 | else: |
| 689 | no_of_training_examples = sum(1 for line in open(train_file)) |
| 690 | no_of_test_examples = sum(1 for line in open(test_file)) |
| 691 | print("Numbers of training dataset is {}".format(no_of_training_examples)) |
| 692 | print("Numbers of test dataset is {}".format(no_of_test_examples)) |
| 693 | |
| 694 | # set batch size, epoch & steps |
| 695 | batch_size = math.ceil( |
| 696 | args.batch_size / args.micro_batch |
| 697 | ) if args.micro_batch and not args.tf else args.batch_size |
| 698 | |
| 699 | if args.steps == 0: |
| 700 | no_of_epochs = 1 |
| 701 | train_steps = math.ceil( |
| 702 | (float(no_of_epochs) * no_of_training_examples) / batch_size) |
| 703 | else: |
| 704 | no_of_epochs = math.ceil( |
| 705 | (float(batch_size) * args.steps) / no_of_training_examples) |
| 706 | train_steps = args.steps |
| 707 | test_steps = math.ceil(float(no_of_test_examples) / batch_size) |
| 708 | print("The training steps is {}".format(train_steps)) |
| 709 | print("The testing steps is {}".format(test_steps)) |
| 710 | |
| 711 | # set fixed random seed |
| 712 | tf.set_random_seed(args.seed) |
| 713 | |
| 714 | # set directory path for checkpoint_dir |
| 715 | model_dir = os.path.join(args.output_dir, |
| 716 | 'model_WIDE_AND_DEEP_' + str(int(time.time()))) |
| 717 | checkpoint_dir = args.checkpoint if args.checkpoint else model_dir |
| 718 | print("Saving model checkpoints to " + checkpoint_dir) |
| 719 | |
| 720 | # create data pipline of train & test dataset |
| 721 | train_dataset = build_model_input(train_file, batch_size, no_of_epochs) |
| 722 | test_dataset = build_model_input(test_file, batch_size, 1) |
| 723 | |
| 724 | dataset_output_types = tf.data.get_output_types(train_dataset) |
| 725 | dataset_output_shapes = tf.data.get_output_shapes(test_dataset) |
no test coverage detected