| 33 | using namespace Tuning; |
| 34 | |
| 35 | void Command::dataprep(int argc, char *argv[]) |
| 36 | { |
| 37 | std::unique_ptr<Dataset> inputDataset; |
| 38 | std::unique_ptr<DataWriter> dataWriter; |
| 39 | DatasetType datasetType; |
| 40 | DataWriterType dataWriterType; |
| 41 | Rule defaultRule; |
| 42 | std::vector<std::string> pathList; |
| 43 | std::vector<std::string> extensions; |
| 44 | std::string outputPath; |
| 45 | size_t maxNumEntriesPerFile; |
| 46 | Time reportInterval; |
| 47 | |
| 48 | cxxopts::Options options("rapfi data preparation utility" |
| 49 | "\nConverting the input dataset to the target dataset."); |
| 50 | options.add_options() // |
| 51 | ("o,output", "Output filename/directory", cxxopts::value<std::string>()) // |
| 52 | ("i,input", |
| 53 | "Input dataset filename/directory(s)", |
| 54 | cxxopts::value<std::vector<std::string>>()) // |
| 55 | ("input-type", |
| 56 | "Input dataset type, one of [bin, binpack, katago]", |
| 57 | cxxopts::value<std::string>()) // |
| 58 | ("output-type", |
| 59 | "Output dataset type, one of [txt, bin, bin_lz4, binpack, binpack_lz4, numpy]", |
| 60 | cxxopts::value<std::string>()->default_value("numpy")) // |
| 61 | ("default-rule", |
| 62 | "Default rule for dataset type that does not contain rule infomation", |
| 63 | cxxopts::value<std::string>()->default_value("freestyle")) // |
| 64 | ("dataset-file-extensions", |
| 65 | "Extensions to filter dataset file in a directory", |
| 66 | cxxopts::value<std::vector<std::string>>()->default_value(".bin,.binpack,.lz4,.npz")) // |
| 67 | ("max-entries-per-file", |
| 68 | "Max number of entries per NPZ file", |
| 69 | cxxopts::value<size_t>()->default_value("25000")) // |
| 70 | ("report-interval", |
| 71 | "Time (ms) between two progress report message", |
| 72 | cxxopts::value<Time>()->default_value("10000")) // |
| 73 | ("h,help", "Print dataprep usage"); |
| 74 | |
| 75 | try { |
| 76 | auto args = options.parse(argc, argv); |
| 77 | |
| 78 | if (args.count("help")) { |
| 79 | std::cout << options.help() << std::endl; |
| 80 | std::exit(EXIT_SUCCESS); |
| 81 | } |
| 82 | |
| 83 | if (!args.count("input")) |
| 84 | throw std::invalid_argument("there must be at least one input dataset"); |
| 85 | |
| 86 | datasetType = parseDatasetType(args["input-type"].as<std::string>()); |
| 87 | dataWriterType = parseDataWriterType(args["output-type"].as<std::string>()); |
| 88 | defaultRule = parseRule(args["default-rule"].as<std::string>()); |
| 89 | pathList = args["input"].as<std::vector<std::string>>(); |
| 90 | extensions = args["dataset-file-extensions"].as<std::vector<std::string>>(); |
| 91 | outputPath = args["output"].as<std::string>(); |
| 92 | maxNumEntriesPerFile = args["max-entries-per-file"].as<size_t>(); |
nothing calls this directly
no test coverage detected