| 98 | // } |
| 99 | |
| 100 | DataConfig searchFiles(const DataConfig& config) { |
| 101 | int n = config.file_size(); |
| 102 | CHECK_GE(n, 1) << "empty files: " << config.DebugString(); |
| 103 | std::vector<std::string> matched_files; |
| 104 | for (int i = 0; i < n; ++i) { |
| 105 | std::regex pattern; |
| 106 | try { |
| 107 | pattern = std::regex(getFilename(config.file(i))); |
| 108 | } catch (const std::regex_error& e) { |
| 109 | CHECK(false) << getFilename(config.file(i)) |
| 110 | << " is not valid (supported) regex, regex_error caught: " |
| 111 | << e.what() << ". you may try gcc>=4.9 or llvm>=3.4"; |
| 112 | } |
| 113 | auto dir = config; dir.clear_file(); |
| 114 | dir.add_file(getPath(config.file(i))); |
| 115 | // match regex |
| 116 | auto files = readFilenamesInDirectory(dir); |
| 117 | |
| 118 | // list all files found in dir |
| 119 | if (FLAGS_verbose) { |
| 120 | size_t file_idx = 1; |
| 121 | for (const auto& file : files) { |
| 122 | LI << "All files found in [" << dir.file(0) << "]; [" << |
| 123 | file_idx++ << "/" << files.size() << "] [" << file << "]"; |
| 124 | } |
| 125 | } |
| 126 | |
| 127 | for (auto& f : files) { |
| 128 | if (std::regex_match(getFilename(f), pattern)) { |
| 129 | auto l = config.format() == DataConfig::TEXT ? f : removeExtension(f); |
| 130 | matched_files.push_back(dir.file(0) + "/" + getFilename(l)); |
| 131 | } |
| 132 | } |
| 133 | |
| 134 | // list all matched files |
| 135 | if (FLAGS_verbose) { |
| 136 | size_t file_idx = 1; |
| 137 | for (const auto& file : matched_files) { |
| 138 | LI << "All matched files [" << file_idx++ << "/" << matched_files.size() << |
| 139 | "] [" << file << "]"; |
| 140 | } |
| 141 | } |
| 142 | } |
| 143 | // remove duplicate files |
| 144 | std::sort(matched_files.begin(), matched_files.end()); |
| 145 | auto it = std::unique(matched_files.begin(), matched_files.end()); |
| 146 | matched_files.resize(std::distance(matched_files.begin(), it)); |
| 147 | DataConfig ret = config; ret.clear_file(); |
| 148 | for (auto& f : matched_files) ret.add_file(f); |
| 149 | return ret; |
| 150 | } |
| 151 | |
| 152 | std::vector<DataConfig> divideFiles(const DataConfig& data, int num) { |
| 153 | // evenly divide files |