| 179 | } |
| 180 | |
| 181 | void ILSVRC::CreateTrainData(string image_list, string input_folder, |
| 182 | string output_folder, size_t file_size = 12800) { |
| 183 | std::vector<std::pair<string, int>> file_list; |
| 184 | size_t *sum = new size_t[kImageNBytes]; |
| 185 | for (size_t i = 0; i < kImageNBytes; i++) sum[i] = 0u; |
| 186 | string image_file_name; |
| 187 | int label; |
| 188 | string outfile; |
| 189 | std::ifstream image_list_file(image_list.c_str(), std::ios::in); |
| 190 | while (image_list_file >> image_file_name >> label) |
| 191 | file_list.push_back(std::make_pair(image_file_name, label)); |
| 192 | LOG(INFO) << "Data Shuffling"; |
| 193 | std::shuffle(file_list.begin(), file_list.end(), |
| 194 | std::default_random_engine()); |
| 195 | LOG(INFO) << "Total number of training images is " << file_list.size(); |
| 196 | size_t num_train_images = file_list.size(); |
| 197 | if (file_size == 0) file_size = num_train_images; |
| 198 | for (size_t imageid = 0; imageid < num_train_images; imageid++) { |
| 199 | string path = input_folder + "/" + file_list[imageid].first; |
| 200 | Tensor image = ReadImage(path); |
| 201 | auto image_data = image.data<unsigned char>(); |
| 202 | for (size_t i = 0; i < kImageNBytes; i++) |
| 203 | sum[i] += static_cast<size_t>(image_data[i]); |
| 204 | label = file_list[imageid].second; |
| 205 | Tensor lb(Shape{1}, kInt); |
| 206 | lb.CopyDataFromHostPtr<int>(&label, 1); |
| 207 | std::vector<Tensor> input; |
| 208 | input.push_back(image); |
| 209 | input.push_back(lb); |
| 210 | string encoded_str = encoder->Encode(input); |
| 211 | if (writer == nullptr) { |
| 212 | writer = new BinFileWriter(); |
| 213 | outfile = output_folder + "/train" + |
| 214 | std::to_string(imageid / file_size + 1) + ".bin"; |
| 215 | writer->Open(outfile, kCreate); |
| 216 | } |
| 217 | writer->Write(path, encoded_str); |
| 218 | if ((imageid + 1) % file_size == 0) { |
| 219 | writer->Flush(); |
| 220 | writer->Close(); |
| 221 | LOG(INFO) << "Write " << file_size << " images into " << outfile; |
| 222 | delete writer; |
| 223 | writer = nullptr; |
| 224 | } |
| 225 | } |
| 226 | if (writer != nullptr) { |
| 227 | writer->Flush(); |
| 228 | writer->Close(); |
| 229 | LOG(INFO) << "Write " << num_train_images % file_size << " images into " |
| 230 | << outfile; |
| 231 | delete writer; |
| 232 | writer = nullptr; |
| 233 | } |
| 234 | size_t num_file = |
| 235 | num_train_images / file_size + ((num_train_images % file_size) ? 1 : 0); |
| 236 | LOG(INFO) << "Write " << num_train_images << " images into " << num_file |
| 237 | << " binary files"; |
| 238 | Tensor mean = Tensor(Shape{3, kImageSize, kImageSize}, kUChar); |