Function ParseIndexFile

dali/operators/reader/loader/webdataset_loader.cc:118–150

}

inline void ParseIndexFile(std::vector<SampleDesc>& samples_container,
                           std::vector<ComponentDesc>& components_container,
                           const std::string& index_path) {
  std::ifstream index_file(index_path);

  // Index Checking
  std::string global_meta;
  getline(index_file, global_meta);
  std::stringstream global_meta_stream(global_meta);
  std::string index_version_str;
  DALI_ENFORCE(global_meta_stream >> index_version_str,
               IndexFileErrMsg(index_path, 0, "no version signature found"));
  DALI_ENFORCE(
      kSupportedIndexVersions.count(index_version_str) > 0,
      IndexFileErrMsg(index_path, 0,
                      make_string("Unsupported version of the index file (",
                                  index_version_str, ").")));
  int index_version = ParseIndexVersion(index_version_str);

  // Getting the number of samples in the index file
  int64_t sample_desc_num_signed;
  DALI_ENFORCE(global_meta_stream >> sample_desc_num_signed,
               IndexFileErrMsg(index_path, 0, "no sample count found"));
  DALI_ENFORCE(sample_desc_num_signed > 0,
               IndexFileErrMsg(index_path, 0, "sample count must be positive"));

  const size_t sample_desc_num = sample_desc_num_signed;
  samples_container.reserve(samples_container.size() + sample_desc_num);
  for (size_t sample_index = 0; sample_index < sample_desc_num; sample_index++) {
    ParseSampleDesc(samples_container, components_container, index_file, index_path,
                    sample_index + 1, index_version);
  }
}

std::tuple<std::string, std::string> split_name(const std::string& filepath) {
  size_t dot_pos = filepath.find('.', filepath.rfind('/') + 1);