| 387 | } |
| 388 | |
| 389 | void WebdatasetLoader::PrepareMetadataImpl() { |
| 390 | if (!dont_use_mmap_) { |
| 391 | mmap_reserver_ = FileStream::MappingReserver(static_cast<unsigned int>(paths_.size())); |
| 392 | } |
| 393 | copy_read_data_ = dont_use_mmap_ || !mmap_reserver_.CanShareMappedData(); |
| 394 | |
| 395 | generate_index_ = index_paths_.size() == 0; |
| 396 | if (generate_index_) { |
| 397 | DALI_WARN("Index file not provided, it may take some time to infer it from the tar file"); |
| 398 | } |
| 399 | |
| 400 | FileStream::Options opts; |
| 401 | opts.read_ahead = read_ahead_; |
| 402 | opts.use_mmap = !copy_read_data_; |
| 403 | opts.use_odirect = false; |
| 404 | |
| 405 | // initializing all the readers |
| 406 | wds_shards_.reserve(paths_.size()); |
| 407 | for (auto& path : paths_) { |
| 408 | // If an actual URI, disable mmap |
| 409 | opts.use_mmap = !copy_read_data_; |
| 410 | wds_shards_.emplace_back(FileStream::Open(path, opts)); |
| 411 | } |
| 412 | |
| 413 | // preparing the map from extensions to outputs |
| 414 | std::unordered_map<std::string, std::vector<size_t>> ext_map; |
| 415 | for (size_t output_index = 0; output_index < ext_.size(); output_index++) { |
| 416 | for (auto& ext : ext_[output_index]) { |
| 417 | ext_map[ext].push_back(output_index); |
| 418 | } |
| 419 | } |
| 420 | |
| 421 | // collecting and filtering the index files |
| 422 | std::vector<detail::wds::SampleDesc> unfiltered_samples; |
| 423 | std::vector<detail::wds::ComponentDesc> unfiltered_components; |
| 424 | bitmask was_output_set; |
| 425 | was_output_set.resize(ext_.size(), false); |
| 426 | output_indicies_.reserve(ext_.size()); |
| 427 | |
| 428 | std::vector<size_t> dtype_sizes_(dtypes_.size()); |
| 429 | for (size_t i = 0; i < dtypes_.size(); i++) { |
| 430 | dtype_sizes_[i] = TypeTable::GetTypeInfo(dtypes_[i]).size(); |
| 431 | } |
| 432 | |
| 433 | for (size_t wds_shard_index = 0; wds_shard_index < paths_.size(); wds_shard_index++) { |
| 434 | unfiltered_samples.resize(0); |
| 435 | unfiltered_components.resize(0); |
| 436 | if (generate_index_) { |
| 437 | detail::wds::ParseTarFile(unfiltered_samples, unfiltered_components, |
| 438 | wds_shards_[wds_shard_index]); |
| 439 | } else { |
| 440 | detail::wds::ParseIndexFile(unfiltered_samples, unfiltered_components, |
| 441 | index_paths_[wds_shard_index]); |
| 442 | } |
| 443 | |
| 444 | for (auto& sample : unfiltered_samples) { |
| 445 | detail::wds::SampleDesc new_sample{ |
| 446 | detail::wds::VectorRange<detail::wds::ComponentDesc>(components_, components_.size()), |
nothing calls this directly
no test coverage detected