| 50 | } // namespace detail |
| 51 | |
| 52 | void NumpyLoader::ReadSample(NumpyFileWrapper& target) { |
| 53 | const auto& entry = file_entries_[current_index_++]; |
| 54 | auto filename = entry.filename; |
| 55 | auto size = entry.size; |
| 56 | |
| 57 | // handle wrap-around |
| 58 | MoveToNextShard(current_index_); |
| 59 | |
| 60 | // metadata info |
| 61 | DALIMeta meta; |
| 62 | meta.SetSourceInfo(filename); |
| 63 | meta.SetSkipSample(false); |
| 64 | |
| 65 | // if data is cached, skip loading |
| 66 | if (ShouldSkipImage(filename)) { |
| 67 | meta.SetSkipSample(true); |
| 68 | target.data.Reset(); |
| 69 | target.data.SetMeta(meta); |
| 70 | target.data.Resize({0}, DALI_UINT8); |
| 71 | target.filename.clear(); |
| 72 | return; |
| 73 | } |
| 74 | |
| 75 | auto path = filesystem::join_path(file_root_, filename); |
| 76 | FileStream::Options opts; |
| 77 | opts.read_ahead = read_ahead_; |
| 78 | opts.use_mmap = !copy_read_data_; |
| 79 | opts.use_odirect = use_o_direct_; |
| 80 | auto current_file = FileStream::Open(path, opts, size); |
| 81 | |
| 82 | // read the header |
| 83 | numpy::HeaderData header; |
| 84 | auto ret = header_cache_.GetFromCache(filename, header); |
| 85 | try { |
| 86 | if (!ret) { |
| 87 | if (opts.use_odirect) { |
| 88 | numpy::ParseODirectHeader(header, current_file.get(), o_direct_alignm_, |
| 89 | o_direct_read_len_alignm_); |
| 90 | } else { |
| 91 | numpy::ParseHeader(header, current_file.get()); |
| 92 | } |
| 93 | header_cache_.UpdateCache(filename, header); |
| 94 | } |
| 95 | } catch (const std::runtime_error &e) { |
| 96 | DALI_FAIL(e.what(), ". File: ", filename); |
| 97 | } |
| 98 | current_file->SeekRead(header.data_offset); |
| 99 | |
| 100 | Index nbytes = header.nbytes(); |
| 101 | target.shape = header.shape; |
| 102 | target.type = header.type(); |
| 103 | target.meta = meta; |
| 104 | target.data_offset = header.data_offset; |
| 105 | target.nbytes = nbytes; |
| 106 | target.filename = std::move(path); |
| 107 | |
| 108 | if (!opts.use_mmap || !current_file->CanMemoryMap()) { |
| 109 | target.current_file = std::move(current_file); |