MCPcopy Create free account
hub / github.com/NVIDIA/DALI / ParseTarFile

Function ParseTarFile

dali/operators/reader/loader/webdataset_loader.cc:157–205  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

155}
156
157inline void ParseTarFile(std::vector<SampleDesc>& samples_container,
158 std::vector<ComponentDesc>& components_container,
159 std::unique_ptr<FileStream>& tar_file) {
160 int64_t initial_file_pos = tar_file->TellRead();
161 TarArchive tar_archive(std::move(tar_file));
162
163 std::string last_filename;
164 // rewind to the first valid entry
165 for (; !tar_archive.EndOfArchive(); tar_archive.NextFile()) {
166 if (tar_archive.GetFileType() == TarArchive::ENTRY_FILE) {
167 std::tie(last_filename, std::ignore) = split_name(tar_archive.GetFileName());
168 break;
169 }
170 }
171 size_t last_components_size = components_container.size();
172 for (; !tar_archive.EndOfArchive(); tar_archive.NextFile()) {
173 if (tar_archive.GetFileType() != TarArchive::ENTRY_FILE) {
174 continue;
175 }
176
177 std::string basename, ext;
178 std::tie(basename, ext) = split_name(tar_archive.GetFileName());
179
180 if (basename.empty()) {
181 continue;
182 }
183
184 if (basename != last_filename) {
185 samples_container.emplace_back();
186 samples_container.back().components =
187 VectorRange<ComponentDesc>(components_container, last_components_size,
188 components_container.size() - last_components_size);
189 last_filename = std::move(basename);
190 last_components_size = components_container.size();
191 }
192
193 components_container.emplace_back();
194 components_container.back().size = tar_archive.GetFileSize();
195 components_container.back().offset = tar_archive.TellArchive() + tar_archive.HeaderSize();
196 components_container.back().ext = std::move(ext);
197 components_container.back().filename = tar_archive.GetFileName();
198 }
199 samples_container.emplace_back();
200 samples_container.back().components =
201 VectorRange<ComponentDesc>(components_container, last_components_size,
202 components_container.size() - last_components_size);
203
204 tar_file = tar_archive.Release();
205}
206
207} // namespace wds
208} // namespace detail

Callers 1

PrepareMetadataImplMethod · 0.85

Calls 13

split_nameFunction · 0.85
EndOfArchiveMethod · 0.80
NextFileMethod · 0.80
GetFileTypeMethod · 0.80
emplace_backMethod · 0.80
GetFileSizeMethod · 0.80
TellArchiveMethod · 0.80
HeaderSizeMethod · 0.80
TellReadMethod · 0.45
sizeMethod · 0.45
emptyMethod · 0.45
backMethod · 0.45

Tested by

no test coverage detected