MCPcopy Create free account
hub / github.com/NVIDIA/DALI / ParseSampleDesc

Function ParseSampleDesc

dali/operators/reader/loader/webdataset_loader.cc:61–104  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

59
60
61inline void ParseSampleDesc(std::vector<SampleDesc>& samples_container,
62 std::vector<ComponentDesc>& components_container,
63 std::ifstream& index_file, const std::string& index_path, int64_t line,
64 int index_version) {
65 // Preparing the SampleDesc
66 samples_container.emplace_back();
67 samples_container.back().components =
68 VectorRange<ComponentDesc>(components_container, components_container.size());
69 samples_container.back().line_number = line;
70
71 // Getting the components data
72 std::string components_metadata;
73 std::getline(index_file, components_metadata);
74 std::stringstream components_stream(components_metadata);
75
76 // Reading consecutive components
77 ComponentDesc component;
78 while (components_stream >> component.ext) {
79 if (index_version == MakeVersionNumber(1, 2)) {
80 DALI_ENFORCE(
81 components_stream >> component.offset >> component.size >> component.filename,
82 IndexFileErrMsg(
83 index_path, line,
84 "Could not find all necessary component parameters (offset, size or filename). Every "
85 "record in the index file should look like: `<ext> <offset> <size> <filename>`."));
86 } else {
87 DALI_ENFORCE(components_stream >> component.offset >> component.size,
88 IndexFileErrMsg(
89 index_path, line,
90 "Could not find all necessary component parameters (offset or size). Every "
91 "record in the index file should look like: `<ext> <offset> <size>`."));
92 }
93 DALI_ENFORCE(
94 component.offset % kBlockSize == 0,
95 IndexFileErrMsg(index_path, line, "tar offset is not a multiple of tar block size (",
96 kBlockSize, "), perhaps the size value is exported before offset?"));
97 components_container.emplace_back(std::move(component));
98 samples_container.back().components.num++;
99 }
100
101 // Finishing up the SampleDesc
102 DALI_ENFORCE(samples_container.back().components.num,
103 IndexFileErrMsg(index_path, line, "no extensions provided for the sample"));
104}
105
106inline int ParseIndexVersion(const string& version_str) {
107 const char *s = version_str.c_str();

Callers 1

ParseIndexFileFunction · 0.85

Calls 5

MakeVersionNumberFunction · 0.85
IndexFileErrMsgFunction · 0.85
emplace_backMethod · 0.80
backMethod · 0.45
sizeMethod · 0.45

Tested by

no test coverage detected