MCPcopy Create free account
hub / github.com/bytedance/bolt / load

Method load

bolt/dwio/common/CachedBufferedInput.cpp:182–233  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

180 cache::SsdFile* FOLLY_NULLABLE ssdFile = nullptr;
181 auto ssdCache = cache_->ssdCache();
182 if (ssdCache) {
183 ssdFile = &ssdCache->file(fileNum_);
184 }
185 // Extra requests made for preloadable regions that are larger then
186 // 'loadQuantum'.
187 std::vector<std::unique_ptr<CacheRequest>> extraRequests;
188 std::vector<CacheRequest*> storageLoad[2];
189 std::vector<CacheRequest*> ssdLoad[2];
190 for (auto& request : requests) {
191 cache::TrackingData trackingData;
192 const bool prefetchAnyway = request.trackingId.empty() ||
193 request.trackingId.id() == StreamIdentifier::sequentialFile().id_;
194 if (!prefetchAnyway && (tracker_ != nullptr)) {
195 trackingData = tracker_->trackingData(request.trackingId);
196 }
197 const int loadIndex =
198 (prefetchAnyway || isPrefetchPct(adjustedReadPct(trackingData))) ? 1
199 : 0;
200 auto parts = makeRequestParts(
201 request, trackingData, options_.loadQuantum(), extraRequests);
202 for (auto part : parts) {
203 if (cache_->exists(part->key)) {
204 continue;
205 }
206 if (ssdFile != nullptr) {
207 part->ssdPin = ssdFile->find(part->key);
208 if (!part->ssdPin.empty() && part->ssdPin.run().size() < part->size) {
209 LOG(INFO) << "IOERR: Ignoring SSD shorter than requested: "
210 << part->ssdPin.run().size() << " vs " << part->size;
211 part->ssdPin.clear();
212 }
213 if (!part->ssdPin.empty()) {
214 ssdLoad[loadIndex].push_back(part);
215 continue;
216 }
217 }
218 storageLoad[loadIndex].push_back(part);
219 }
220 }
221
222 std::sort(storageLoad[0].begin(), storageLoad[0].end(), lessThan<false>);
223 std::sort(storageLoad[1].begin(), storageLoad[1].end(), lessThan<false>);
224 std::sort(ssdLoad[0].begin(), ssdLoad[0].end(), lessThan<true>);
225 std::sort(ssdLoad[1].begin(), ssdLoad[1].end(), lessThan<true>);
226 makeLoads<false>(storageLoad);
227 makeLoads<true>(ssdLoad);
228}
229
230template <bool kSsd>
231void CachedBufferedInput::makeLoads(std::vector<CacheRequest*> requests[2]) {
232 std::vector<int32_t> groupEnds[2];
233 groupEnds[1] = groupRequests<kSsd>(requests[1], true);
234 moveCoalesced(
235 requests[1],
236 groupEnds[1],

Callers 1

loadDataMethod · 0.45

Calls 15

isPrefetchPctFunction · 0.85
makeRequestPartsFunction · 0.85
ssdCacheMethod · 0.80
trackingDataMethod · 0.80
fileMethod · 0.45
emptyMethod · 0.45
idMethod · 0.45
loadQuantumMethod · 0.45
existsMethod · 0.45
findMethod · 0.45
sizeMethod · 0.45
runMethod · 0.45

Tested by

no test coverage detected