MCPcopy Create free account
hub / github.com/apache/arrow / GetFragmentsImpl

Method GetFragmentsImpl

cpp/src/arrow/dataset/file_base.cc:308–347  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

306}
307
308Result<FragmentIterator> FileSystemDataset::GetFragmentsImpl(
309 compute::Expression predicate) {
310 if (predicate == compute::literal(true)) {
311 // trivial predicate; skip subtree pruning
312 return MakeVectorIterator(FragmentVector(fragments_.begin(), fragments_.end()));
313 }
314
315 std::vector<int> fragment_indices;
316
317 std::vector<compute::Expression> predicates{predicate};
318 RETURN_NOT_OK(subtrees_->forest.Visit(
319 [&](Forest::Ref ref) -> Result<bool> {
320 if (auto fragment_index =
321 std::get_if<int>(&subtrees_->fragments_and_subtrees[ref.i])) {
322 fragment_indices.push_back(*fragment_index);
323 return false;
324 }
325
326 const auto& subtree_expr =
327 std::get<compute::Expression>(subtrees_->fragments_and_subtrees[ref.i]);
328 ARROW_ASSIGN_OR_RAISE(auto simplified,
329 SimplifyWithGuarantee(predicates.back(), subtree_expr));
330
331 if (!simplified.IsSatisfiable()) {
332 return false;
333 }
334
335 predicates.push_back(std::move(simplified));
336 return true;
337 },
338 [&](Forest::Ref ref) { predicates.pop_back(); }));
339
340 std::sort(fragment_indices.begin(), fragment_indices.end());
341
342 FragmentVector fragments(fragment_indices.size());
343 std::transform(fragment_indices.begin(), fragment_indices.end(), fragments.begin(),
344 [this](int i) { return fragments_[i]; });
345
346 return MakeVectorIterator(std::move(fragments));
347}
348
349Status FileWriter::Write(RecordBatchReader* batches) {
350 while (true) {

Callers

nothing calls this directly

Calls 8

MakeVectorIteratorFunction · 0.85
push_backMethod · 0.80
IsSatisfiableMethod · 0.80
literalFunction · 0.50
beginMethod · 0.45
endMethod · 0.45
VisitMethod · 0.45
sizeMethod · 0.45

Tested by

no test coverage detected