MCPcopy Create free account
hub / github.com/apache/arrow / ScanBatchesAsync

Method ScanBatchesAsync

cpp/src/arrow/dataset/dataset.cc:106–153  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

104}
105
106Result<RecordBatchGenerator> InMemoryFragment::ScanBatchesAsync(
107 const std::shared_ptr<ScanOptions>& options) {
108 struct State {
109 State(std::shared_ptr<InMemoryFragment> fragment, int64_t batch_size)
110 : fragment(std::move(fragment)),
111 batch_index(0),
112 offset(0),
113 batch_size(batch_size) {}
114
115 std::shared_ptr<RecordBatch> Next() {
116 const auto& next_parent = fragment->record_batches_[batch_index];
117 if (offset < next_parent->num_rows()) {
118 auto next = next_parent->Slice(offset, batch_size);
119 offset += batch_size;
120 return next;
121 }
122 batch_index++;
123 offset = 0;
124 return nullptr;
125 }
126
127 bool Finished() { return batch_index >= fragment->record_batches_.size(); }
128
129 std::shared_ptr<InMemoryFragment> fragment;
130 std::size_t batch_index;
131 int64_t offset;
132 int64_t batch_size;
133 };
134
135 struct Generator {
136 Generator(std::shared_ptr<InMemoryFragment> fragment, int64_t batch_size)
137 : state(std::make_shared<State>(std::move(fragment), batch_size)) {}
138
139 Future<std::shared_ptr<RecordBatch>> operator()() {
140 while (!state->Finished()) {
141 auto next = state->Next();
142 if (next) {
143 return Future<std::shared_ptr<RecordBatch>>::MakeFinished(std::move(next));
144 }
145 }
146 return AsyncGeneratorEnd<std::shared_ptr<RecordBatch>>();
147 }
148
149 std::shared_ptr<State> state;
150 };
151 return Generator(checked_pointer_cast<InMemoryFragment>(shared_from_this()),
152 options->batch_size);
153}
154
155Future<std::optional<int64_t>> InMemoryFragment::CountRows(
156 compute::Expression predicate, const std::shared_ptr<ScanOptions>& options) {

Callers

nothing calls this directly

Calls 1

GeneratorClass · 0.70

Tested by

no test coverage detected