MCPcopy Create free account
hub / github.com/apache/arrow / GetPutData

Function GetPutData

cpp/src/arrow/flight/flight_benchmark.cc:188–239  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

186};
187
188arrow::Result<std::vector<SizedBatch>> GetPutData(const perf::Token& token) {
189 if (!FLAGS_data_file.empty()) {
190 ARROW_ASSIGN_OR_RAISE(auto file, arrow::io::ReadableFile::Open(FLAGS_data_file));
191 ARROW_ASSIGN_OR_RAISE(auto reader,
192 arrow::ipc::RecordBatchFileReader::Open(std::move(file)));
193 std::vector<SizedBatch> batches(reader->num_record_batches());
194 for (int i = 0; i < reader->num_record_batches(); i++) {
195 ARROW_ASSIGN_OR_RAISE(batches[i].batch, reader->ReadRecordBatch(i));
196 RETURN_NOT_OK(arrow::ipc::GetRecordBatchSize(*batches[i].batch, &batches[i].bytes));
197 }
198 return batches;
199 }
200
201 std::shared_ptr<Schema> schema =
202 arrow::schema({field("a", int64()), field("b", int64()), field("c", int64()),
203 field("d", int64())});
204
205 // This is hard-coded for right now, 4 columns each with int64
206 const int bytes_per_record = 32;
207
208 std::shared_ptr<ResizableBuffer> buffer;
209 std::vector<std::shared_ptr<Array>> arrays;
210
211 const int64_t total_records = token.definition().records_per_stream();
212 const int32_t length = token.definition().records_per_batch();
213 const int32_t ncolumns = 4;
214 for (int i = 0; i < ncolumns; ++i) {
215 RETURN_NOT_OK(MakeRandomByteBuffer(length * sizeof(int64_t), default_memory_pool(),
216 &buffer, static_cast<int32_t>(i) /* seed */));
217 arrays.push_back(std::make_shared<Int64Array>(length, buffer));
218 RETURN_NOT_OK(arrays.back()->Validate());
219 }
220
221 std::shared_ptr<RecordBatch> batch = RecordBatch::Make(schema, length, arrays);
222 std::vector<SizedBatch> batches;
223
224 int64_t records_sent = 0;
225 while (records_sent < total_records) {
226 if (records_sent + length > total_records) {
227 const int64_t last_length = total_records - records_sent;
228 // Hard-coded
229 batches.push_back(SizedBatch{batch->Slice(0, last_length),
230 /*bytes=*/last_length * bytes_per_record});
231 records_sent += last_length;
232 } else {
233 // Hard-coded
234 batches.push_back(SizedBatch{batch, /*bytes=*/length * bytes_per_record});
235 records_sent += length;
236 }
237 }
238 return batches;
239}
240
241arrow::Result<PerformanceResult> RunDoPutTest(FlightClient* client,
242 const FlightCallOptions& call_options,

Callers

nothing calls this directly

Calls 14

GetRecordBatchSizeFunction · 0.85
MakeRandomByteBufferFunction · 0.85
default_memory_poolFunction · 0.85
push_backMethod · 0.80
backMethod · 0.80
ARROW_ASSIGN_OR_RAISEFunction · 0.70
schemaFunction · 0.50
fieldFunction · 0.50
MakeFunction · 0.50
emptyMethod · 0.45
num_record_batchesMethod · 0.45
ReadRecordBatchMethod · 0.45

Tested by

no test coverage detected