MCPcopy Create free account
hub / github.com/apache/arrow / OpenReaderAsync

Function OpenReaderAsync

cpp/src/arrow/dataset/file_csv.cc:292–344  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

290}
291
292static inline Future<std::shared_ptr<csv::StreamingReader>> OpenReaderAsync(
293 const FileSource& source, const CsvFileFormat& format,
294 const std::shared_ptr<ScanOptions>& scan_options, Executor* cpu_executor) {
295#ifdef ARROW_WITH_OPENTELEMETRY
296 auto tracer = arrow::internal::tracing::GetTracer();
297 auto span = tracer->StartSpan("arrow::dataset::CsvFileFormat::OpenReaderAsync");
298#endif
299 ARROW_ASSIGN_OR_RAISE(
300 auto fragment_scan_options,
301 GetFragmentScanOptions<CsvFragmentScanOptions>(
302 kCsvTypeName, scan_options.get(), format.default_fragment_scan_options));
303 ARROW_ASSIGN_OR_RAISE(auto reader_options, GetReadOptions(format, scan_options));
304 ARROW_ASSIGN_OR_RAISE(auto input, source.OpenCompressed());
305 if (fragment_scan_options->stream_transform_func) {
306 ARROW_ASSIGN_OR_RAISE(input, fragment_scan_options->stream_transform_func(input));
307 }
308 const auto& path = source.path();
309 ARROW_ASSIGN_OR_RAISE(
310 input, io::BufferedInputStream::Create(reader_options.block_size,
311 default_memory_pool(), std::move(input)));
312
313 // Grab the first block and use it to determine the schema and create a reader. The
314 // input->Peek call blocks so we run the whole thing on the I/O thread pool.
315 auto reader_fut = DeferNotOk(input->io_context().executor()->Submit(
316 [=]() -> Future<std::shared_ptr<csv::StreamingReader>> {
317 ARROW_ASSIGN_OR_RAISE(auto first_block, input->Peek(reader_options.block_size));
318 const auto& parse_options = format.parse_options;
319 ARROW_ASSIGN_OR_RAISE(
320 auto convert_options,
321 GetConvertOptions(format, scan_options ? scan_options.get() : nullptr,
322 first_block));
323 return csv::StreamingReader::MakeAsync(io::default_io_context(), std::move(input),
324 cpu_executor, reader_options,
325 parse_options, convert_options);
326 }));
327 return reader_fut.Then(
328 // Adds the filename to the error
329 [=](const std::shared_ptr<csv::StreamingReader>& reader)
330 -> Result<std::shared_ptr<csv::StreamingReader>> {
331#ifdef ARROW_WITH_OPENTELEMETRY
332 span->SetStatus(opentelemetry::trace::StatusCode::kOk);
333 span->End();
334#endif
335 return reader;
336 },
337 [=](const Status& err) -> Result<std::shared_ptr<csv::StreamingReader>> {
338#ifdef ARROW_WITH_OPENTELEMETRY
339 arrow::internal::tracing::MarkSpan(err, span.get());
340 span->End();
341#endif
342 return err.WithMessage("Could not open CSV input source '", path, "': ", err);
343 });
344}
345
346static inline Result<std::shared_ptr<csv::StreamingReader>> OpenReader(
347 const FileSource& source, const CsvFileFormat& format,

Callers 2

OpenReaderFunction · 0.70
ScanBatchesAsyncMethod · 0.70

Calls 10

GetTracerFunction · 0.85
default_memory_poolFunction · 0.85
DeferNotOkFunction · 0.85
MarkSpanFunction · 0.85
ThenMethod · 0.80
ARROW_ASSIGN_OR_RAISEFunction · 0.50
pathMethod · 0.45
executorMethod · 0.45
EndMethod · 0.45
getMethod · 0.45

Tested by

no test coverage detected