MCPcopy Create free account
hub / github.com/apache/arrow / Make

Method Make

cpp/src/arrow/dataset/file_csv.cc:117–140  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

115 }
116
117 static Future<std::shared_ptr<FragmentScanner>> Make(
118 const CsvFragmentScanOptions& csv_options, const FragmentScanRequest& scan_request,
119 const CsvInspectedFragment& inspected_fragment, Executor* cpu_executor) {
120 auto read_options = csv_options.read_options;
121
122 int num_batches = static_cast<int>(bit_util::CeilDiv(
123 inspected_fragment.num_bytes, static_cast<int64_t>(read_options.block_size)));
124 // Could be better, but a reasonable starting point. CSV presumably takes up more
125 // space than an in-memory format so this should be conservative.
126 int64_t best_guess_bytes_per_batch = read_options.block_size;
127 ARROW_ASSIGN_OR_RAISE(
128 csv::ConvertOptions convert_options,
129 GetConvertOptions(csv_options, scan_request, inspected_fragment));
130
131 return csv::StreamingReader::MakeAsync(
132 io::default_io_context(), inspected_fragment.input_stream, cpu_executor,
133 read_options, csv_options.parse_options, convert_options)
134 .Then([num_batches, best_guess_bytes_per_batch](
135 const std::shared_ptr<csv::StreamingReader>& reader)
136 -> std::shared_ptr<FragmentScanner> {
137 return std::make_shared<CsvFileScanner>(reader, num_batches,
138 best_guess_bytes_per_batch);
139 });
140 }
141
142 private:
143 std::shared_ptr<csv::StreamingReader> reader_;

Callers

nothing calls this directly

Calls 2

CeilDivFunction · 0.85
ThenMethod · 0.80

Tested by

no test coverage detected