MCPcopy Create free account
hub / github.com/apache/arrow / GetConvertOptions

Function GetConvertOptions

cpp/src/arrow/dataset/file_csv.cc:228–276  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

226}
227
228static inline Result<csv::ConvertOptions> GetConvertOptions(
229 const CsvFileFormat& format, const ScanOptions* scan_options,
230 const std::string_view first_block) {
231 ARROW_ASSIGN_OR_RAISE(
232 auto csv_scan_options,
233 GetFragmentScanOptions<CsvFragmentScanOptions>(
234 kCsvTypeName, scan_options, format.default_fragment_scan_options));
235 ARROW_ASSIGN_OR_RAISE(
236 auto column_names,
237 GetColumnNames(csv_scan_options->read_options, format.parse_options, first_block,
238 scan_options ? scan_options->pool : default_memory_pool()));
239
240 auto convert_options = csv_scan_options->convert_options;
241
242 if (!scan_options) return convert_options;
243
244 auto field_refs = scan_options->MaterializedFields();
245 std::unordered_set<std::string> materialized_fields;
246 materialized_fields.reserve(field_refs.size());
247 // Preprocess field refs. We try to avoid FieldRef::GetFoo here since that's
248 // quadratic (and this is significant overhead with 1000+ columns)
249 for (const auto& ref : field_refs) {
250 if (const std::string* name = ref.name()) {
251 // Common case
252 materialized_fields.emplace(*name);
253 continue;
254 }
255 // Currently CSV reader doesn't support reading any nested types, so this
256 // path shouldn't be hit. However, implement it in the same way as IPC/ORC:
257 // load the entire top-level field if a nested field is selected.
258 ARROW_ASSIGN_OR_RAISE(auto field, ref.GetOneOrNone(*scan_options->dataset_schema));
259 if (column_names.find(field->name()) == column_names.end()) continue;
260 // Only read the requested columns
261 convert_options.include_columns.push_back(field->name());
262 // Properly set conversion types
263 convert_options.column_types[field->name()] = field->type();
264 }
265
266 for (auto field : scan_options->dataset_schema->fields()) {
267 if (materialized_fields.find(field->name()) == materialized_fields.end()) continue;
268 // Ignore virtual columns.
269 if (column_names.find(field->name()) == column_names.end()) continue;
270 // Only read the requested columns
271 convert_options.include_columns.push_back(field->name());
272 // Properly set conversion types
273 convert_options.column_types[field->name()] = field->type();
274 }
275 return convert_options;
276}
277
278static inline Result<csv::ReadOptions> GetReadOptions(
279 const CsvFileFormat& format, const std::shared_ptr<ScanOptions>& scan_options) {

Callers

nothing calls this directly

Calls 9

MaterializedFieldsMethod · 0.80
push_backMethod · 0.80
reserveMethod · 0.45
sizeMethod · 0.45
nameMethod · 0.45
findMethod · 0.45
endMethod · 0.45
typeMethod · 0.45
fieldsMethod · 0.45

Tested by

no test coverage detected