| 226 | } |
| 227 | |
| 228 | static inline Result<csv::ConvertOptions> GetConvertOptions( |
| 229 | const CsvFileFormat& format, const ScanOptions* scan_options, |
| 230 | const std::string_view first_block) { |
| 231 | ARROW_ASSIGN_OR_RAISE( |
| 232 | auto csv_scan_options, |
| 233 | GetFragmentScanOptions<CsvFragmentScanOptions>( |
| 234 | kCsvTypeName, scan_options, format.default_fragment_scan_options)); |
| 235 | ARROW_ASSIGN_OR_RAISE( |
| 236 | auto column_names, |
| 237 | GetColumnNames(csv_scan_options->read_options, format.parse_options, first_block, |
| 238 | scan_options ? scan_options->pool : default_memory_pool())); |
| 239 | |
| 240 | auto convert_options = csv_scan_options->convert_options; |
| 241 | |
| 242 | if (!scan_options) return convert_options; |
| 243 | |
| 244 | auto field_refs = scan_options->MaterializedFields(); |
| 245 | std::unordered_set<std::string> materialized_fields; |
| 246 | materialized_fields.reserve(field_refs.size()); |
| 247 | // Preprocess field refs. We try to avoid FieldRef::GetFoo here since that's |
| 248 | // quadratic (and this is significant overhead with 1000+ columns) |
| 249 | for (const auto& ref : field_refs) { |
| 250 | if (const std::string* name = ref.name()) { |
| 251 | // Common case |
| 252 | materialized_fields.emplace(*name); |
| 253 | continue; |
| 254 | } |
| 255 | // Currently CSV reader doesn't support reading any nested types, so this |
| 256 | // path shouldn't be hit. However, implement it in the same way as IPC/ORC: |
| 257 | // load the entire top-level field if a nested field is selected. |
| 258 | ARROW_ASSIGN_OR_RAISE(auto field, ref.GetOneOrNone(*scan_options->dataset_schema)); |
| 259 | if (column_names.find(field->name()) == column_names.end()) continue; |
| 260 | // Only read the requested columns |
| 261 | convert_options.include_columns.push_back(field->name()); |
| 262 | // Properly set conversion types |
| 263 | convert_options.column_types[field->name()] = field->type(); |
| 264 | } |
| 265 | |
| 266 | for (auto field : scan_options->dataset_schema->fields()) { |
| 267 | if (materialized_fields.find(field->name()) == materialized_fields.end()) continue; |
| 268 | // Ignore virtual columns. |
| 269 | if (column_names.find(field->name()) == column_names.end()) continue; |
| 270 | // Only read the requested columns |
| 271 | convert_options.include_columns.push_back(field->name()); |
| 272 | // Properly set conversion types |
| 273 | convert_options.column_types[field->name()] = field->type(); |
| 274 | } |
| 275 | return convert_options; |
| 276 | } |
| 277 | |
| 278 | static inline Result<csv::ReadOptions> GetReadOptions( |
| 279 | const CsvFileFormat& format, const std::shared_ptr<ScanOptions>& scan_options) { |