| 214 | } |
| 215 | |
| 216 | static inline Result<csv::ConvertOptions> GetConvertOptions( |
| 217 | const CsvFileFormat& format, const ScanOptions* scan_options, |
| 218 | const std::string_view first_block) { |
| 219 | ARROW_ASSIGN_OR_RAISE( |
| 220 | auto csv_scan_options, |
| 221 | GetFragmentScanOptions<CsvFragmentScanOptions>( |
| 222 | kCsvTypeName, scan_options, format.default_fragment_scan_options)); |
| 223 | ARROW_ASSIGN_OR_RAISE( |
| 224 | auto column_names, |
| 225 | GetColumnNames(csv_scan_options->read_options, format.parse_options, first_block, |
| 226 | scan_options ? scan_options->pool : default_memory_pool())); |
| 227 | |
| 228 | auto convert_options = csv_scan_options->convert_options; |
| 229 | |
| 230 | if (!scan_options) return convert_options; |
| 231 | |
| 232 | auto field_refs = scan_options->MaterializedFields(); |
| 233 | std::unordered_set<std::string> materialized_fields; |
| 234 | materialized_fields.reserve(field_refs.size()); |
| 235 | // Preprocess field refs. We try to avoid FieldRef::GetFoo here since that's |
| 236 | // quadratic (and this is significant overhead with 1000+ columns) |
| 237 | for (const auto& ref : field_refs) { |
| 238 | if (const std::string* name = ref.name()) { |
| 239 | // Common case |
| 240 | materialized_fields.emplace(*name); |
| 241 | continue; |
| 242 | } |
| 243 | // Currently CSV reader doesn't support reading any nested types, so this |
| 244 | // path shouldn't be hit. However, implement it in the same way as IPC/ORC: |
| 245 | // load the entire top-level field if a nested field is selected. |
| 246 | ARROW_ASSIGN_OR_RAISE(auto field, ref.GetOneOrNone(*scan_options->dataset_schema)); |
| 247 | if (column_names.find(field->name()) == column_names.end()) continue; |
| 248 | // Only read the requested columns |
| 249 | convert_options.include_columns.push_back(field->name()); |
| 250 | // Properly set conversion types |
| 251 | convert_options.column_types[field->name()] = field->type(); |
| 252 | } |
| 253 | |
| 254 | for (auto field : scan_options->dataset_schema->fields()) { |
| 255 | if (materialized_fields.find(field->name()) == materialized_fields.end()) continue; |
| 256 | // Ignore virtual columns. |
| 257 | if (column_names.find(field->name()) == column_names.end()) continue; |
| 258 | // Only read the requested columns |
| 259 | convert_options.include_columns.push_back(field->name()); |
| 260 | // Properly set conversion types |
| 261 | convert_options.column_types[field->name()] = field->type(); |
| 262 | } |
| 263 | return convert_options; |
| 264 | } |
| 265 | |
| 266 | static inline Result<csv::ReadOptions> GetReadOptions( |
| 267 | const CsvFileFormat& format, const std::shared_ptr<ScanOptions>& scan_options) { |