| 554 | |
| 555 | template <typename SpecializedOptions> |
| 556 | Status ParseSpecialized(const std::vector<std::string_view>& views, bool is_final, |
| 557 | uint32_t* out_size) { |
| 558 | internal::PreferredBulkFilterType<SpecializedOptions> bulk_filter(options_); |
| 559 | |
| 560 | batch_ = DataBatch{batch_.num_cols_}; |
| 561 | values_size_ = 0; |
| 562 | |
| 563 | size_t total_view_length = 0; |
| 564 | for (const auto& view : views) { |
| 565 | total_view_length += view.length(); |
| 566 | } |
| 567 | if (total_view_length > std::numeric_limits<uint32_t>::max()) { |
| 568 | return Status::Invalid("CSV block too large"); |
| 569 | } |
| 570 | |
| 571 | PresizedDataWriter parsed_writer(pool_, static_cast<uint32_t>(total_view_length)); |
| 572 | uint32_t total_parsed_length = 0; |
| 573 | |
| 574 | for (const auto& view : views) { |
| 575 | const char* data = view.data(); |
| 576 | const char* data_end = view.data() + view.length(); |
| 577 | bool finished_parsing = false; |
| 578 | |
| 579 | if (batch_.num_cols_ == -1) { |
| 580 | // Can't presize values when the number of columns is not known, first parse |
| 581 | // a single line |
| 582 | const int32_t rows_in_chunk = 1; |
| 583 | ARROW_ASSIGN_OR_RAISE(auto values_writer, ResizableValueDescWriter::Make(pool_)); |
| 584 | values_writer.Start(parsed_writer); |
| 585 | |
| 586 | RETURN_NOT_OK(ParseChunk<SpecializedOptions>( |
| 587 | &values_writer, &parsed_writer, data, data_end, is_final, rows_in_chunk, |
| 588 | &data, &finished_parsing, bulk_filter)); |
| 589 | if (batch_.num_cols_ == -1) { |
| 590 | return ParseError("Empty CSV file or block: cannot infer number of columns"); |
| 591 | } |
| 592 | } |
| 593 | |
| 594 | while (!finished_parsing && data < data_end && batch_.num_rows_ < max_num_rows_) { |
| 595 | // We know the number of columns, so can presize a values array for |
| 596 | // a given number of rows |
| 597 | DCHECK_GE(batch_.num_cols_, 0); |
| 598 | |
| 599 | int32_t rows_in_chunk; |
| 600 | constexpr int32_t kTargetChunkSize = 32768; // in number of values |
| 601 | if (batch_.num_cols_ > 0) { |
| 602 | rows_in_chunk = std::min(std::max(kTargetChunkSize / batch_.num_cols_, 512), |
| 603 | max_num_rows_ - batch_.num_rows_); |
| 604 | } else { |
| 605 | rows_in_chunk = std::min(kTargetChunkSize, max_num_rows_ - batch_.num_rows_); |
| 606 | } |
| 607 | |
| 608 | // The values array holds one ParsedValueDesc per cell and those offsets |
| 609 | // are 31-bit, so the number of values in a chunk must fit in an int32. |
| 610 | // A first line with millions of fields can drive `num_cols_` high enough |
| 611 | // to overflow that, so error out rather than presize past the limit. |
| 612 | if (static_cast<int64_t>(rows_in_chunk) * batch_.num_cols_ > |
| 613 | std::numeric_limits<int32_t>::max()) { |