| 608 | |
| 609 | template <typename SpecializedOptions, bool IgnoreExtraColumns> |
| 610 | Status ParseSpecialized(const std::vector<std::string_view>& views, bool is_final, |
| 611 | uint32_t* out_size) { |
| 612 | internal::PreferredBulkFilterType<SpecializedOptions> bulk_filter(options_); |
| 613 | |
| 614 | batch_ = DataBatch{batch_.num_cols_}; |
| 615 | values_size_ = 0; |
| 616 | |
| 617 | size_t total_view_length = 0; |
| 618 | block_has_nul_ = false; |
| 619 | for (const auto& view : views) { |
| 620 | total_view_length += view.length(); |
| 621 | if (!block_has_nul_ && !bulk_filter.CanUseOnBlock(view)) { |
| 622 | block_has_nul_ = true; |
| 623 | } |
| 624 | } |
| 625 | if (block_has_nul_) { |
| 626 | // Clear a bulk filter left on by an earlier NUL-free block. |
| 627 | use_bulk_filter_ = false; |
| 628 | } |
| 629 | if (total_view_length > std::numeric_limits<uint32_t>::max()) { |
| 630 | return Status::Invalid("CSV block too large"); |
| 631 | } |
| 632 | |
| 633 | PresizedDataWriter parsed_writer(pool_, static_cast<uint32_t>(total_view_length)); |
| 634 | uint32_t total_parsed_length = 0; |
| 635 | |
| 636 | for (const auto& view : views) { |
| 637 | const char* data = view.data(); |
| 638 | const char* data_end = view.data() + view.length(); |
| 639 | bool finished_parsing = false; |
| 640 | |
| 641 | if (batch_.num_cols_ == -1) { |
| 642 | // Can't presize values when the number of columns is not known, first parse |
| 643 | // a single line |
| 644 | const int32_t rows_in_chunk = 1; |
| 645 | ARROW_ASSIGN_OR_RAISE(auto values_writer, ResizableValueDescWriter::Make(pool_)); |
| 646 | values_writer.Start(parsed_writer); |
| 647 | |
| 648 | RETURN_NOT_OK((ParseChunk<SpecializedOptions, IgnoreExtraColumns>( |
| 649 | &values_writer, &parsed_writer, data, data_end, is_final, rows_in_chunk, |
| 650 | &data, &finished_parsing, bulk_filter))); |
| 651 | if (batch_.num_cols_ == -1) { |
| 652 | return ParseError("Empty CSV file or block: cannot infer number of columns"); |
| 653 | } |
| 654 | } |
| 655 | |
| 656 | while (!finished_parsing && data < data_end && batch_.num_rows_ < max_num_rows_) { |
| 657 | // We know the number of columns, so can presize a values array for |
| 658 | // a given number of rows |
| 659 | DCHECK_GE(batch_.num_cols_, 0); |
| 660 | |
| 661 | int32_t rows_in_chunk; |
| 662 | constexpr int32_t kTargetChunkSize = 32768; // in number of values |
| 663 | if (batch_.num_cols_ > 0) { |
| 664 | rows_in_chunk = std::min(std::max(kTargetChunkSize / batch_.num_cols_, 512), |
| 665 | max_num_rows_ - batch_.num_rows_); |
| 666 | } else { |
| 667 | rows_in_chunk = std::min(kTargetChunkSize, max_num_rows_ - batch_.num_rows_); |
nothing calls this directly
no test coverage detected