MCPcopy Create free account
hub / github.com/apache/arrow / ParseSpecialized

Method ParseSpecialized

cpp/src/arrow/csv/parser.cc:610–725  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

608
609 template <typename SpecializedOptions, bool IgnoreExtraColumns>
610 Status ParseSpecialized(const std::vector<std::string_view>& views, bool is_final,
611 uint32_t* out_size) {
612 internal::PreferredBulkFilterType<SpecializedOptions> bulk_filter(options_);
613
614 batch_ = DataBatch{batch_.num_cols_};
615 values_size_ = 0;
616
617 size_t total_view_length = 0;
618 block_has_nul_ = false;
619 for (const auto& view : views) {
620 total_view_length += view.length();
621 if (!block_has_nul_ && !bulk_filter.CanUseOnBlock(view)) {
622 block_has_nul_ = true;
623 }
624 }
625 if (block_has_nul_) {
626 // Clear a bulk filter left on by an earlier NUL-free block.
627 use_bulk_filter_ = false;
628 }
629 if (total_view_length > std::numeric_limits<uint32_t>::max()) {
630 return Status::Invalid("CSV block too large");
631 }
632
633 PresizedDataWriter parsed_writer(pool_, static_cast<uint32_t>(total_view_length));
634 uint32_t total_parsed_length = 0;
635
636 for (const auto& view : views) {
637 const char* data = view.data();
638 const char* data_end = view.data() + view.length();
639 bool finished_parsing = false;
640
641 if (batch_.num_cols_ == -1) {
642 // Can't presize values when the number of columns is not known, first parse
643 // a single line
644 const int32_t rows_in_chunk = 1;
645 ARROW_ASSIGN_OR_RAISE(auto values_writer, ResizableValueDescWriter::Make(pool_));
646 values_writer.Start(parsed_writer);
647
648 RETURN_NOT_OK((ParseChunk<SpecializedOptions, IgnoreExtraColumns>(
649 &values_writer, &parsed_writer, data, data_end, is_final, rows_in_chunk,
650 &data, &finished_parsing, bulk_filter)));
651 if (batch_.num_cols_ == -1) {
652 return ParseError("Empty CSV file or block: cannot infer number of columns");
653 }
654 }
655
656 while (!finished_parsing && data < data_end && batch_.num_rows_ < max_num_rows_) {
657 // We know the number of columns, so can presize a values array for
658 // a given number of rows
659 DCHECK_GE(batch_.num_cols_, 0);
660
661 int32_t rows_in_chunk;
662 constexpr int32_t kTargetChunkSize = 32768; // in number of values
663 if (batch_.num_cols_ > 0) {
664 rows_in_chunk = std::min(std::max(kTargetChunkSize / batch_.num_cols_, 512),
665 max_num_rows_ - batch_.num_rows_);
666 } else {
667 rows_in_chunk = std::min(kTargetChunkSize, max_num_rows_ - batch_.num_rows_);

Callers

nothing calls this directly

Calls 10

backMethod · 0.80
ParseErrorFunction · 0.70
InvalidFunction · 0.50
OKFunction · 0.50
lengthMethod · 0.45
CanUseOnBlockMethod · 0.45
dataMethod · 0.45
StartMethod · 0.45
FinishMethod · 0.45
sizeMethod · 0.45

Tested by

no test coverage detected