Method operator()

r/src/io.cpp:495–593

        n_pending_(0) {}

  arrow::Result<std::shared_ptr<arrow::Buffer>> operator()(
      const std::shared_ptr<arrow::Buffer>& src) {
    // A pre-allocation factor to account for possible data growth when
    // converting to UTF-8.
    constexpr double kOversizeFactor = 1.2;

    arrow::BufferBuilder builder;
    const int64_t initial_size = static_cast<int64_t>(src->size() * kOversizeFactor);
    RETURN_NOT_OK(builder.Reserve(initial_size));

    int64_t out_bytes_left = builder.capacity();
    uint8_t* out_buf = builder.mutable_data();

    int64_t in_bytes_left;
    const uint8_t* in_buf;
    int64_t n_src_bytes_in_pending = 0;

    // There may be a few left over bytes from the last call to iconv.
    // Process these first using the internal buffer (with as many bytes
    // as possible added from src) as the source. This may also result in
    // a partial character left over but will always get us into the src buffer.
    if (n_pending_ > 0) {
      n_src_bytes_in_pending =
          std::min<int64_t>(sizeof(pending_) - n_pending_, src->size());
      memcpy(pending_ + n_pending_, src->data(), n_src_bytes_in_pending);
      in_buf = pending_;
      in_bytes_left = n_pending_ + n_src_bytes_in_pending;

      iconv_->iconv(&in_buf, &in_bytes_left, &out_buf, &out_bytes_left);

      // Rather than check the error return code (which is often returned
      // in the case of a partial character at the end of the pending_
      // buffer), check that we have read enough characters to get into
      // `src` (after which the loop below will error for invalid characters).
      int64_t bytes_read_in = in_buf - pending_;
      if (bytes_read_in < n_pending_) {
        return StatusInvalidInput();
      }

      int64_t bytes_read_out = out_buf - builder.mutable_data();
      builder.UnsafeAdvance(bytes_read_out);

      int64_t chars_read_in = n_pending_ + n_src_bytes_in_pending - in_bytes_left;
      in_buf = src->data() + chars_read_in - n_pending_;
      in_bytes_left = src->size() + n_pending_ - chars_read_in;
    } else {
      in_buf = src->data();
      in_bytes_left = src->size();
    }

    // Try to call iconv() as many times as we need, potentially enlarging
    // the output buffer as needed. When zero bytes are appended, the loop
    // will either error (if there are more than 4 bytes left) or copy the
    // bytes to pending_ and wait for more input. We use 4 bytes because
    // this is the maximum number of bytes per complete character in UTF-8,
    // UTF-16, and UTF-32.
    while (in_bytes_left > 0) {
      // Make enough place in the output to hopefully consume all of the input.