Function BuildSegmentsFromLengths

src/PluginSegmentation.cpp:250–285

}

SegmentsPtr BuildSegmentsFromLengths(std::string_view text,
                                     const opencc_segment_length_array_t& lengths) {
  SegmentsPtr segments(new Segments);
  const char* bytes = text.data();
  const size_t inputSize = text.size();
  size_t byteOffset = 0;

  for (size_t i = 0; i < lengths.segment_count; i++) {
    const uint32_t codepointLength = lengths.codepoint_lengths[i];
    if (codepointLength == 0) {
      throw Exception("Segmentation plugin returned a zero-length segment.");
    }

    const size_t segmentStart = byteOffset;
    for (uint32_t j = 0; j < codepointLength; j++) {
      if (byteOffset >= inputSize) {
        throw Exception("Segmentation plugin returned lengths past end of input.");
      }
      const size_t charLength = UTF8Util::NextCharLengthNoException(bytes + byteOffset);
      if (charLength == 0 || byteOffset + charLength > inputSize) {
        throw Exception("Input text is not valid UTF-8.");
      }
      byteOffset += charLength;
    }

    segments->AddSegment(text.substr(segmentStart, byteOffset - segmentStart));
  }

  if (byteOffset != inputSize) {
    throw Exception("Segmentation plugin lengths do not cover the full input. "
                    "Consumed bytes: " + std::to_string(byteOffset) +
                    ", total bytes: " + std::to_string(inputSize) +
                    ", segments: " + std::to_string(lengths.segment_count) + ".");
  }
  return segments;
}

class PluginSegmentationAdapter : public Segmentation {
public: