Function SegmentText

src/MaxMatchSegmentation.cpp:28–71

namespace {

SegmentsPtr SegmentText(const std::shared_ptr<PrefixMatch>& prefixMatch,
                        std::string_view text) {
  SegmentsPtr segments(new Segments);
  if (text.empty()) {
    return segments;
  }

  const char* segStart = text.data();
  size_t segLength = 0;
  auto clearBuffer = [&segments, &segStart, &segLength]() {
    if (segLength > 0) {
      segments->AddSegment(std::string_view(segStart, segLength));
      segLength = 0;
    }
  };
  const char* textEnd = text.data() + text.length();
  for (const char* pstr = text.data(); pstr < textEnd;) {
    size_t remainingLength = textEnd - pstr;
    const PrefixMatchView matched =
        prefixMatch->MatchPrefixView(pstr, remainingLength);
    size_t matchedLength;
    if (!matched.matched) {
      matchedLength =
          UTF8Util::NextIdeographicDescriptionSequenceLength(pstr,
                                                             remainingLength);
      if (matchedLength == 0) {
        matchedLength = UTF8Util::NextCharLength(pstr);
      }
      // Ensure we don't advance beyond the string boundary
      if (matchedLength > remainingLength) {
        matchedLength = remainingLength;
      }
      segLength += matchedLength;
    } else {
      clearBuffer();
      matchedLength = matched.keyLength;
      segments->AddSegment(matched.key);
      segStart = pstr + matchedLength;
    }
    pstr += matchedLength;
  }
  clearBuffer();
  return segments;
}

} // namespace