Function segmentAndStem

keyserver/src/database/search-utils.js:17–36

(message: string)

const { stopwords } = natural;

function segmentAndStem(message: string): string {
  const segmentsIterator = segmenter.segment(message.toLowerCase());

  const stemmedSegments = [];
  for (const segmentObj of segmentsIterator) {
    const { segment } = segmentObj;
    if (segment.match(whiteSpacesRegex) || stopwords.indexOf(segment) !== -1) {
      continue;
    }
    const stemmedSegment = natural.PorterStemmer.stem(segment).replaceAll(
      punctuationRegex,
      '',
    );
    stemmedSegments.push(stemmedSegment);
  }

  return stemmedSegments
    .filter(segment => !segment.match(whiteSpacesRegex))
    .join(' ');
}

async function processMessagesForSearch(
  messages: $ReadOnlyArray<RawMessageInfo | ProcessedForSearchRow>,