Method SplitTextToWords

segmenter.go:211–251

SplitTextToWords splits a string to token words

(text Text)

// SplitTextToWords splits a string to token words
func (seg *Segmenter) SplitTextToWords(text Text) []Text {
	output := make([]Text, 0, len(text)/3)
	current, alphanumericStart := 0, 0
	inAlphanumeric := true

	for current < len(text) {
		r, size := utf8.DecodeRune(text[current:])
		isNum := unicode.IsNumber(r) && !seg.Num
		isAlpha := unicode.IsLetter(r) && !seg.Alpha
		if size <= 2 && (isAlpha || isNum) {
			// Currently is Latin alphabet or numbers (not in CJK)
			if !inAlphanumeric {
				alphanumericStart = current
				inAlphanumeric = true
			}

			if seg.AlphaNum {
				output = append(output, toLow(text[current:current+size]))
			}
		} else {
			if inAlphanumeric {
				inAlphanumeric = false
				if current != 0 && !seg.AlphaNum {
					output = append(output, toLow(text[alphanumericStart:current]))
				}
			}

			output = append(output, text[current:current+size])
		}
		current += size
	}

	// process last byte is alpha and num
	if inAlphanumeric && !seg.AlphaNum {
		if current != 0 {
			output = append(output, toLow(text[alphanumericStart:current]))
		}
	}

	return output
}

func toLow(text []byte) []byte {
	if ToLower {