Function Tokenize

tools/bash/tokenizer.go:52–97

(command string, stripQuotes bool)

var envNameRe = regexp.MustCompile(`^[A-Za-z_]\w*$`)

func Tokenize(command string, stripQuotes bool) []string {
	var tokens []string
	var current []rune
	inSingle := false
	inDouble := false
	i := 0
	runes := []rune(command)
	n := len(runes)

	for i < n {
		ch := runes[i]
		if ch == '\\' && i+1 < n {
			nextCh := runes[i+1]
			if nextCh == ' ' || nextCh == '\t' || nextCh == '\\' || nextCh == '\'' || nextCh == '"' {
				current = append(current, nextCh)
				i += 2
				continue
			}
			i += 1
			current = append(current, ch)
			continue
		} else if ch == '\'' && !inDouble {
			inSingle = !inSingle
			if !stripQuotes {
				current = append(current, ch)
			}
		} else if ch == '"' && !inSingle {
			inDouble = !inDouble
			if !stripQuotes {
				current = append(current, ch)
			}
		} else if (ch == ' ' || ch == '\t') && !inSingle && !inDouble {
			if current != nil && len(current) > 0 {
				tokens = append(tokens, string(current))
				current = []rune{}
			}
		} else {
			current = append(current, ch)
		}
		i += 1
	}
	if current != nil && len(current) > 0 {
		tokens = append(tokens, string(current))
	}
	return tokens
}

func SplitPipeline(command string) []string {
	var segments []string