Method tokenize

shells/tokenize.go:125–199

http://man7.org/linux/man-pages/man1/bash.1.html#QUOTING

()

// http://man7.org/linux/man-pages/man1/bash.1.html#QUOTING
func (t *tokenizer) tokenize() error {
	for {
		c, err := t.reader.ReadByte()
		if err == io.EOF {
			break
		} else if err != nil {
			// If we are working with strings we don't expect any exceptions here and can panic if the error occurred.
			// But in the future we might want to use other ByteReaders so we handle error in ordinary way.
			return err
		}
		switch c {
		case '\\':
			c, err = t.reader.ReadByte()
			if err == io.EOF {
				t.broken = true
				t.emitWord()
				return nil
			} else if err != nil {
				return err
			}
			if c == '\n' {
				// ignore it
			} else {
				t.builder.WriteByte(c)
			}
		case '\n':
			t.emitWord()
			t.builder.WriteByte(c)
			t.emitWord()
		case '\'':
			err = t.parseSingleQuote()
			if err == io.EOF {
				return nil
			} else if err != nil {
				return err
			}
		case '"':
			err = t.parseDoubleQuote()
			if err == io.EOF {
				return nil
			} else if err != nil {
				return err
			}
		case ' ', '\t':
			t.emitWord()
		case '$':
			c, err = t.reader.ReadByte()
			switch {
			case err == io.EOF:
				t.builder.WriteByte('$')
			case err != nil:
				return err
			case c == '\'' || c == '"':
				err = t.parseAsciiString(c)
				if err != nil {
					return err
				}
			default: