Tokens converts the string into tokens
(ln string)
| 304 | |
| 305 | // Tokens converts the string into tokens |
| 306 | func (sh *Shell) Tokens(ln string) Tokens { |
| 307 | fset := token.NewFileSet() |
| 308 | f := fset.AddFile("", fset.Base(), len(ln)) |
| 309 | var sc scanner.Scanner |
| 310 | sc.Init(f, []byte(ln), sh.errHandler, scanner.ScanComments|2) // 2 is non-exported dontInsertSemis |
| 311 | // note to Go team: just export this stuff. seriously. |
| 312 | |
| 313 | var toks Tokens |
| 314 | for { |
| 315 | pos, tok, lit := sc.Scan() |
| 316 | if tok == token.EOF { |
| 317 | break |
| 318 | } |
| 319 | // logx.PrintfDebug(" token: %s\t%s\t%q\n", fset.Position(pos), tok, lit) |
| 320 | toks = append(toks, &Token{Tok: tok, Pos: pos, Str: lit}) |
| 321 | } |
| 322 | return toks |
| 323 | } |
| 324 | |
| 325 | func (sh *Shell) errHandler(pos token.Position, msg string) { |
| 326 | logx.PrintlnDebug("Scan Error:", pos, msg) |