Scan reads and returns the next available token from the Tokenizer's buffer (trimmed!). Empty tokens are skipped if t.keepEmptyTokens is not set (which is the default). Returns [io.EOF] error when there are no more tokens to scan.
()
| 84 | // |
| 85 | // Returns [io.EOF] error when there are no more tokens to scan. |
| 86 | func (t *Tokenizer) Scan() (string, error) { |
| 87 | ch := t.read() |
| 88 | if ch == eof { |
| 89 | return "", io.EOF |
| 90 | } |
| 91 | t.unread() |
| 92 | |
| 93 | token, err := t.readToken() |
| 94 | if err != nil { |
| 95 | return "", err |
| 96 | } |
| 97 | |
| 98 | if !t.keepEmptyTokens && token == "" { |
| 99 | return t.Scan() |
| 100 | } |
| 101 | |
| 102 | return token, err |
| 103 | } |
| 104 | |
| 105 | // ScanAll reads the entire Tokenizer's buffer and return all found tokens. |
| 106 | func (t *Tokenizer) ScanAll() ([]string, error) { |