NewTokenizer creates a new tokenizer. Notice: we append an additional eofRune in the statement. This is a sentinel rune.
(statement string)
| 39 | // NewTokenizer creates a new tokenizer. |
| 40 | // Notice: we append an additional eofRune in the statement. This is a sentinel rune. |
| 41 | func NewTokenizer(statement string) *Tokenizer { |
| 42 | t := &Tokenizer{ |
| 43 | buffer: []rune(statement), |
| 44 | cursor: 0, |
| 45 | line: 1, |
| 46 | } |
| 47 | for i := range statement { |
| 48 | t.bufferByteOffset = append(t.bufferByteOffset, i) |
| 49 | } |
| 50 | t.len = uint(len(t.buffer)) |
| 51 | // append an additional eofRune. |
| 52 | t.buffer = append(t.buffer, eofRune) |
| 53 | t.bufferByteOffset = append(t.bufferByteOffset, len(statement)) |
| 54 | |
| 55 | return t |
| 56 | } |
| 57 | |
| 58 | func (t *Tokenizer) SetLineForMySQLCreateTableStmt(node *tidbast.CreateTableStmt, firstLine int) error { |
| 59 | // We assume that the parser will parse the columns and table constraints according to the order of the raw SQL statements |
no outgoing calls
no test coverage detected