Creates a new token.
(id: LeafId, kind: TokenKind, content: Vec<u8>, state: LeafState)
| 85 | impl Token { |
| 86 | /// Creates a new token. |
| 87 | pub fn new(id: LeafId, kind: TokenKind, content: Vec<u8>, state: LeafState) -> Self { |
| 88 | Self { |
| 89 | id, |
| 90 | kind, |
| 91 | content, |
| 92 | state, |
| 93 | } |
| 94 | } |
| 95 | |
| 96 | /// Returns the token content as a string (lossy UTF-8 conversion). |
| 97 | pub fn content_str(&self) -> String { |