(t *testing.T)
| 121 | } |
| 122 | |
| 123 | func TestTokenize(t *testing.T) { |
| 124 | testCases := []struct { |
| 125 | input string |
| 126 | tokens []token |
| 127 | }{ |
| 128 | { |
| 129 | input: "ab<dnotehl>c</dnotehl>", |
| 130 | tokens: []token{ |
| 131 | token{ |
| 132 | Kind: tokenKindChar, |
| 133 | Value: 'a', |
| 134 | }, |
| 135 | token{ |
| 136 | Kind: tokenKindChar, |
| 137 | Value: 'b', |
| 138 | }, |
| 139 | token{ |
| 140 | Kind: tokenKindHLBegin, |
| 141 | }, |
| 142 | token{ |
| 143 | Kind: tokenKindChar, |
| 144 | Value: 'c', |
| 145 | }, |
| 146 | token{ |
| 147 | Kind: tokenKindHLEnd, |
| 148 | }, |
| 149 | token{ |
| 150 | Kind: tokenKindEOL, |
| 151 | }, |
| 152 | }, |
| 153 | }, |
| 154 | { |
| 155 | input: "ab<dnotehl>c</dnotehl>d", |
| 156 | tokens: []token{ |
| 157 | token{ |
| 158 | Kind: tokenKindChar, |
| 159 | Value: 'a', |
| 160 | }, |
| 161 | token{ |
| 162 | Kind: tokenKindChar, |
| 163 | Value: 'b', |
| 164 | }, |
| 165 | token{ |
| 166 | Kind: tokenKindHLBegin, |
| 167 | }, |
| 168 | token{ |
| 169 | Kind: tokenKindChar, |
| 170 | Value: 'c', |
| 171 | }, |
| 172 | token{ |
| 173 | Kind: tokenKindHLEnd, |
| 174 | }, |
| 175 | token{ |
| 176 | Kind: tokenKindChar, |
| 177 | Value: 'd', |
| 178 | }, |
| 179 | token{ |
| 180 | Kind: tokenKindEOL, |