| 138 | } |
| 139 | |
| 140 | fn convert_token(source: &str, cursor: Cursor) -> CSSTokenizerTestToken { |
| 141 | let raw = cursor.str_slice(source); |
| 142 | let structured = match cursor.token().kind() { |
| 143 | Kind::Number => Some(Structured::Number(NumberStructured { |
| 144 | value: cursor.token().value(), |
| 145 | kind: Some(String::from(if cursor.token().is_int() { "integer" } else { "number" })), |
| 146 | })), |
| 147 | Kind::Dimension => { |
| 148 | if SourceCursor::from(cursor, raw).parse(Global) == "%" { |
| 149 | Some(Structured::Number(NumberStructured { value: cursor.token().value(), kind: None })) |
| 150 | } else { |
| 151 | Some(Structured::Dimension(DimensionStructured { |
| 152 | value: cursor.token().value(), |
| 153 | unit: SourceCursor::from(cursor, raw).parse(Global).to_owned(), |
| 154 | kind: String::from(if cursor.token().is_int() { "integer" } else { "number" }), |
| 155 | })) |
| 156 | } |
| 157 | } |
| 158 | Kind::Ident | Kind::String | Kind::AtKeyword | Kind::Function | Kind::Url | Kind::Hash => { |
| 159 | Some(Structured::String(StringStructured { |
| 160 | value: SourceCursor::from(cursor, raw).parse(Global).to_owned(), |
| 161 | })) |
| 162 | } |
| 163 | Kind::Delim => Some(Structured::String(StringStructured { value: cursor.token().char().unwrap().to_string() })), |
| 164 | _ => None, |
| 165 | }; |
| 166 | // token.start/end count utf8 bytes because rust strings are utf8. The tokenizer tests, |
| 167 | // however, are JS based and JS strings are utf16. So recalculate the indexes to utf16 |
| 168 | let start_index = source[0..cursor.offset().into()].encode_utf16().count() as u32; |
| 169 | let end_index = start_index + cursor.str_slice(source).encode_utf16().count() as u32; |
| 170 | CSSTokenizerTestToken { |
| 171 | kind: match cursor.token().kind() { |
| 172 | Kind::Comment => CSSTokenizerTestKind::Comment, |
| 173 | Kind::Ident => CSSTokenizerTestKind::Ident, |
| 174 | Kind::Function => CSSTokenizerTestKind::Function, |
| 175 | Kind::AtKeyword => CSSTokenizerTestKind::AtKeyword, |
| 176 | Kind::Hash => CSSTokenizerTestKind::Hash, |
| 177 | Kind::String => CSSTokenizerTestKind::String, |
| 178 | Kind::BadString => CSSTokenizerTestKind::BadString, |
| 179 | Kind::Url => CSSTokenizerTestKind::Url, |
| 180 | Kind::BadUrl => CSSTokenizerTestKind::BadUrl, |
| 181 | Kind::Delim => CSSTokenizerTestKind::Delim, |
| 182 | Kind::Number => CSSTokenizerTestKind::Number, |
| 183 | Kind::Dimension => { |
| 184 | if SourceCursor::from(cursor, raw).parse(Global) == "%" { |
| 185 | CSSTokenizerTestKind::Percentage |
| 186 | } else { |
| 187 | CSSTokenizerTestKind::Dimension |
| 188 | } |
| 189 | } |
| 190 | Kind::Whitespace => CSSTokenizerTestKind::Whitespace, |
| 191 | Kind::CdcOrCdo => { |
| 192 | if cursor.token().is_cdc() { |
| 193 | CSSTokenizerTestKind::Cdc |
| 194 | } else { |
| 195 | CSSTokenizerTestKind::Cdo |
| 196 | } |
| 197 | } |