MCPcopy Index your code
hub / github.com/stemkoski/stemkoski.github.com / tokenize

Function tokenize

MathBox/mathbox-bundle.js:47308–47614  ·  view source on GitHub ↗
()

Source from the content-addressed store, hash-verified

47306 , ID = 'id'
47307
47308function tokenize() {
47309 var escaped = false
47310 , gathered = []
47311 , state = READY
47312 , data = []
47313 , idx = 0
47314 , stream
47315 , length
47316 , quote
47317 , depth
47318 , lhs
47319 , rhs
47320 , cmp
47321 , c
47322
47323 return stream = through(ondata, onend)
47324
47325 function ondata(chunk) {
47326 data = data.concat(chunk.split(''))
47327 length = data.length
47328
47329 while(idx < length && (c = data[idx++])) {
47330 switch(state) {
47331 case READY: state_ready(); break
47332 case ANY_CHILD: state_any_child(); break
47333 case OPERATION: state_op(); break
47334 case ATTR_START: state_attr_start(); break
47335 case ATTR_COMP: state_attr_compare(); break
47336 case ATTR_END: state_attr_end(); break
47337 case PSEUDOCLASS:
47338 case PSEUDOPSEUDO: state_pseudo(); break
47339 case PSEUDOSTART: state_pseudostart(); break
47340 case ID:
47341 case TAG:
47342 case CLASS: state_gather(); break
47343 }
47344 }
47345
47346 data = data.slice(idx)
47347 }
47348
47349 function onend(chunk) {
47350 if(arguments.length) {
47351 ondata(chunk)
47352 }
47353
47354 if(gathered.length) {
47355 stream.queue(token())
47356 }
47357 }
47358
47359 function state_ready() {
47360 switch(true) {
47361 case '#' === c: state = ID; break
47362 case '.' === c: state = CLASS; break
47363 case ':' === c: state = PSEUDOCLASS; break
47364 case '[' === c: state = ATTR_START; break
47365 case '!' === c: subject(); break

Callers

nothing calls this directly

Calls 1

throughFunction · 0.70

Tested by

no test coverage detected