Function tokenize

lib/codemirror/mode/shell/shell.js:123–125

(stream, state)

  };

  function tokenize(stream, state) {
    return (state.tokens[0] || tokenBase) (stream, state);
  };

  return {
    startState: function() {return {tokens:[]};},