Function tokenize

mini-code-edit/cm/mode/shell/shell.js:90–92

(stream, state)

  };

  function tokenize(stream, state) {
    return (state.tokens[0] || tokenBase) (stream, state);
  };

  return {
    startState: function() {return {tokens:[]}},