Function tokenize

tools/tokenizer/pytokenize.c:229–712

Tokenization of Python programming language source text. Returns 1 when a next valid token is recognized. Then the token is made avaliable in the NUL-terminated string `token`, and its token class is indicated via `type` and its location in the source via `line` and `col`. Returns 0 upon EOF or error. */

   Returns 0 upon EOF or error.
*/
static int tokenize(char *token, const char **type,
                    unsigned *line, unsigned *col)
{
  unsigned len;
  int cc;
  *type = "unknown";

  do { // infinite loop; after token recognized breaks out.
    len = 0;
    cc = get();

  restart:
    // cc already read.

    /*** WHITE-SPACE ***/

    /* Can have [ \t\f]* inbetween tokens.
       May have indentation \f?[ \t]* at beginning of logical line.
       Tokens (except strings) cannot extend beyond a physical line
       and may not be interrupted by a line continuation.

       \n signals completion of a logical line
       \r signals a line joiner and generally will be ignored
       blank lines do not generate NEWLINE tokens
    */

    if (strchr(" \t\f", cc)) {
      cc = process_ws(cc);
      goto restart;
    }

    if (cc == '\n') {
      prev_was_newline = 1;
      cc = get();
      // Maybe EOF!
      if (!brackets_opened && !strchr(" \t\n#\r\f", cc))
        process_newline(0);
      goto restart;
    }

    if (cc == '\r') {
      cc = get();
      goto restart;
    }

    // cc !in [ \t\n\r\f], maybe EOF
    if (cc == EOF) {
      // Undo any outstanding indents:
      while (!indents_empty()) {
        emit("DEDENT", linenr, column);
        indents_pop();
      }
      return 0;
    }

    /*** LINE COMMENT ***/

    if (cc == '#') {