Function C_tokenize_int

tools/tokenizer/libtoken.c:534–1181

*/

unsigned C_tokenize_int(const char **token, enum TokenClass *type,
			unsigned *line, unsigned *col, unsigned *pos)
{
  int cc;
  *type = ENDOFFILE;

  do { // infinite loop; after token recognized breaks out.
    // Start collecting a token.
    token_buf_reset();
    *line = linenr;
    *col = column;
    *pos = char_count;
    // white-space tokens see continuation lines:
    logical_lines = 0;
    cc = get();

  restart:
    // cc already read; coordinates for it are correct.

    /*** WHITE-SPACE ***/

    /* In principle all consecutive white-space including \n and \r (and some
       other control chars) are collected and form a single whitespace token.
       However, when newlines are requested to be reported as separate tokens,
       they break this pattern. Note that we cannot issues multiple tokens
       in a single call to this function.

       Token buf will only hold some white-space chars when implicitly
       requested via whitespace_token; otherwise stays empty.
       Same for the \n and \r requests.
     */

    if (cc == '\n' && newline_token) { // end of a logical line
      // Here we assume the buf is empty.
      token_buf_push(cc);
      *type = NEWLINE;
      break;
    }

    if (cc == '\r' && continuation_token) { // end of a physical line
      // Here we assume the buf is empty.
      token_buf_push('\\');
      token_buf_push('\n');
      *type = CONTINUATION;
      break;
    }

    // Aggregate as much white-space as possible.
    // FIXME: officially a NUL should be considered white-space.
    while (isspace(cc)) {	// i.e., cc in [ \f\n\r\t\v]
      // Here: !newline_token (!continuation_token)
      if (whitespace_token)
        if (cc == '\r') { // line continuation
          // Convert back to original char sequence:
          token_buf_push('\\');
          token_buf_push('\n');
        }
        else