Function tokenize

tools/tokenizer/jstokenize.c:81–603

Tokenization of JavaScript programming language source text. Returns 1 when a next valid token is recognized. Then the token is made avaliable in the NUL-terminated string `token`, and its token class is indicated via `type` and its start location in the source via `line` and `col`. Returns 0 upon EOF or error. */

   Returns 0 upon EOF or error.
*/
static int tokenize(char *token, const char **type,
                    unsigned *line, unsigned *col)
{
  unsigned len;
  int cc;
  *type = "unknown";

  do { // infinite loop; after token recognized breaks out.
    len = 0;
    cc = get();

  restart:
    // cc already read.

    /*** WHITE-SPACE ***/

    while (strchr(" \t\n\r\f\v", cc))
      cc = get();
    // cc !in [ \t\n\r\f\v], maybe EOF
    if (cc == EOF)
      return 0;

    /*** LINE COMMENT AND BLOCK COMMENT ***/

    if (cc == '/') {
      cc = get();
      if (cc == '/') {
        // Skip till end-of-line (\n exclusive):
        while ((cc = get()) != EOF && cc != '\n' && cc != '\r')
          ;
        // cc == '\n' || cc == '\r' || cc == EOF
        if (cc == '\r') {
          if (!nowarn)
            fprintf(stderr,
                    "(W): Unexpected continuation in line comment.\n");
          // Effectively ignore any \ and terminate logical line:
          cc == '\n';
        }
        goto restart;
      }

      if (cc == '*') {
        // Remember start position:
        unsigned lin = linenr;

        // Skip till */ inclusive:
        int nc = get(); // if EOF next get will be EOF too
        do {
          cc = nc;
          nc = get();
          if (nc == EOF) { // Error!
            fprintf(stderr,
                    "(E): [%s:%u] Unexpected end-of-file in /* comment.\n",
                    filename, lin);
            unexpect_eof++;
            return 0;
          }
        } while (cc != '*' || nc != '/');