Function remove_BOM

tools/tokenizer/token_common.c:85–100

Must be called right after a file is opened as stdin. Will attempt to remove any UTF-8 unicode signature (byte-order mark, BOM) at the beginning of the file. Unicode: U+FEFF UTF-8: EF BB BF First bytes Encoding Must remove? 00 00 FE FF UTF-32 big endian Yes FF FE 00 00 UTF-32 little endian Yes FE FF UTF-16 big endian Yes FF FE UTF-16 li

   otherwise   UTF-8                 No
*/
void remove_BOM(void)
{
  int c1 = getchar();
  if (c1 == 0xEF) {
    int c2 = getchar();
    if (c2 == 0xBB) {
      int c3 = getchar();
      if (c3 == 0xBF) {
        return;
      }
      if (c3 != EOF) buffer[buffered++] = c3;
    }
    if (c2 != EOF) buffer[buffered++] = c2;
  }
  if (c1 != EOF) buffer[buffered++] = c1;
}

/* Deal with DOS (\r \n) and classic Mac OS (\r) (physical) line endings.
   In case of CR LF skip (but count) the CR and return LF.