Function remove_BOM

tools/tokenizer/libtoken.c:262–277

Must be called right after a file is opened as stdin. Will attempt to remove any UTF-8 unicode signature (byte-order mark, BOM) at the beginning of the file. Unicode: U+FEFF UTF-8: EF BB BF First bytes Encoding Must remove? 00 00 FE FF UTF-32 big endian Yes FF FE 00 00 UTF-32 little endian Yes FE FF UTF-16 big endian Yes FF FE UTF-16 li

   otherwise   UTF-8                 No
*/
static void remove_BOM(void)
{
  int c1 = getchar();
  if (c1 == 0xEF) {
    int c2 = getchar();
    if (c2 == 0xBB) {
      int c3 = getchar();
      if (c3 == 0xBF) {
        return;
      }
      if (c3 != EOF) buffer[buffered++] = c3;
    }
    if (c2 != EOF) buffer[buffered++] = c2;
  }
  if (c1 != EOF) buffer[buffered++] = c1;
}

int open_as_stdin(const char *file)
{