Function mark_utf8_units

common/train.cpp:806–819

mark each byte with its utf8 unit number. returns the number of utf8 characters. e.g. when bytes == '\x61\xD0\xB0\x62', then utf8_units will become [0,0,1,0] utf8_nunits will become [1,2,2,1] and 3 is returned. bytes where utf8_units is zero, are the begin of an utf8 character.

// utf8_nunits will become [1,2,2,1] and 3 is returned.
// bytes where utf8_units is zero, are the begin of an utf8 character.
static size_t mark_utf8_units(const char* bytes, int * utf8_units, int * utf8_nunits, size_t count) {
    size_t offs = 0;
    size_t count_utf8 = 0;
    while(offs < count) {
        int len = (int) utf8_len(bytes[offs]);
        for (int i=0; i<len; ++i) {
            utf8_units[offs+i]  = i;
            utf8_nunits[offs+i] = len;
        }
        offs += len;
        ++count_utf8;
    }
    return count_utf8;
}

size_t tokenize_file(
        struct llama_context     * lctx,