Function tokenize_string

nanofi/src/core/string_utils.c:126–155

}

token_list tokenize_string(const char * begin, char delim) {
    token_list tks;
    memset(&tks, 0, sizeof(struct token_list));

    if (!begin) return tks;

    const char * end = NULL;

    while ((end = strchr(begin, delim))) {
        if (begin == end) {
            begin++;
            tks.total_bytes++;
            continue;
        }
        size_t len = end - begin;
        add_token_to_list(&tks, begin, len);
        tks.total_bytes++;
        begin = (end+1);
    }

    if (begin && *begin != '\0') {
        size_t len = strlen(begin);
        if (len < MAX_BYTES_READ) {
            tks.has_non_delimited_token = 1;
        }
        add_token_to_list(&tks, begin, len);
    }

    return tks;
}

token_list tokenize_string_tailfile(const char * str, char delim) {
    token_list tks = tokenize_string(str, delim);