Function tokenize

src/lexing.cpp:42–68

}

std::vector<std::string_view>
tokenize(const char* start, const char* end, const std::vector<lexer>& lexers)
{
    std::vector<std::string_view> result;
    while(start != end)
    {
        bool error = true;
        for(const auto& l : lexers)
        {
            const auto* next = l(start, end);
            if(next != start)
            {
                result.emplace_back(start, next - start);
                start = next;
                error = false;
                break;
            }
        }

        if(error)
        {
            MIGRAPHX_THROW("TOKENIZE: no token found!");
        }
    }

    return result;
}

} // namespace MIGRAPHX_INLINE_NS
} // namespace migraphx