Function main

subprojects/llama.cpp/tools/tokenize/tokenize.cpp:186–416

}

int main(int raw_argc, char ** raw_argv) {
    const std::vector<std::string> argv = ingest_args(raw_argc, raw_argv);
    const int argc = argv.size();

    if (argc <= 1) {
        print_usage_information(argv[0].c_str());
        return 1;
    }

    //////
    // Read out all the command line arguments.
    //////

    // variables where to put any arguments we see.
    bool printing_ids = false;
    bool no_bos = false;
    bool no_escape = false;
    bool no_parse_special = false;
    bool disable_logging = false;
    bool show_token_count = false;
    const char * model_path = NULL;
    const char * prompt_path = NULL;
    const char * prompt_arg = NULL;

    // track which arguments were explicitly given
    // used for sanity checking down the line
    bool model_path_set = false;
    bool prompt_path_set = false;
    bool prompt_set = false;
    bool stdin_set = false;

    int iarg = 1;
    for (; iarg < argc; ++iarg) {
        std::string arg{argv[iarg]};
        if (arg == "-h" || arg == "--help") {
            print_usage_information(argv[0].c_str());
            return 0;
        }
        else if (arg == "--ids") {
            printing_ids = true;
        }
        else if (arg == "-m" || arg == "--model") {
            if (model_path_set) {
                fprintf(stderr, "Error: -m or --model specified multiple times.\n");
                return 1;
            }
            model_path = argv[++iarg].c_str();
            model_path_set = true;
        }
        else if (arg == "--no-bos") {
            no_bos = true;
        }
        else if (arg == "--no-escape") {
            no_escape = true;
        }
        else if (arg == "--no-parse-special") {
            no_parse_special = true;
        }