Function ingest_args

subprojects/llama.cpp/tools/tokenize/tokenize.cpp:80–119

Function: ingest_args(...) -> vector Takes argc and argv arguments, and converts them to a vector of UTF-8 encoded strings, as an STL vector . In particular, it handles character encoding shenanigans on Windows. Note: raw_argc and raw_argv are not actually read at all on Windows. On Windows we call GetCommandLineW to get the arguments in wchar_t format, ignoring the regular argc/

// TODO: potential opportunity to roll common stuff into common/console.cpp
//       in relation to Windows wchar_t shenanigans.
static std::vector<std::string> ingest_args(int raw_argc, char ** raw_argv) {
    std::vector<std::string> argv;

    // Handle Windows, if given non-ASCII arguments.
    // We convert wchar_t arguments into UTF-8 char* on this platform.
    // Lets you invoke 'tokenize' on Windows cmd.exe with non-ASCII characters
    // without throwing tantrums.
#if defined(_WIN32)
    int argc;
    const LPWSTR cmdline_wargv = GetCommandLineW();
    LPWSTR * wargv = CommandLineToArgvW(cmdline_wargv, &argc);

    // silence unused arg warnings
    (void) raw_argc;
    (void) raw_argv;

    for (int i = 0; i < argc; ++i) {
        int length_needed = WideCharToMultiByte(CP_UTF8, 0, wargv[i], wcslen(wargv[i]), 0, 0, NULL, NULL);
        char * output_buf = (char *) calloc(length_needed+1, sizeof(char));
        GGML_ASSERT(output_buf);

        WideCharToMultiByte(CP_UTF8, 0, wargv[i], wcslen(wargv[i]), output_buf, length_needed, NULL, NULL);
        output_buf[length_needed] = '\0';

        argv.push_back(output_buf);
        free(output_buf);
    }

    LocalFree((HLOCAL) wargv);
#else
    int argc = raw_argc;
    for (int i = 0; i < argc; ++i) {
        argv.push_back(raw_argv[i]);
    }
#endif

    GGML_ASSERT((unsigned int) argc == argv.size());

    return argv;
}

//
// Function: write_utf8_cstr_to_stdout(const char *) -> <writes to stdout>