| 918 | }; |
| 919 | |
| 920 | void normalize(const std::string& input, std::string * normalized) { |
| 921 | normalized->clear(); |
| 922 | normalized->reserve(input.size() * 3); |
| 923 | |
| 924 | const std::string space = vocab.get_escape_whitespaces() ? tokenizer.escaped_space : " "; |
| 925 | |
| 926 | const bool shall_prepend_space = !vocab.get_treat_whitespace_as_suffix() && vocab.get_add_space_prefix(); |
| 927 | const bool shall_append_space = vocab.get_treat_whitespace_as_suffix() && vocab.get_add_space_prefix(); |
| 928 | const bool shall_merge_spaces = vocab.get_remove_extra_whitespaces(); |
| 929 | |
| 930 | bool is_space_prepended = false; |
| 931 | bool processing_non_ws = false; |
| 932 | |
| 933 | size_t input_len = input.size(); |
| 934 | |
| 935 | for (size_t input_offset = 0; input_offset < input_len; ) { |
| 936 | auto norm_res = normalize_prefix(input, input_offset); |
| 937 | for (size_t i = 0; i < norm_res.normalized_len; i++) { |
| 938 | char c = norm_res.normalized[i]; |
| 939 | if (c != ' ') { |
| 940 | if (!processing_non_ws) { |
| 941 | processing_non_ws = true; |
| 942 | if ((shall_prepend_space && !is_space_prepended) || shall_merge_spaces) { |
| 943 | normalized->append(space); |
| 944 | is_space_prepended = true; |
| 945 | } |
| 946 | } |
| 947 | normalized->push_back(c); |
| 948 | } else { |
| 949 | if (processing_non_ws) { |
| 950 | processing_non_ws = false; |
| 951 | } |
| 952 | if (!shall_merge_spaces) { |
| 953 | normalized->append(space); |
| 954 | } |
| 955 | } |
| 956 | } |
| 957 | |
| 958 | input_offset += norm_res.consumed_input; |
| 959 | } |
| 960 | |
| 961 | if (shall_append_space) { |
| 962 | normalized->append(space); |
| 963 | } |
| 964 | } |
| 965 | |
| 966 | /* |
| 967 | * This structure is a view wrapper for XOR-compressed double array (XCDA) |