| 863 | |
| 864 | |
| 865 | size_t generate(char * data, size_t desired_size, size_t buffer_size, |
| 866 | UInt64 seed, const char * determinator_data, size_t determinator_size) |
| 867 | { |
| 868 | code_points.resize(params.order); |
| 869 | |
| 870 | char * pos = data; |
| 871 | char * end = data + buffer_size; |
| 872 | |
| 873 | while (pos < end) |
| 874 | { |
| 875 | Table::LookupResult it = {}; |
| 876 | |
| 877 | size_t context_size = params.order; |
| 878 | while (true) |
| 879 | { |
| 880 | it = table.find(hashContext(code_points.data() + code_points.size() - context_size, code_points.data() + code_points.size())); |
| 881 | if (it && it->getMapped().total + it->getMapped().count_end != 0) |
| 882 | break; |
| 883 | |
| 884 | if (context_size == 0) |
| 885 | break; |
| 886 | --context_size; |
| 887 | } |
| 888 | |
| 889 | if (!it) |
| 890 | throw Exception(ErrorCodes::LOGICAL_ERROR, "Logical error in markov model"); |
| 891 | |
| 892 | size_t offset_from_begin_of_string = pos - data; |
| 893 | size_t determinator_sliding_window_size = std::min(params.determinator_sliding_window_size, determinator_size); |
| 894 | |
| 895 | size_t determinator_sliding_window_overflow = offset_from_begin_of_string + determinator_sliding_window_size > determinator_size |
| 896 | ? offset_from_begin_of_string + determinator_sliding_window_size - determinator_size : 0; |
| 897 | |
| 898 | const char * determinator_sliding_window_begin = determinator_data + offset_from_begin_of_string - determinator_sliding_window_overflow; |
| 899 | |
| 900 | SipHash hash; |
| 901 | hash.update(seed); |
| 902 | hash.update(determinator_sliding_window_begin, determinator_sliding_window_size); |
| 903 | hash.update(determinator_sliding_window_overflow); |
| 904 | UInt64 determinator = hash.get64(); |
| 905 | |
| 906 | /// If string is greater than desired_size, increase probability of end. |
| 907 | double end_probability_multiplier = 0; |
| 908 | Int64 num_bytes_after_desired_size = (pos - data) - desired_size; |
| 909 | |
| 910 | if (num_bytes_after_desired_size > 0) |
| 911 | end_probability_multiplier = std::pow(1.25, num_bytes_after_desired_size); |
| 912 | |
| 913 | CodePoint code = it->getMapped().sample(determinator, end_probability_multiplier); |
| 914 | |
| 915 | if (code == END) |
| 916 | break; |
| 917 | |
| 918 | if (num_bytes_after_desired_size > 0) |
| 919 | { |
| 920 | /// Heuristic: break at ASCII non-alnum code point. |
| 921 | /// This allows to be close to desired_size but not break natural looking words. |
| 922 | if (code < 128 && !isAlphaNumericASCII(static_cast<char>(code))) |
nothing calls this directly
no test coverage detected