Method generate_batch

python/cpp/generator.cc:14–79

      using ReplicaPoolHelper::ReplicaPoolHelper;

      std::variant<std::vector<GenerationResult>,
                   std::vector<AsyncResult<GenerationResult>>>
      generate_batch(const BatchTokens& tokens,
                     size_t max_batch_size,
                     const std::string& batch_type_str,
                     bool asynchronous,
                     size_t beam_size,
                     float patience,
                     size_t num_hypotheses,
                     float length_penalty,
                     float repetition_penalty,
                     size_t no_repeat_ngram_size,
                     bool disable_unk,
                     const std::optional<std::vector<std::vector<std::string>>>& suppress_sequences,
                     const std::optional<EndToken>& end_token,
                     bool return_end_token,
                     size_t max_length,
                     size_t min_length,
                     const std::optional<std::vector<std::string>>& static_prompt,
                     bool cache_static_prompt,
                     bool include_prompt_in_result,
                     bool return_scores,
                     bool return_logits_vocab,
                     bool return_alternatives,
                     float min_alternative_expansion_prob,
                     size_t sampling_topk,
                     float sampling_topp,
                     float sampling_temperature,
                     std::function<bool(GenerationStepResult)> callback) {
        if (tokens.empty())
          return {};

        BatchType batch_type = str_to_batch_type(batch_type_str);
        GenerationOptions options;
        options.beam_size = beam_size;
        options.patience = patience;
        options.length_penalty = length_penalty;
        options.repetition_penalty = repetition_penalty;
        options.no_repeat_ngram_size = no_repeat_ngram_size;
        options.disable_unk = disable_unk;
        options.sampling_topk = sampling_topk;
        options.sampling_topp = sampling_topp;
        options.sampling_temperature = sampling_temperature;
        options.max_length = max_length;
        options.min_length = min_length;
        options.num_hypotheses = num_hypotheses;
        options.return_end_token = return_end_token;
        options.return_scores = return_scores;
        options.return_logits_vocab = return_logits_vocab;
        options.return_alternatives = return_alternatives;
        options.cache_static_prompt = cache_static_prompt;
        options.include_prompt_in_result = include_prompt_in_result;
        options.min_alternative_expansion_prob = min_alternative_expansion_prob;
        options.callback = std::move(callback);
        if (suppress_sequences)
          options.suppress_sequences = suppress_sequences.value();
        if (end_token)
          options.end_token = end_token.value();