| 79 | } |
| 80 | |
| 81 | std::variant<std::vector<ScoringResult>, |
| 82 | std::vector<AsyncResult<ScoringResult>>> |
| 83 | score_batch(const BatchTokens& tokens, |
| 84 | size_t max_batch_size, |
| 85 | const std::string& batch_type_str, |
| 86 | size_t max_input_length, |
| 87 | bool asynchronous) { |
| 88 | const auto batch_type = str_to_batch_type(batch_type_str); |
| 89 | ScoringOptions options; |
| 90 | options.max_input_length = max_input_length; |
| 91 | std::shared_lock lock(_mutex); |
| 92 | assert_model_is_ready(); |
| 93 | |
| 94 | auto futures = _pool->score_batch_async(tokens, options, max_batch_size, batch_type); |
| 95 | return maybe_wait_on_futures(std::move(futures), asynchronous); |
| 96 | } |
| 97 | |
| 98 | StorageView |
| 99 | forward_batch(const std::variant<BatchTokens, BatchIds, StorageView>& inputs, |