| 94 | } |
| 95 | |
| 96 | StorageView |
| 97 | forward_batch(const std::variant<BatchTokens, BatchIds, StorageView>& inputs, |
| 98 | const std::optional<StorageView>& lengths, |
| 99 | const bool return_log_probs) { |
| 100 | std::future<StorageView> future; |
| 101 | |
| 102 | switch (inputs.index()) { |
| 103 | case 0: |
| 104 | future = _pool->forward_batch_async(std::get<BatchTokens>(inputs), return_log_probs); |
| 105 | break; |
| 106 | case 1: |
| 107 | future = _pool->forward_batch_async(std::get<BatchIds>(inputs), return_log_probs); |
| 108 | break; |
| 109 | case 2: |
| 110 | if (!lengths) |
| 111 | throw std::invalid_argument("lengths vector is required when passing a dense input"); |
| 112 | future = _pool->forward_batch_async(std::get<StorageView>(inputs), |
| 113 | lengths.value(), |
| 114 | return_log_probs); |
| 115 | break; |
| 116 | } |
| 117 | |
| 118 | return future.get(); |
| 119 | } |
| 120 | }; |
| 121 | |
| 122 |
nothing calls this directly
no test coverage detected