| 96 | } |
| 97 | |
| 98 | StorageView |
| 99 | forward_batch(const std::variant<BatchTokens, BatchIds, StorageView>& inputs, |
| 100 | const std::optional<StorageView>& lengths, |
| 101 | const bool return_log_probs) { |
| 102 | std::future<StorageView> future; |
| 103 | |
| 104 | switch (inputs.index()) { |
| 105 | case 0: |
| 106 | future = _pool->forward_batch_async(std::get<BatchTokens>(inputs), return_log_probs); |
| 107 | break; |
| 108 | case 1: |
| 109 | future = _pool->forward_batch_async(std::get<BatchIds>(inputs), return_log_probs); |
| 110 | break; |
| 111 | case 2: |
| 112 | if (!lengths) |
| 113 | throw std::invalid_argument("lengths vector is required when passing a dense input"); |
| 114 | future = _pool->forward_batch_async(std::get<StorageView>(inputs), |
| 115 | lengths.value(), |
| 116 | return_log_probs); |
| 117 | break; |
| 118 | } |
| 119 | |
| 120 | return future.get(); |
| 121 | } |
| 122 | }; |
| 123 | |
| 124 |
nothing calls this directly
no test coverage detected