| 95 | } |
| 96 | |
| 97 | dim_t get_sample_begin(const dim_t batch_size, |
| 98 | const dim_t batch_id, |
| 99 | const std::vector<dim_t>& batch_offset, |
| 100 | const std::vector<std::vector<size_t>>* prefix) const { |
| 101 | return prefix ? prefix->at(get_batch_index(batch_size, batch_id, batch_offset)).size() : 0; |
| 102 | } |
| 103 | }; |
| 104 | |
| 105 | // Apply a penalty to the score of previously generated tokens. |