| 144 | return scores |
| 145 | |
| 146 | def sample_top_p(scores, top_p): |
| 147 | sorted_logits, sorted_indices = jt.sort(scores, descending=False) |
| 148 | cumulative_probs = sorted_logits.softmax(dim=-1).cumsum(dim=-1) |
| 149 | |
| 150 | # Remove tokens with cumulative top_p above the threshold (token with 0 are kept) |
| 151 | sorted_indices_to_remove = cumulative_probs <= (1 - top_p) |
| 152 | |
| 153 | # scatter sorted tensors to original indexing |
| 154 | indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove) |
| 155 | scores = scores.masked_fill(indices_to_remove, -float("Inf")) |
| 156 | |
| 157 | return scores |