Returns the indices of the tokens in the log probs that correspond to the tokens in the log_prob_range.
(self, offsets, log_prob_range)
| 238 | return log_probs, tokens |
| 239 | |
| 240 | def get_token_indices(self, offsets, log_prob_range): |
| 241 | """Returns the indices of the tokens in the log probs that correspond to the tokens in the log_prob_range.""" |
| 242 | # For the lower index, find the highest index that is less than or equal to the lower index |
| 243 | lower_index = 0 |
| 244 | for i in range(len(offsets)): |
| 245 | if offsets[i] <= log_prob_range[0]: |
| 246 | lower_index = i |
| 247 | else: |
| 248 | break |
| 249 | |
| 250 | upper_index = len(offsets) |
| 251 | for i in range(len(offsets)): |
| 252 | if offsets[i] >= log_prob_range[1]: |
| 253 | upper_index = i |
| 254 | break |
| 255 | |
| 256 | return lower_index, upper_index |
| 257 | |
| 258 | |
| 259 | class GPT_Insert(LLM): |