Detokenize the generated tokens. Args: tokens_gpu_tensor (torch.Tensor): The generated tokens as a GPU tensor. lengths_gpu_tensor (torch.Tensor): The lengths of the generated tokens as a GPU tensor. return_segments (bool): Whether to return the tokenized segments or
(tokens_gpu_tensor, lengths_gpu_tensor,
return_segments)
| 22 | |
| 23 | |
| 24 | def detokenize_generations(tokens_gpu_tensor, lengths_gpu_tensor, |
| 25 | return_segments): |
| 26 | """ |
| 27 | Detokenize the generated tokens. |
| 28 | |
| 29 | Args: |
| 30 | tokens_gpu_tensor (torch.Tensor): The generated tokens as a GPU tensor. |
| 31 | lengths_gpu_tensor (torch.Tensor): The lengths of the generated tokens as a GPU tensor. |
| 32 | return_segments (bool): Whether to return the tokenized segments or not. |
| 33 | |
| 34 | Returns: |
| 35 | tuple: A tuple containing the generated tokens, the detokenized generations, |
| 36 | and optionally the tokenized segments. |
| 37 | |
| 38 | """ |
| 39 | |
| 40 | tokenizer = get_tokenizer() |
| 41 | args = get_args() |
| 42 | prompts_plus_generations = [] |
| 43 | if return_segments: |
| 44 | prompts_plus_generations_segments = [] |
| 45 | |
| 46 | tokens = tokens_gpu_tensor.cpu().numpy().tolist() |
| 47 | lengths = lengths_gpu_tensor.cpu().numpy().tolist() |
| 48 | for sequence_tokens, length in zip(tokens, lengths): |
| 49 | sequence_tokens = sequence_tokens[:length] |
| 50 | prompts_plus_generations.append(tokenizer.decode(sequence_tokens)) |
| 51 | if return_segments: |
| 52 | words = [] |
| 53 | for token in sequence_tokens: |
| 54 | if args.tokenizer_type in [ |
| 55 | 'SentencePieceTokenizer', 'GPTSentencePieceTokenizer' |
| 56 | ]: |
| 57 | word = tokenizer.decoder[token] |
| 58 | else: |
| 59 | word = tokenizer.decode(token) |
| 60 | words.append(word) |
| 61 | prompts_plus_generations_segments.append(words) |
| 62 | |
| 63 | if return_segments: |
| 64 | return tokens, prompts_plus_generations, \ |
| 65 | prompts_plus_generations_segments |
| 66 | |
| 67 | return tokens, prompts_plus_generations |
| 68 | |
| 69 | |
| 70 | def tokenize_prompts(prompts=None, |
no test coverage detected