Text generation Inputs: model: the model for inferencing origin_inputs: the original inputs based on which the model will continue writing config: inference configurations Returns: outputs: the ids for the generated text
(model, origin_inputs, config, verbose=False)
| 82 | |
| 83 | |
| 84 | def generate(model, origin_inputs, config, verbose=False): |
| 85 | """ |
| 86 | Text generation |
| 87 | |
| 88 | Inputs: |
| 89 | model: the model for inferencing |
| 90 | origin_inputs: the original inputs based on which the model will continue writing |
| 91 | config: inference configurations |
| 92 | |
| 93 | Returns: |
| 94 | outputs: the ids for the generated text |
| 95 | """ |
| 96 | # Get configurations for inference |
| 97 | frequency_penalty = config.frequency_penalty |
| 98 | presence_penalty = config.presence_penalty |
| 99 | top_p = config.top_p |
| 100 | top_k_num = config.top_k_num |
| 101 | temperature = config.temperature |
| 102 | max_generate_length = config.max_generate_length |
| 103 | seq_length = config.seq_length |
| 104 | end_token = config.end_token |
| 105 | use_pynative = config.use_pynative_op |
| 106 | vocab_embedding_vocab_size = (config.vocab_size // 1024 + 1) * 1024 |
| 107 | |
| 108 | _, valid_length = origin_inputs.shape |
| 109 | if verbose: |
| 110 | print("Original input shape", origin_inputs.shape) |
| 111 | |
| 112 | # If target length exceeds seq_length, use seq_length instead |
| 113 | target_length = valid_length + max_generate_length |
| 114 | target_length = seq_length if target_length > seq_length else target_length |
| 115 | |
| 116 | # A list of the frequency of each token |
| 117 | frequency_list = np.array([[0 for _ in range(vocab_embedding_vocab_size)]]) |
| 118 | pad_length = seq_length - origin_inputs.shape[-1] |
| 119 | # Pad original inputs to seq_length |
| 120 | print("Original shape:", origin_inputs.shape) |
| 121 | input_ids = np.pad(origin_inputs, ((0, 0), (0, pad_length)), |
| 122 | 'constant', constant_values=(end_token, end_token)) |
| 123 | # print("input_ids is ", input_ids) |
| 124 | |
| 125 | # A single loop generates one token, loop until reaching target seq_length or generating eod token |
| 126 | while valid_length < target_length: |
| 127 | inputs = Tensor(input_ids, mstype.int32) |
| 128 | |
| 129 | # Indicate the exact token position |
| 130 | current_index = valid_length - 1 if valid_length - 1 > 0 else 0 |
| 131 | current_index = Tensor([current_index], mstype.int32) |
| 132 | # Call a single inference |
| 133 | log_probs = model.predict(inputs, current_index) |
| 134 | # Get the revised log_probs considering frequency and presence penalty to eliminate duplicate in generated results |
| 135 | log_probs = log_probs.asnumpy().reshape(1, -1) |
| 136 | log_probs_revised = log_probs - frequency_list * \ |
| 137 | frequency_penalty - (frequency_list > 0) * presence_penalty |
| 138 | log_probs_revised /= temperature |
| 139 | |
| 140 | p, p_args = sampler(log_probs_revised, top_p, top_k_num, use_pynative) |
| 141 | # Random select a token as final output for this round |