Text generation for incremental inference Inputs: model: the model for inferencing origin_inputs: the original inputs based on which the model will continue writing config: inference configurations Returns: outputs: the ids for the generated text
(model, origin_inputs, config, verbose=False)
| 168 | |
| 169 | |
| 170 | def generate_increment(model, origin_inputs, config, verbose=False): |
| 171 | """ |
| 172 | Text generation for incremental inference |
| 173 | |
| 174 | Inputs: |
| 175 | model: the model for inferencing |
| 176 | origin_inputs: the original inputs based on which the model will continue writing |
| 177 | config: inference configurations |
| 178 | |
| 179 | Returns: |
| 180 | outputs: the ids for the generated text |
| 181 | """ |
| 182 | # Get configurations for inference |
| 183 | frequency_penalty = config.frequency_penalty |
| 184 | presence_penalty = config.presence_penalty |
| 185 | top_p = config.top_p |
| 186 | top_k_num = config.top_k_num |
| 187 | temperature = config.temperature |
| 188 | max_generate_length = config.max_generate_length |
| 189 | seq_length = config.seq_length |
| 190 | end_token = config.end_token |
| 191 | use_pynative = config.use_pynative_op |
| 192 | vocab_embedding_vocab_size = (config.vocab_size // 1024 + 1) * 1024 |
| 193 | |
| 194 | _, valid_length = origin_inputs.shape |
| 195 | # Init outputs with original inputs |
| 196 | outputs = [origin_inputs[0][i] for i in range(valid_length)] |
| 197 | # If target length exceeds seq_length, use seq_length instead |
| 198 | target_length = valid_length + max_generate_length |
| 199 | target_length = seq_length if target_length > seq_length else target_length |
| 200 | |
| 201 | # A list of the frequency of each token |
| 202 | frequency_list = np.array([[0 for _ in range(vocab_embedding_vocab_size)]]) |
| 203 | pad_length = seq_length - origin_inputs.shape[-1] |
| 204 | # Pad original inputs to seq_length |
| 205 | input_ids = np.pad(origin_inputs, ((0, 0), (0, pad_length)), |
| 206 | 'constant', constant_values=(end_token, end_token)) |
| 207 | print("input_ids is ", input_ids) |
| 208 | |
| 209 | # Indicate the exact token position |
| 210 | current_index = valid_length - 1 if valid_length - 1 > 0 else 0 |
| 211 | batch_valid_length = Tensor(np.array([current_index]), mstype.int32) |
| 212 | current_index = Tensor(np.array([current_index]), mstype.int32) |
| 213 | # For first graph, not_init should be false |
| 214 | init_true = Tensor([True], mstype.bool_) |
| 215 | init_false = Tensor([False], mstype.bool_) |
| 216 | init = init_false |
| 217 | # Claim the first graph |
| 218 | model.predict_network.add_flags_recursive(is_first_iteration=True) |
| 219 | # Call a single inference with input size of (bs, seq_length) |
| 220 | logits = model.predict(Tensor(input_ids, mstype.int32), |
| 221 | current_index, init, batch_valid_length) |
| 222 | |
| 223 | # Claim the second graph and set not_init to true |
| 224 | init = init_true |
| 225 | model.predict_network.add_flags_recursive(is_first_iteration=False) |
| 226 | |
| 227 | # A single loop generates one token, loop until reaching target seq_length or generating eod token |