Text generation for incremental inference Inputs: model: the model for inferencing origin_inputs: the original inputs based on which the model will continue writing config: inference configurations Returns: outputs: the ids for the generated text
(model, origin_inputs, config, tokenizer, verbose=False)
| 128 | |
| 129 | |
| 130 | def generate_increment(model, origin_inputs, config, tokenizer, verbose=False): |
| 131 | """ |
| 132 | Text generation for incremental inference |
| 133 | |
| 134 | Inputs: |
| 135 | model: the model for inferencing |
| 136 | origin_inputs: the original inputs based on which the model will continue writing |
| 137 | config: inference configurations |
| 138 | |
| 139 | Returns: |
| 140 | outputs: the ids for the generated text |
| 141 | """ |
| 142 | # Get configurations for inference |
| 143 | frequency_penalty = config.frequency_penalty |
| 144 | presence_penalty = config.presence_penalty |
| 145 | top_p = config.top_p |
| 146 | top_k_num = config.top_k_num |
| 147 | temperature = config.temperature |
| 148 | max_generate_length = config.max_generate_length |
| 149 | seq_length = config.seq_length |
| 150 | end_token = config.end_token |
| 151 | use_pynative = config.use_pynative_op |
| 152 | vocab_embedding_vocab_size = (config.vocab_size // 1024 + 1) * 1024 |
| 153 | |
| 154 | batch_size, valid_length = origin_inputs.shape |
| 155 | # Init outputs with original inputs |
| 156 | # outputs = deepcopy(origin_inputs) |
| 157 | outputs = [[origin_inputs[i][j] for j in range(valid_length)] for i in range(batch_size)] |
| 158 | output_codes = ["" for _ in range(batch_size)] |
| 159 | # If target length exceeds seq_length, use seq_length instead |
| 160 | target_length = valid_length + max_generate_length |
| 161 | if verbose: |
| 162 | print("target_length was ", valid_length, " + ", max_generate_length, " = ", target_length) |
| 163 | target_length = seq_length if target_length > seq_length else target_length |
| 164 | if verbose: |
| 165 | print("target_length is ", target_length) |
| 166 | gen_end = [False for _ in range(batch_size)] |
| 167 | allow_comments_next = [True for _ in range(batch_size)] |
| 168 | allow_comments = [True for _ in range(batch_size)] |
| 169 | |
| 170 | # A list of the frequency of each token |
| 171 | frequency_list = np.zeros((batch_size, vocab_embedding_vocab_size)) |
| 172 | pad_length = seq_length - origin_inputs.shape[-1] |
| 173 | # Pad original inputs to seq_length |
| 174 | input_ids = np.pad(origin_inputs, ((0, 0), (0, pad_length)), |
| 175 | 'constant', constant_values=(end_token, end_token)) |
| 176 | if verbose: |
| 177 | print("input_ids is ", input_ids) |
| 178 | |
| 179 | # Indicate the exact token position |
| 180 | current_index = valid_length - 1 if valid_length - 1 > 0 else 0 |
| 181 | batch_valid_length = Tensor(np.array([current_index for _ in range(batch_size)]), mstype.int32) |
| 182 | current_index = Tensor(np.array([current_index + i * seq_length for i in range(batch_size)]), mstype.int32) |
| 183 | # For first graph, not_init should be false |
| 184 | init_true = Tensor([True], mstype.bool_) |
| 185 | init_false = Tensor([False], mstype.bool_) |
| 186 | init = init_false |
| 187 | # Claim the first graph |
nothing calls this directly
no test coverage detected