| 120 | return np.array(results) |
| 121 | |
| 122 | def _get_ppl(self, input: str, max_out_len: int) -> float: |
| 123 | max_num_retries = 0 |
| 124 | if max_out_len is None: |
| 125 | max_out_len = 1 |
| 126 | while max_num_retries < self.retry: |
| 127 | self.wait() |
| 128 | header = {'content-type': 'application/json'} |
| 129 | try: |
| 130 | data = dict(inputs=input, parameters=self.generation_kwargs) |
| 131 | raw_response = requests.post(self.url, |
| 132 | headers=header, |
| 133 | data=json.dumps(data)) |
| 134 | except requests.ConnectionError: |
| 135 | self.logger.error('Got connection error, retrying...') |
| 136 | continue |
| 137 | try: |
| 138 | response = raw_response.json() |
| 139 | |
| 140 | assert ('prompt_token_ids' in response and 'prompt_logprobs' |
| 141 | in response), f'prompt_token_ids and prompt_logprobs \ |
| 142 | must be in the output. \ |
| 143 | Please consider adding \ |
| 144 | --return_all_prompt_logprobs argument \ |
| 145 | when starting lightllm service. Response: {str(response)}' |
| 146 | |
| 147 | prompt_token_ids = response['prompt_token_ids'][1:] |
| 148 | prompt_logprobs = [ |
| 149 | item[1] for item in response['prompt_logprobs'] |
| 150 | ] |
| 151 | logprobs = [ |
| 152 | item[str(token_id)] for token_id, item in zip( |
| 153 | prompt_token_ids, prompt_logprobs) |
| 154 | ] |
| 155 | if len(logprobs) == 0: |
| 156 | return 0.0 |
| 157 | ce_loss = -sum(logprobs) / len(logprobs) |
| 158 | return ce_loss |
| 159 | except requests.JSONDecodeError: |
| 160 | self.logger.error('JsonDecode error, got', |
| 161 | str(raw_response.content)) |
| 162 | max_num_retries += 1 |
| 163 | raise RuntimeError('Calling LightllmAPI failed after retrying for ' |
| 164 | f'{max_num_retries} times. Check the logs for ' |
| 165 | 'details.') |
| 166 | |
| 167 | def wait(self): |
| 168 | """Wait till the next query can be sent. |