For some case, when it involves multiprocessing with multiple gpus, there might be the chance that inputs are different among different gpus. Therefore, we need to sync inputs for rank0. Args: inputs (str): Inputs for each rank.
(self, inputs: str)
| 226 | return token_lens[0] if not is_batched else token_lens |
| 227 | |
| 228 | def sync_inputs(self, inputs: str) -> str: |
| 229 | """For some case, when it involves multiprocessing with multiple gpus, |
| 230 | there might be the chance that inputs are different among different |
| 231 | gpus. Therefore, we need to sync inputs for rank0. |
| 232 | |
| 233 | Args: |
| 234 | inputs (str): Inputs for each rank. |
| 235 | """ |
| 236 | rank = dist.get_rank() |
| 237 | |
| 238 | if rank == 0: |
| 239 | tokens = self.encode(inputs) |
| 240 | length = self.get_token_len(inputs) |
| 241 | if length > 2048: |
| 242 | from opencompass.utils import get_logger |
| 243 | get_logger().info(f'Large tokens nums: {length}') |
| 244 | size = torch.tensor([tokens.shape], dtype=torch.long) |
| 245 | else: |
| 246 | tokens = None |
| 247 | size = torch.empty(2, dtype=torch.long) |
| 248 | |
| 249 | # broadcast data size |
| 250 | dist.broadcast(size, src=0) |
| 251 | |
| 252 | if rank != 0: |
| 253 | tokens = torch.empty(size.tolist(), dtype=torch.long) |
| 254 | |
| 255 | # broadcast tokens |
| 256 | dist.broadcast(tokens, src=0) |
| 257 | # the final input might be different from original input |
| 258 | # due to the max sequence limitation |
| 259 | return self.decode(tokens) |
| 260 | |
| 261 | def to(self, device): |
| 262 | self.model.to(device) |
no test coverage detected