Execute forward propagation Arguments: *inputs: Variable length input list **kwargs: variable length keyword arguments
(self, *inputs, **kwargs)
| 555 | return sub_module_cuda_graph |
| 556 | |
| 557 | def forward(self, *inputs, **kwargs): |
| 558 | """Execute forward propagation |
| 559 | |
| 560 | Arguments: |
| 561 | *inputs: Variable length input list |
| 562 | **kwargs: variable length keyword arguments |
| 563 | """ |
| 564 | start = None |
| 565 | if self.model_profile_enabled and get_accelerator().device_name() == 'cuda' and self._config.enable_cuda_graph: |
| 566 | get_accelerator().synchronize() |
| 567 | start = time.time() |
| 568 | |
| 569 | if get_accelerator().device_name() == 'cuda' and self._config.enable_cuda_graph and not self.local_cuda_graph: |
| 570 | if self.cuda_graph_created: |
| 571 | outputs = self._graph_replay(*inputs, **kwargs) |
| 572 | else: |
| 573 | self._create_cuda_graph(*inputs, **kwargs) |
| 574 | outputs = self._graph_replay(*inputs, **kwargs) |
| 575 | |
| 576 | else: |
| 577 | outputs = self.module(*inputs, **kwargs) |
| 578 | |
| 579 | if self.model_profile_enabled and self._config.enable_cuda_graph: |
| 580 | get_accelerator().synchronize() |
| 581 | duration = (time.time() - start) * 1e3 # convert seconds to ms |
| 582 | self._model_times.append(duration) |
| 583 | |
| 584 | return outputs |
| 585 | |
| 586 | def _generate(self, *inputs, **kwargs): |
| 587 | # Reset KV-cache at the beginning of generate |
nothing calls this directly
no test coverage detected