A lightweight class for recording time without any distributed barrier. This class allows for recording elapsed time between events without requiring synchronization across distributed processes. It maintains the previous message and time to calculate the duration between consecuti
| 20 | |
| 21 | |
| 22 | class EventPathTimer: |
| 23 | """ |
| 24 | A lightweight class for recording time without any distributed barrier. |
| 25 | |
| 26 | This class allows for recording elapsed time between events without requiring |
| 27 | synchronization across distributed processes. It maintains the previous message |
| 28 | and time to calculate the duration between consecutive records. |
| 29 | """ |
| 30 | |
| 31 | def __init__(self): |
| 32 | """ |
| 33 | Initialize the EventPathTimer. |
| 34 | |
| 35 | This constructor sets the previous message and time to None, preparing |
| 36 | the instance for recording events. |
| 37 | """ |
| 38 | self.prev_message: str = None |
| 39 | self.prev_time: datetime = None |
| 40 | |
| 41 | def reset(self): |
| 42 | """ |
| 43 | Reset the recorded message and time. |
| 44 | |
| 45 | This method clears the previous message and time, allowing for a fresh |
| 46 | start in recording new events. |
| 47 | """ |
| 48 | self.prev_message = None |
| 49 | self.prev_time = None |
| 50 | |
| 51 | def synced_record(self, message): |
| 52 | """ |
| 53 | Record the current time with a message. |
| 54 | |
| 55 | Args: |
| 56 | message (str): A message to log along with the current time. |
| 57 | |
| 58 | This method synchronizes the CUDA operations, records the current time, |
| 59 | and calculates the elapsed time since the last recorded message, if any. |
| 60 | It then logs the elapsed time along with the previous and current messages. |
| 61 | """ |
| 62 | torch.cuda.synchronize() |
| 63 | current_time = datetime.now() |
| 64 | if self.prev_message is not None: |
| 65 | print_rank_0( |
| 66 | f"\nTime Elapsed: [{current_time - self.prev_time}] From [{self.prev_message} ({self.prev_time})] To [{message} ({current_time})]" |
| 67 | ) |
| 68 | self.prev_message = message |
| 69 | self.prev_time = current_time |
| 70 | |
| 71 | |
| 72 | _GLOBAL_LIGHT_TIMER = EventPathTimer() |