(self)
| 545 | return self._config.data_efficiency_config[DATA_ROUTING][RANDOM_LTD] |
| 546 | |
| 547 | def random_ltd_initialize(self): |
| 548 | assert self.random_ltd_enabled() |
| 549 | random_ltd_config = self.random_ltd_config() |
| 550 | random_ltd_queue = deque([x for x in sorted(random_ltd_config[RANDOM_LTD_LAYER_ID])]) |
| 551 | count = 0 |
| 552 | for name, layer in self.module.named_modules(): |
| 553 | if isinstance(layer, RandomLayerTokenDrop): |
| 554 | if len(random_ltd_queue) != 0 and str(random_ltd_queue[0]) in name: ###[1,2,3] |
| 555 | layer.init_config(random_ltd_config, self.random_ltd_scheduler, count) |
| 556 | random_ltd_queue.popleft() |
| 557 | count += 1 |
| 558 | |
| 559 | if random_ltd_config[RANDOM_LTD_LAYER_NUM] != count: |
| 560 | raise ValueError(f'random_ltd_layer_num {random_ltd_config[RANDOM_LTD_LAYER_NUM]} must be \ |
| 561 | equivalent to the len of random_ltd_layer_id {count}') |
| 562 | |
| 563 | if random_ltd_config[RANDOM_LTD_LAYER_TOKEN_LR_SCHEDULE][RANDOM_LTD_LAYER_TOKEN_LR_ENABLED]: |
| 564 | assert self.client_lr_scheduler is None |
| 565 | raise ValueError(f'not yet support') |
| 566 | #self.lr_scheduler = lr_schedules.WarmupLayerTokenDecayLR(self.optimizer, self.random_ltd_scheduler) |
| 567 | |
| 568 | def wall_clock_breakdown(self): |
| 569 | return self._config.wall_clock_breakdown |
no test coverage detected