| 42 | |
| 43 | |
| 44 | class OutlierTracer: |
| 45 | _instance = None |
| 46 | |
| 47 | def __init__(self): |
| 48 | raise RuntimeError("Call get_instance() instead") |
| 49 | |
| 50 | def initialize(self, model): |
| 51 | self.last_w = None |
| 52 | self.current_outlier_dims = None |
| 53 | self.hvalues = [] |
| 54 | self.outliers = [] |
| 55 | self.hvalue2outlier_idx = {} |
| 56 | self.initialized = True |
| 57 | self.hooks = [] |
| 58 | |
| 59 | for n, m in model.named_modules(): |
| 60 | if isinstance(m, torch.nn.Linear): |
| 61 | self.hooks.append(m.register_forward_pre_hook(outlier_hook)) |
| 62 | |
| 63 | def is_initialized(self): |
| 64 | return getattr(self, "initialized", False) |
| 65 | |
| 66 | def get_hvalue(self, weight): |
| 67 | return weight.data.storage().data_ptr() |
| 68 | |
| 69 | def get_outliers(self, weight): |
| 70 | if not self.is_initialized(): |
| 71 | logger.warning("Outlier tracer is not initialized...") |
| 72 | return None |
| 73 | hvalue = self.get_hvalue(weight) |
| 74 | if hvalue in self.hvalue2outlier_idx: |
| 75 | return self.hvalue2outlier_idx[hvalue] |
| 76 | else: |
| 77 | return None |
| 78 | |
| 79 | @classmethod |
| 80 | def get_instance(cls): |
| 81 | if cls._instance is None: |
| 82 | cls._instance = cls.__new__(cls) |
| 83 | return cls._instance |
| 84 | |
| 85 | |
| 86 | def find_outlier_dims(weight, reduction_dim=0, zscore=4.0, topk=None, rdm=False): |
nothing calls this directly
no outgoing calls
no test coverage detected