| 39 | |
| 40 | |
| 41 | def check_sparsity(model): |
| 42 | use_cache = model.config.use_cache |
| 43 | model.config.use_cache = False |
| 44 | |
| 45 | layers = model.model.layers |
| 46 | count = 0 |
| 47 | total_params = 0 |
| 48 | for i in range(len(layers)): |
| 49 | layer = layers[i] |
| 50 | subset = find_layers(layer) |
| 51 | |
| 52 | sub_count = 0 |
| 53 | sub_params = 0 |
| 54 | for name in subset: |
| 55 | W = subset[name].weight.data |
| 56 | count += (W == 0).sum().item() |
| 57 | total_params += W.numel() |
| 58 | |
| 59 | sub_count += (W == 0).sum().item() |
| 60 | sub_params += W.numel() |
| 61 | |
| 62 | print(f"layer {i} sparsity {float(sub_count)/sub_params:.6f}") |
| 63 | |
| 64 | model.config.use_cache = use_cache |
| 65 | return float(count) / total_params |
| 66 | |
| 67 | |
| 68 | def check_sparsity_layerwise(model): |