(model)
| 66 | |
| 67 | |
| 68 | def check_sparsity_layerwise(model): |
| 69 | use_cache = model.config.use_cache |
| 70 | model.config.use_cache = False |
| 71 | |
| 72 | layers = model.model.layers |
| 73 | count = 0 |
| 74 | total_params = 0 |
| 75 | for i in range(len(layers)): |
| 76 | layer = layers[i] |
| 77 | subset = find_layers(layer) |
| 78 | |
| 79 | sub_count = 0 |
| 80 | sub_params = 0 |
| 81 | for name in subset: |
| 82 | W = subset[name].weight.data |
| 83 | count += (W == 0).sum().item() |
| 84 | total_params += W.numel() |
| 85 | |
| 86 | sub_count += (W == 0).sum().item() |
| 87 | sub_params += W.numel() |
| 88 | print(f"{float((W==0).sum().item())/W.numel():.6f},") |
| 89 | |
| 90 | model.config.use_cache = use_cache |
| 91 | |
| 92 | |
| 93 | def prepare_calibration_input(model, dataloader, device, nsamples): |
nothing calls this directly
no test coverage detected