Prune a linear layer (a model parameters) to keep only entries in index. Return the pruned layer as a new layer with requires_grad=True. Used to remove heads.
(layer, index, dim=0)
| 2095 | |
| 2096 | |
| 2097 | def prune_linear_layer(layer, index, dim=0): |
| 2098 | """ Prune a linear layer (a model parameters) to keep only entries in index. |
| 2099 | Return the pruned layer as a new layer with requires_grad=True. |
| 2100 | Used to remove heads. |
| 2101 | """ |
| 2102 | index = index.to(layer.weight.device) |
| 2103 | W = layer.weight.index_select(dim, index).clone().detach() |
| 2104 | if layer.bias is not None: |
| 2105 | if dim == 1: |
| 2106 | b = layer.bias.clone().detach() |
| 2107 | else: |
| 2108 | b = layer.bias[index].clone().detach() |
| 2109 | new_size = list(layer.weight.size()) |
| 2110 | new_size[dim] = len(index) |
| 2111 | new_layer = nn.Linear(new_size[1], new_size[0], bias=layer.bias is not None).to(layer.weight.device) |
| 2112 | new_layer.weight.requires_grad = False |
| 2113 | new_layer.weight.copy_(W.contiguous()) |
| 2114 | new_layer.weight.requires_grad = True |
| 2115 | if layer.bias is not None: |
| 2116 | new_layer.bias.requires_grad = False |
| 2117 | new_layer.bias.copy_(b.contiguous()) |
| 2118 | new_layer.bias.requires_grad = True |
| 2119 | return new_layer |
| 2120 | |
| 2121 | |
| 2122 | def prune_conv1d_layer(layer, index, dim=1): |
no test coverage detected