(self, config: PretrainedConfig)
| 1083 | """ |
| 1084 | |
| 1085 | def __init__(self, config: PretrainedConfig): |
| 1086 | super().__init__() |
| 1087 | |
| 1088 | self.summary_type = getattr(config, "summary_type", "last") |
| 1089 | if self.summary_type == "attn": |
| 1090 | # We should use a standard multi-head attention module with absolute positional embedding for that. |
| 1091 | # Cf. https://github.com/zihangdai/xlnet/blob/master/modeling.py#L253-L276 |
| 1092 | # We can probably just use the multi-head attention module of PyTorch >=1.1.0 |
| 1093 | raise NotImplementedError |
| 1094 | |
| 1095 | self.summary = Identity() |
| 1096 | if hasattr(config, "summary_use_proj") and config.summary_use_proj: |
| 1097 | if hasattr(config, "summary_proj_to_labels") and config.summary_proj_to_labels and config.num_labels > 0: |
| 1098 | num_classes = config.num_labels |
| 1099 | else: |
| 1100 | num_classes = config.hidden_size |
| 1101 | self.summary = nn.Linear(config.hidden_size, num_classes) |
| 1102 | |
| 1103 | activation_string = getattr(config, "summary_activation", None) |
| 1104 | self.activation: Callable = (get_activation(activation_string) if activation_string else Identity()) |
| 1105 | |
| 1106 | self.first_dropout = Identity() |
| 1107 | if hasattr(config, "summary_first_dropout") and config.summary_first_dropout > 0: |
| 1108 | self.first_dropout = nn.Dropout(config.summary_first_dropout) |
| 1109 | |
| 1110 | self.last_dropout = Identity() |
| 1111 | if hasattr(config, "summary_last_dropout") and config.summary_last_dropout > 0: |
| 1112 | self.last_dropout = nn.Dropout(config.summary_last_dropout) |
| 1113 | |
| 1114 | def forward(self, hidden_states, cls_index=None): |
| 1115 | """ hidden_states: float Tensor in shape [bsz, ..., seq_len, hidden_size], the hidden-states of the last layer. |
nothing calls this directly
no test coverage detected