Generates `relative_att_ids` for purely distance-based relative positions. This implements the clipped relative position representations originally described in https://arxiv.org/abs/1803.02155 . Attributes: max_distance: Integer passed from `__init__`. ignore_direction: Bool passed
| 199 | |
| 200 | |
| 201 | class RelativePositionGenerator(object): |
| 202 | """Generates `relative_att_ids` for purely distance-based relative positions. |
| 203 | |
| 204 | This implements the clipped relative position representations originally |
| 205 | described in https://arxiv.org/abs/1803.02155 . |
| 206 | |
| 207 | Attributes: |
| 208 | max_distance: Integer passed from `__init__`. |
| 209 | ignore_direction: Bool passed from `__init__`. |
| 210 | relative_vocab_size: Integer representing the maximum number of unique ids |
| 211 | output from this generator. |
| 212 | left_pad_value: Integer id for all positions at or beyond max_distance to |
| 213 | the left. |
| 214 | right_pad_value: Integer id for all positions at or beyond max_distance to |
| 215 | the right. |
| 216 | """ |
| 217 | |
| 218 | def __init__(self, max_distance: int, ignore_direction: bool = False): |
| 219 | """Init. |
| 220 | |
| 221 | Args: |
| 222 | max_distance: The maximum distance to represent. Must not be negative. All |
| 223 | larger distances will be clipped to this value. |
| 224 | ignore_direction: If True, both left and right position representations |
| 225 | will have the same ids based on absolute distance (resulting in |
| 226 | symmetric ids around the center token). |
| 227 | """ |
| 228 | if max_distance < 0: |
| 229 | raise ValueError('`max_distance` must not be negative.') |
| 230 | self.max_distance = max_distance |
| 231 | self.ignore_direction = ignore_direction |
| 232 | |
| 233 | self.right_pad_value = max_distance |
| 234 | self.left_pad_value = max_distance if ignore_direction else 2 * max_distance |
| 235 | |
| 236 | # 0 is the first id, so vocab size is 1 + the largest id (left pad value). |
| 237 | self.relative_vocab_size = self.left_pad_value + 1 |
| 238 | |
| 239 | def make_relative_att_ids(self, |
| 240 | seq_len: Union[int, tf.Tensor], |
| 241 | batch_size: Optional[Union[int, tf.Tensor]] = 1, |
| 242 | name: Optional[Text] = None) -> tf.Tensor: |
| 243 | """Makes relative position ids for full self-attention. |
| 244 | |
| 245 | For example, if `max_distance` is 3, `ignore_direction` is False, `seq_len` |
| 246 | is 6, and `batch_size` is 1, the result is the following: |
| 247 | [[ |
| 248 | [0, 1, 2, 3, 3, 3], |
| 249 | [4, 0, 1, 2, 3, 3], |
| 250 | [5, 4, 0, 1, 2, 3], |
| 251 | [6, 5, 4, 0, 1, 2], |
| 252 | [6, 6, 5, 4, 0, 1], |
| 253 | [6, 6, 6, 5, 4, 0], |
| 254 | ]] |
| 255 | |
| 256 | Args: |
| 257 | seq_len: The sequence length to create ids for. Must be positive. If a |
| 258 | Tensor, must be a scalar int. |