(batch: BatchTokensType, pad_id: int, seq_length: int)
| 23 | |
| 24 | |
| 25 | def pad_batch(batch: BatchTokensType, pad_id: int, seq_length: int) -> BatchTokensType: |
| 26 | for tokens in batch: |
| 27 | context_length = len(tokens) |
| 28 | if context_length < seq_length: |
| 29 | tokens.extend([pad_id] * (seq_length - context_length)) |
| 30 | return batch |
| 31 | |
| 32 | |
| 33 | def get_ltor_masks_and_position_ids( |
nothing calls this directly
no outgoing calls
no test coverage detected