Method batchify

ppo/ppo_datahelper.py:337–351

(self, batch_samples: List[Dict[str, Any]])

        return output

    def batchify(self, batch_samples: List[Dict[str, Any]]) -> Dict[str, Any]:
        batch = dict()
        batch_text_vec = torch.tensor(pad_sequences(
            [sample['text_vec'] for sample in batch_samples], pad_value=self.tokenizer.pad_token_id, pad_left=False
            ), dtype=torch.long)
        loss_mask = torch.tensor(pad_sequences(
            [sample['loss_mask'] for sample in batch_samples], pad_value=0, pad_left=False
            ), dtype=torch.bool)
   
        batch.update({
            'text_vec': batch_text_vec,
            'loss_mask': loss_mask
        })
        
        return batch
            
    def batch_generator(self):
        while True: