Method generate_blank_data

blocklm_utils.py:270–281

(self, sample, masked_lengths, attention_mask, rng, task='bert')

                return tokens, targets, loss_masks, position_ids, source_length

    def generate_blank_data(self, sample, masked_lengths, attention_mask, rng, task='bert'):
        rng.shuffle(masked_lengths)
        tokens, loss_masks = sample['text'], sample['loss_mask']
        assert tokens[0] == self.tokenizer.get_command('ENC').Id
        block_spans = self.sample_span_in_document(tokens, masked_lengths, rng)
        if len(block_spans) < len(masked_lengths):
            return None
        if self.masked_lm:
            data = self.make_masked_data(tokens, loss_masks, attention_mask, block_spans, rng)
        else:
            data = self.make_block_data(tokens, loss_masks, attention_mask, block_spans, rng, task=task)
        return data

    def split_samples(self, samples, rng):
        target_length = rng.randrange(32, self.max_seq_length - 1)