Method create_examples

tasks/seq2seq/dataset.py:195–230

(self, split)

        self.tokenizer = tokenizer

    def create_examples(self, split):
        if split == "train":
            filename = "train.json"
        elif split == "dev":
            filename = "dev.json"
        elif split == "test":
            filename = "test.json"
        else:
            raise NotImplementedError(split)
        print_rank_0(f"Creating SQuAD-{split} dataset from {self.data_dir}")
        example_list = []
        idx = 0
        with open(os.path.join(self.data_dir, filename), encoding='utf-8') as file:
            dataset = json.load(file)
            for paragraphs in dataset:
                for paragraph in paragraphs['paragraphs']:
                    context = paragraph['context']
                    for qa in paragraph['qas']:
                        question = qa["question"]
                        answers = {answer["text"] for answer in qa["answers"]}
                        answer_starts = {answer["text"]: answer["answer_start"] for answer in qa["answers"]}
                        for answer in answers:
                            guid = "%s-%s" % (split, idx)
                            meta = {
                                "answer_start": answer_starts[answer],
                                "answer": answer,
                                "question": question,
                                "ref": self.tokenizer.DecodeIds(self.tokenizer.EncodeAsIds(question).tokenization)}
                            example = InputExample(guid=guid, text_a=context, meta=meta)
                            if idx < 10:
                                print_rank_0(
                                    (context.encode('utf-8'), answer.encode('utf-8'), meta["ref"].encode('utf-8')))
                            example_list.append(example)
                            idx += 1
        print_rank_0(f"Creating {len(example_list)} examples for {split}")
        return example_list


def generate_token_to_char_map(tokens, raw_text, tokenizer):