Load WebQuestionsSP datasets from files. Args: train_file: filename for train data test_file: filename for test data max_webqsp_ans: max number of answers, dropped otherwise Returns: a list of train data and test data
(self,
train_file,
test_file,
max_webqsp_ans = 1000)
| 426 | return train_data, test_data |
| 427 | |
| 428 | def construct_webqsp(self, |
| 429 | train_file, |
| 430 | test_file, |
| 431 | max_webqsp_ans = 1000): |
| 432 | """Load WebQuestionsSP datasets from files. |
| 433 | |
| 434 | Args: |
| 435 | train_file: filename for train data |
| 436 | test_file: filename for test data |
| 437 | max_webqsp_ans: max number of answers, dropped otherwise |
| 438 | |
| 439 | Returns: |
| 440 | a list of train data and test data |
| 441 | |
| 442 | """ |
| 443 | self.max_webqsp_ans = max_webqsp_ans |
| 444 | train_data, test_data = [], [] |
| 445 | self.ent2ent_relations = set() |
| 446 | self.ent2cvt_relations = set() |
| 447 | self.cvt2ent_relations = set() |
| 448 | for filename, data_split in zip([train_file, test_file], |
| 449 | [train_data, test_data]): |
| 450 | all_data = json.load(gfile.GFile(self.root_dir + filename)) |
| 451 | for data in all_data['Questions']: |
| 452 | qid = data['QuestionId'] |
| 453 | |
| 454 | # Parse question and question entities. |
| 455 | question = data['ProcessedQuestion'] |
| 456 | _, (question_token_ids, segment_ids, question_mask) = \ |
| 457 | self.bert_tokenizer.tokenize(question) |
| 458 | question_entity = self.convert_fbid(data['Parses'][0]['TopicEntityMid']) |
| 459 | if question_entity is None or question_entity not in self.entity2id: |
| 460 | tf.logging.warn('Question %s illegal TopicEntityMid %s ' % |
| 461 | (qid, question_entity)) |
| 462 | continue |
| 463 | question_entity_id = self.entity2id[question_entity] |
| 464 | |
| 465 | # Parse answers. Convert answers into a list of ids and pad with -1. |
| 466 | answers = set() |
| 467 | for parse in data['Parses']: |
| 468 | answers.update([ |
| 469 | self.convert_fbid(ans['AnswerArgument']) |
| 470 | if ans['AnswerType'] == 'Entity' else ans['AnswerArgument'] |
| 471 | for ans in parse['Answers'] |
| 472 | ]) |
| 473 | answers = list(answers)[:max_webqsp_ans] |
| 474 | answer_ids = np.array( |
| 475 | [self.entity2id[ans] for ans in answers if ans in self.entity2id], |
| 476 | dtype=np.int32) |
| 477 | if len(answer_ids) == 0: # pylint: disable=g-explicit-length-test |
| 478 | tf.logging.warn('Question %s no answers' % (qid)) |
| 479 | continue |
| 480 | answer_ids = np.pad( |
| 481 | answer_ids, (0, max_webqsp_ans - len(answer_ids)), |
| 482 | mode='constant', |
| 483 | constant_values=-1) |
| 484 | |
| 485 | # Parse constraints. |
no test coverage detected