MCPcopy Create free account
hub / github.com/google-research/language / construct_webqsp

Method construct_webqsp

language/emql/data_loader.py:428–514  ·  view source on GitHub ↗

Load WebQuestionsSP datasets from files. Args: train_file: filename for train data test_file: filename for test data max_webqsp_ans: max number of answers, dropped otherwise Returns: a list of train data and test data

(self,
                       train_file,
                       test_file,
                       max_webqsp_ans = 1000)

Source from the content-addressed store, hash-verified

426 return train_data, test_data
427
428 def construct_webqsp(self,
429 train_file,
430 test_file,
431 max_webqsp_ans = 1000):
432 """Load WebQuestionsSP datasets from files.
433
434 Args:
435 train_file: filename for train data
436 test_file: filename for test data
437 max_webqsp_ans: max number of answers, dropped otherwise
438
439 Returns:
440 a list of train data and test data
441
442 """
443 self.max_webqsp_ans = max_webqsp_ans
444 train_data, test_data = [], []
445 self.ent2ent_relations = set()
446 self.ent2cvt_relations = set()
447 self.cvt2ent_relations = set()
448 for filename, data_split in zip([train_file, test_file],
449 [train_data, test_data]):
450 all_data = json.load(gfile.GFile(self.root_dir + filename))
451 for data in all_data['Questions']:
452 qid = data['QuestionId']
453
454 # Parse question and question entities.
455 question = data['ProcessedQuestion']
456 _, (question_token_ids, segment_ids, question_mask) = \
457 self.bert_tokenizer.tokenize(question)
458 question_entity = self.convert_fbid(data['Parses'][0]['TopicEntityMid'])
459 if question_entity is None or question_entity not in self.entity2id:
460 tf.logging.warn('Question %s illegal TopicEntityMid %s ' %
461 (qid, question_entity))
462 continue
463 question_entity_id = self.entity2id[question_entity]
464
465 # Parse answers. Convert answers into a list of ids and pad with -1.
466 answers = set()
467 for parse in data['Parses']:
468 answers.update([
469 self.convert_fbid(ans['AnswerArgument'])
470 if ans['AnswerType'] == 'Entity' else ans['AnswerArgument']
471 for ans in parse['Answers']
472 ])
473 answers = list(answers)[:max_webqsp_ans]
474 answer_ids = np.array(
475 [self.entity2id[ans] for ans in answers if ans in self.entity2id],
476 dtype=np.int32)
477 if len(answer_ids) == 0: # pylint: disable=g-explicit-length-test
478 tf.logging.warn('Question %s no answers' % (qid))
479 continue
480 answer_ids = np.pad(
481 answer_ids, (0, max_webqsp_ans - len(answer_ids)),
482 mode='constant',
483 constant_values=-1)
484
485 # Parse constraints.

Callers 1

construct_datasetMethod · 0.95

Calls 5

convert_fbidMethod · 0.95
padMethod · 0.80
loadMethod · 0.45
tokenizeMethod · 0.45
addMethod · 0.45

Tested by

no test coverage detected