MCPcopy Create free account
hub / github.com/google-research/language / construct_metaqa_data

Method construct_metaqa_data

language/emql/data_loader.py:396–426  ·  view source on GitHub ↗

Load MetaQA datasets from files. Args: train_file: filename for train data test_file: filename for test data Returns: a list of train pairs and test pairs

(self, train_file,
                            test_file)

Source from the content-addressed store, hash-verified

394 return train_facts, test_facts
395
396 def construct_metaqa_data(self, train_file,
397 test_file):
398 """Load MetaQA datasets from files.
399
400 Args:
401 train_file: filename for train data
402 test_file: filename for test data
403
404 Returns:
405 a list of train pairs and test pairs
406
407 """
408 self.max_question_len = 0
409 train_data, test_data = list(), list()
410 for filename, data in zip([train_file, test_file], [train_data, test_data]):
411 with gfile.GFile(self.root_dir + filename) as f:
412 for line in f:
413 line = json.loads(line)
414 question = [self.word2id[word] for word in line['question'].split()]
415 self.max_question_len = max(self.max_question_len, len(question))
416 question_entity = line['question_entities'][0]
417 question_entity_id = self.entity2id[question_entity]
418
419 answers = set(line['answers'])
420 answer_fact_ids = [
421 f_id for (subj, relation, obj), f_id in self.fact2id.items()
422 if obj in answers
423 ]
424 data.append([question, question_entity_id, answer_fact_ids])
425
426 return train_data, test_data
427
428 def construct_webqsp(self,
429 train_file,

Callers 1

construct_datasetMethod · 0.95

Calls 1

splitMethod · 0.80

Tested by

no test coverage detected