MCPcopy Create free account
hub / github.com/google-research/language / __init__

Method __init__

language/emql/data_loader.py:70–116  ·  view source on GitHub ↗
(self,
               params,
               name,
               root_dir,
               kb_file,
               vocab_file = None)

Source from the content-addressed store, hash-verified

68 """
69
70 def __init__(self,
71 params,
72 name,
73 root_dir,
74 kb_file,
75 vocab_file = None):
76 # Load knowledge base.
77 self.root_dir = root_dir
78 self.max_set = params['max_set']
79 self.kb, self.fact2id, self.entity2id, self.relation2id = \
80 self.load_kb(name, root_dir + kb_file)
81 self.id2entity = {idx: ent for ent, idx in self.entity2id.items()}
82 self.id2fact = {idx: fact for fact, idx in self.fact2id.items()}
83 self.id2relation = {idx: rel for rel, idx in self.relation2id.items()}
84 self.num_entities = len(self.entity2id)
85 self.num_relations = len(self.relation2id)
86 self.num_facts = len(self.fact2id)
87
88 # Load vocab.
89 if vocab_file is not None:
90 self.word2id = self.load_vocab(self.root_dir + vocab_file)
91 self.id2word = {idx: word for word, idx in self.word2id.items()}
92 self.num_vocab = len(self.word2id)
93
94 # Construct count min sketch for entities in the KB.
95 self.cm_depth = params['cm_depth']
96 self.cm_width = params['cm_width']
97 self.cm_context = cm_sketch.CountMinContext(
98 depth=self.cm_depth, width=self.cm_width, n=self.num_entities)
99 self.cm_context_rel = cm_sketch.CountMinContext(
100 depth=self.cm_depth, width=self.cm_width, n=self.num_relations)
101
102 # Record fact info in numpy array.
103 self.all_entity_ids = np.arange(self.num_entities)
104 self.all_entity_sketches = np.array(
105 [self.cm_context.get_hashes(i) for i in range(self.num_entities)])
106 self.all_relation_sketches = np.array(
107 [self.cm_context_rel.get_hashes(i) for i in range(self.num_relations)])
108 self.all_fact_subjids = np.array(
109 [self.entity2id[self.id2fact[i][0]] for i in range(self.num_facts)])
110 self.all_fact_relids = np.array(
111 [self.relation2id[self.id2fact[i][1]] for i in range(self.num_facts)])
112 self.all_fact_objids = np.array(
113 [self.entity2id[self.id2fact[i][2]] for i in range(self.num_facts)])
114
115 # Construct training data for different datasets
116 self.construct_dataset(name, train_split=TRAIN_SPLIT)
117
118 def load_kb(
119 self, name, kb_file

Callers

nothing calls this directly

Calls 4

load_kbMethod · 0.95
load_vocabMethod · 0.95
construct_datasetMethod · 0.95
get_hashesMethod · 0.80

Tested by

no test coverage detected