MCPcopy Create free account
hub / github.com/google-research/language / construct_dataset

Method construct_dataset

language/emql/data_loader.py:235–269  ·  view source on GitHub ↗

Construct datasets for training and testing. Args: name: name of the learning task train_split: percentage of data for training

(self, name, train_split)

Source from the content-addressed store, hash-verified

233 ##############################################################
234
235 def construct_dataset(self, name, train_split):
236 """Construct datasets for training and testing.
237
238 Args:
239 name: name of the learning task
240 train_split: percentage of data for training
241 """
242 if name in ['membership', 'mixture']:
243 self.train_data_membership, self.test_data_membership = \
244 self.construct_membership(train_split)
245 if name in ['intersection', 'union', 'mixture']:
246 self.train_data_set_pair, self.test_data_set_pair = \
247 self.construct_set_pairs(train_split)
248 if name == 'follow':
249 self.train_data_follow, self.test_data_follow = \
250 self.construct_follow_facts(train_split)
251 if name in ['set_follow', 'mixture']:
252 self.train_data_follow, self.test_data_follow = \
253 self.construct_set_follow_facts(train_split)
254 if name in ['metaqa2', 'metaqa3']:
255 self.train_data_metaqa, self.test_data_metaqa = \
256 self.construct_metaqa_data(
257 train_file='train.json', test_file='test.json')
258 if name in ['webqsp']:
259 self.bert_tokenizer = util.BertTokenizer()
260 self.train_data_webqsp, self.test_data_webqsp = self.construct_webqsp(
261 train_file='train.json', test_file='test.json')
262 self.all_entity_is_cvt = self.check_cvt_entity()
263 if name.startswith('query2box'):
264 self.q2b_id2entity = pickle.load(
265 gfile.GFile(self.root_dir + 'ind2ent.pkl', 'rb'))
266 self.q2b_id2relation = pickle.load(
267 gfile.GFile(self.root_dir + 'ind2rel.pkl', 'rb'))
268 self.train_data_query2box, self.test_data_query2box = \
269 self.construct_query2box(name)
270
271 def construct_membership(
272 self, train_split):

Callers 1

__init__Method · 0.95

Calls 8

construct_membershipMethod · 0.95
construct_set_pairsMethod · 0.95
construct_metaqa_dataMethod · 0.95
construct_webqspMethod · 0.95
check_cvt_entityMethod · 0.95
construct_query2boxMethod · 0.95
loadMethod · 0.45

Tested by

no test coverage detected