Method
__init__
(self,
tokenizer_class,
tokenizer_name,
cache_dir,
data_path,
dataset,
batch_size,
num_data_workers,
max_nodes,
max_edges,
edges_as_classes)
Source from the content-addressed store, hash-verified
| 13 | class GraphDataModule(pl.LightningDataModule): |
| 14 | |
| 15 | def __init__(self, |
| 16 | tokenizer_class, |
| 17 | tokenizer_name, |
| 18 | cache_dir, |
| 19 | data_path, |
| 20 | dataset, |
| 21 | batch_size, |
| 22 | num_data_workers, |
| 23 | max_nodes, |
| 24 | max_edges, |
| 25 | edges_as_classes): |
| 26 | super().__init__() |
| 27 | |
| 28 | self.tokenizer = tokenizer_class.from_pretrained(tokenizer_name, cache_dir=cache_dir) |
| 29 | self.tokenizer.add_tokens('__no_node__') |
| 30 | self.tokenizer.add_tokens('__no_edge__') |
| 31 | self.tokenizer.add_tokens('__node_sep__') |
| 32 | |
| 33 | self.batch_size = batch_size |
| 34 | self.num_data_workers = num_data_workers |
| 35 | self.data_path = data_path |
| 36 | self.max_nodes = max_nodes |
| 37 | self.max_edges = max_edges |
| 38 | self.edges_as_classes = edges_as_classes |
| 39 | self.output_path = os.path.join(data_path, 'processed') |
| 40 | os.makedirs(self.output_path, exist_ok=True) |
| 41 | self.dataset = dataset |
| 42 | |
| 43 | def prepareWebNLG(self): |
| 44 | |
Callers
nothing calls this directly
Tested by
no test coverage detected