| 4 | from codepile.dataset import Dataset |
| 5 | |
| 6 | class CodePile(Dataset): |
| 7 | def __init__(self, tempdir, target_dir): |
| 8 | self.subdatasets = [] |
| 9 | |
| 10 | subsets = [ |
| 11 | StackExchangeDataset |
| 12 | ] |
| 13 | for d in subsets: |
| 14 | self.subdatasets.append(d(tempdir, target_dir)) |
| 15 | |
| 16 | def download(self): |
| 17 | for d in self.subdatasets: |
| 18 | d.scraper.scrape() |
| 19 | |
| 20 | def process(self): |
| 21 | for d in self.subdatasets: |
| 22 | d.processor.process() |
| 23 | |
| 24 | def merge(self): |
| 25 | raise NotImplementedError() |
| 26 | |
| 27 | |
| 28 | def download(args): |