| 18 | |
| 19 | |
| 20 | class Corpus(object): |
| 21 | def __init__(self, path): |
| 22 | self.dictionary = Dictionary() |
| 23 | self.train = self.tokenize(os.path.join(path, 'train.txt')) |
| 24 | self.valid = self.tokenize(os.path.join(path, 'valid.txt')) |
| 25 | self.test = self.tokenize(os.path.join(path, 'test.txt')) |
| 26 | |
| 27 | def tokenize(self, path): |
| 28 | """Tokenizes a text file.""" |
| 29 | assert os.path.exists(path) |
| 30 | # Add words to the dictionary |
| 31 | with open(path, 'r', encoding="utf8") as f: |
| 32 | for line in f: |
| 33 | words = line.split() + ['<eos>'] |
| 34 | for word in words: |
| 35 | self.dictionary.add_word(word) |
| 36 | |
| 37 | # Tokenize file content |
| 38 | with open(path, 'r', encoding="utf8") as f: |
| 39 | idss = [] |
| 40 | for line in f: |
| 41 | words = line.split() + ['<eos>'] |
| 42 | ids = [] |
| 43 | for word in words: |
| 44 | ids.append(self.dictionary.word2idx[word]) |
| 45 | idss.append(torch.tensor(ids).type(torch.int64)) |
| 46 | ids = torch.cat(idss) |
| 47 | |
| 48 | return ids |
nothing calls this directly
no outgoing calls
no test coverage detected