| 34 | |
| 35 | |
| 36 | class CharRNNData(RNGDataFlow): |
| 37 | def __init__(self, input_file, size): |
| 38 | self.seq_length = param.seq_len |
| 39 | self._size = size |
| 40 | |
| 41 | logger.info("Loading corpus...") |
| 42 | # preprocess data |
| 43 | with open(input_file, 'rb') as f: |
| 44 | data = f.read() |
| 45 | data = [chr(c) for c in data if c < 128] |
| 46 | counter = Counter(data) |
| 47 | char_cnt = sorted(counter.items(), key=operator.itemgetter(1), reverse=True) |
| 48 | self.chars = [x[0] for x in char_cnt] |
| 49 | print(sorted(self.chars)) |
| 50 | self.vocab_size = len(self.chars) |
| 51 | param.vocab_size = self.vocab_size |
| 52 | self.char2idx = {c: i for i, c in enumerate(self.chars)} |
| 53 | self.whole_seq = np.array([self.char2idx[c] for c in data], dtype='int32') |
| 54 | logger.info("Corpus loaded. Vocab size: {}".format(self.vocab_size)) |
| 55 | |
| 56 | def __len__(self): |
| 57 | return self._size |
| 58 | |
| 59 | def __iter__(self): |
| 60 | random_starts = self.rng.randint( |
| 61 | 0, self.whole_seq.shape[0] - self.seq_length - 1, (self._size,)) |
| 62 | for st in random_starts: |
| 63 | seq = self.whole_seq[st:st + self.seq_length + 1] |
| 64 | yield [seq[:-1], seq[1:]] |
| 65 | |
| 66 | |
| 67 | class Model(ModelDesc): |
no outgoing calls
no test coverage detected