Converts a space-separated string of tokens to a list of ids.
(self, s)
| 159 | self.seg_index = self.token_to_id[SEG] if SEG in self.token_to_id else self.eos_index |
| 160 | |
| 161 | def encode(self, s): |
| 162 | """Converts a space-separated string of tokens to a list of ids.""" |
| 163 | if isinstance(s, str): |
| 164 | sentence = s |
| 165 | tokens = sentence.strip().split() |
| 166 | else: |
| 167 | tokens = s |
| 168 | if self._replace_oov is not None: |
| 169 | tokens = [t if t in self.token_to_id else self._replace_oov |
| 170 | for t in tokens] |
| 171 | ret = [self.token_to_id[tok] for tok in tokens] |
| 172 | return ret[::-1] if self._reverse else ret |
| 173 | |
| 174 | def decode(self, ids, strip_eos=False, strip_padding=False): |
| 175 | if strip_padding and self.pad() in list(ids): |
nothing calls this directly
no outgoing calls
no test coverage detected