Return a string from a token id seq
(tks, sort=False)
| 880 | tokens_by_tid = self.query.idx.tokens_by_tid |
| 881 | |
| 882 | def tokens_string(tks, sort=False): |
| 883 | "Return a string from a token id seq" |
| 884 | tks = ('None' if tid is None else tokens_by_tid[tid] for tid in tks) |
| 885 | ascii_text = partial(toascii, translit=True) |
| 886 | tks = map(ascii_text, tks) |
| 887 | if sort: |
| 888 | tks = sorted(tks) |
| 889 | return ' '.join(tks) |
| 890 | |
| 891 | if brief and len(self.tokens) > 10: |
| 892 | tokens = tokens_string(self.tokens[:5]) + ' ... ' + tokens_string(self.tokens[-5:]) |