(self, json_line)
| 76 | Encoder.splitter = IdentitySplitter() |
| 77 | |
| 78 | def encode(self, json_line): |
| 79 | data = json.loads(json_line) |
| 80 | ids = {} |
| 81 | for key in self.args.json_keys: |
| 82 | text = data[key] |
| 83 | doc_ids = [] |
| 84 | for sentence in Encoder.splitter.tokenize(text): |
| 85 | sentence_ids = Encoder.tokenizer.tokenize(sentence) |
| 86 | if len(sentence_ids) > 0: |
| 87 | doc_ids.append(sentence_ids) |
| 88 | if self.args.append_eod: |
| 89 | doc_ids[-1].append(Encoder.tokenizer.eod) |
| 90 | ids[key] = doc_ids |
| 91 | return ids, len(json_line) |
| 92 | |
| 93 | def get_args(): |
| 94 | parser = argparse.ArgumentParser() |
no test coverage detected