(self, input, output, info, tokenizer, tokenize)
| 231 | return (summary, summary_mask), (text, text_mask) |
| 232 | |
| 233 | def tokenize_worker(self, input, output, info, tokenizer, tokenize): |
| 234 | for row in iter(input.get, 'STOP'): |
| 235 | data = json.loads(row) |
| 236 | summary, content = self.process_line(data, tokenizer, tokenize) |
| 237 | output.put((summary, content)) |
| 238 | output.put("COMPLETE") |
| 239 | |
| 240 | @staticmethod |
| 241 | def write_result(data, writers): |
nothing calls this directly
no test coverage detected