(self, input, output, info, tokenizer, tokenize)
| 188 | is_json = True |
| 189 | |
| 190 | def tokenize_worker(self, input, output, info, tokenizer, tokenize): |
| 191 | for row in iter(input.get, 'STOP'): |
| 192 | if row: |
| 193 | if self.is_json: |
| 194 | row = row.rstrip() |
| 195 | row = json.loads(row) |
| 196 | prompts, texts = self.process_line(row, tokenizer, tokenize) |
| 197 | for prompt, text in zip(prompts, texts): |
| 198 | output.put((prompt, text)) |
| 199 | output.put("COMPLETE") |
| 200 | |
| 201 | @staticmethod |
| 202 | def write_result(data, writers): |
nothing calls this directly
no test coverage detected