(self, data, tokenizer, tokenize)
| 210 | assert_str = "make sure to set PATH for wikipedia data_utils/corpora.py" |
| 211 | |
| 212 | def process_line(self, data, tokenizer, tokenize): |
| 213 | keys, contents = data['key'], data["content"] |
| 214 | assert len(keys) == len(contents) |
| 215 | for i in range(1, len(keys)): |
| 216 | keys[i] = " " + keys[i] |
| 217 | contents = [" " + content for content in contents] |
| 218 | keys = [tokenizer.EncodeAsIds(key).tokenization for key in keys] |
| 219 | contents = [tokenizer.EncodeAsIds(content).tokenization for content in contents] |
| 220 | summary = sum(keys, []) |
| 221 | summary_prefix = self.process_sample("Summary: ", tokenizer, tokenize) |
| 222 | summary_mask = [len(summary_prefix), len(summary)] |
| 223 | summary = summary_prefix + summary |
| 224 | text, text_mask = [], [] |
| 225 | for key, content in zip(keys, contents): |
| 226 | content = content + [tokenizer.get_command('eop').Id] |
| 227 | text += key |
| 228 | text += content |
| 229 | text_mask.append(len(key)) |
| 230 | text_mask.append(len(content)) |
| 231 | return (summary, summary_mask), (text, text_mask) |
| 232 | |
| 233 | def tokenize_worker(self, input, output, info, tokenizer, tokenize): |
| 234 | for row in iter(input.get, 'STOP'): |
no test coverage detected