Docstring for Seg Model
| 246 | return lac_result |
| 247 | |
| 248 | class SegModel(Model): |
| 249 | """Docstring for Seg Model""" |
| 250 | def __init__(self, model_path, mode, use_cuda): |
| 251 | super(SegModel, self).__init__(model_path, mode, use_cuda) |
| 252 | self.dataset = reader.SegDataset(self.args) |
| 253 | |
| 254 | def run(self, texts): |
| 255 | crf_result = super(SegModel, self).run(texts)["crf_result"] |
| 256 | result = [word for word, tag, tag_for_rank in crf_result] if self.batch else crf_result[0][0] |
| 257 | return result |
| 258 | |
| 259 | def texts2tensor(self, texts): |
| 260 | """文本输入转为Paddle输入的Tensor""" |
| 261 | lod, data, words_length = [0], [], [] |
| 262 | for i, text in enumerate(texts): |
| 263 | |
| 264 | text_inds = self.dataset.word_to_ids(text) |
| 265 | data += text_inds |
| 266 | lod.append(len(text_inds) + lod[i]) |
| 267 | |
| 268 | tensor = self.to_tensor(data, lod) if len(data) != 0 else None |
| 269 | |
| 270 | return tensor, words_length |
| 271 | |
| 272 | def parse_result(self, lines, crf_decode, dataset, words_length): |
| 273 | """将SEG模型输出的Tensor转为明文""" |
| 274 | offset_list = crf_decode.lod[0] |
| 275 | crf_decode = crf_decode.data.int64_data() |
| 276 | batch_size = len(offset_list) - 1 |
| 277 | |
| 278 | batch_out = [] |
| 279 | for sent_index in range(batch_size): |
| 280 | begin, end = offset_list[sent_index], offset_list[sent_index + 1] |
| 281 | |
| 282 | sent = lines[sent_index] |
| 283 | tags = [dataset.id2label_dict[str(id)] |
| 284 | for id in crf_decode[begin:end]] |
| 285 | tags_for_rank = [] |
| 286 | |
| 287 | if self.custom: |
| 288 | self.custom.parse_customization(sent, tags) |
| 289 | |
| 290 | sent_out, tags_out = [], [] |
| 291 | for ind, tag in enumerate(tags): |
| 292 | # for the first char |
| 293 | if len(sent_out) == 0 or tag.endswith("B") or tag.endswith("S"): |
| 294 | sent_out.append(sent[ind]) |
| 295 | tags_out.append(tag[:-2]) |
| 296 | continue |
| 297 | sent_out[-1] += sent[ind] |
| 298 | # 取最后一个tag作为标签 |
| 299 | tags_out[-1] = tag[:-2] |
| 300 | |
| 301 | sent_out = [''] if len(sent_out) == 0 else sent_out |
| 302 | batch_out.append([sent_out, tags_out, tags_for_rank]) |
| 303 | return batch_out |
| 304 | |
| 305 | class RankModel(Model): |