| 302 | } |
| 303 | |
| 304 | Status BertTokenizer::buildInput(std::string paragraph, std::string question, std::shared_ptr<BertTokenizerInput> input) { |
| 305 | std::vector<size_t> code1, code2; |
| 306 | Status status; |
| 307 | features_.clear(); |
| 308 | |
| 309 | features_.push_back("[CLS]"); |
| 310 | code1 = Encode(question, status); |
| 311 | features_.push_back("[SEP]"); |
| 312 | code2 = Encode(paragraph, status); |
| 313 | features_.push_back("[SEP]"); |
| 314 | |
| 315 | code1.insert(code1.begin(), ClsId()); |
| 316 | code1.insert(code1.end(), SepId()); |
| 317 | for (size_t i = 0; i < MaxSeqCount; i++) { |
| 318 | if (i >= code1.size() && i < code1.size() + code2.size() + 1) reinterpret_cast<int*>(input->segments)[i] = 1; |
| 319 | else reinterpret_cast<int*>(input->segments)[i] = 0; |
| 320 | } |
| 321 | code1.insert(code1.end(), code2.begin(), code2.end()); |
| 322 | code1.insert(code1.end(), SepId()); |
| 323 | |
| 324 | if (code1.size() < MaxSeqCount) { |
| 325 | code1.insert(code1.end(), (MaxSeqCount - code1.size()), 0); |
| 326 | } |
| 327 | |
| 328 | for (size_t i = 0; i < MaxSeqCount; i++) { |
| 329 | if (code1[i]) { |
| 330 | reinterpret_cast<int*>(input->inputIds)[i] = code1[i]; |
| 331 | reinterpret_cast<int*>(input->inputMasks)[i] = 1; |
| 332 | } else { |
| 333 | reinterpret_cast<int*>(input->inputIds)[i] = 0; |
| 334 | reinterpret_cast<int*>(input->inputMasks)[i] = 0; |
| 335 | reinterpret_cast<int*>(input->segments)[i] = 0; |
| 336 | } |
| 337 | } |
| 338 | |
| 339 | return TNN_OK; |
| 340 | } |
| 341 | |
| 342 | BertTokenizerInput::BertTokenizerInput(DeviceType device_type) { |
| 343 | inputIds = (void*)malloc(sizeof(float) * MaxSeqCount); |