| 569 | |
| 570 | template <typename P> |
| 571 | void SetStringTensorFromPyArray(phi::StringTensor *self, |
| 572 | const py::array &array, |
| 573 | const P &place) { |
| 574 | bool is_string_pyarray = |
| 575 | array.dtype().kind() == 'S' || array.dtype().kind() == 'U'; |
| 576 | PADDLE_ENFORCE_EQ(is_string_pyarray, |
| 577 | true, |
| 578 | common::errors::InvalidArgument( |
| 579 | "Expect the dtype of numpy array is string or " |
| 580 | "unicode, but receive dtype %s", |
| 581 | array.dtype())); |
| 582 | std::vector<int64_t> dims; |
| 583 | dims.reserve(array.ndim()); |
| 584 | dims.reserve(array.ndim()); |
| 585 | for (decltype(array.ndim()) i = 0; i < array.ndim(); ++i) { |
| 586 | dims.push_back(static_cast<int>(array.shape()[i])); |
| 587 | } |
| 588 | self->Resize(common::make_ddim(dims)); |
| 589 | auto itemsize = array.itemsize(); |
| 590 | if (phi::is_cpu_place(place)) { |
| 591 | auto dst = self->mutable_data(place); |
| 592 | if (array.dtype().kind() == 'S') { |
| 593 | for (int i = 0; i < self->numel(); ++i) { |
| 594 | dst[i] = |
| 595 | pstring(reinterpret_cast<const char *>(array.data()) + itemsize * i, |
| 596 | itemsize); |
| 597 | } |
| 598 | } else { |
| 599 | // array.dtype().kind() == 'U' |
| 600 | VLOG(6) << "numpy array itemsize: " << itemsize; |
| 601 | for (int i = 0; i < self->numel(); ++i) { |
| 602 | // Note(zhoushunjie): The itemsize of unicode numpy array is the |
| 603 | // the size of each unicode string. Each unicode string is aligned |
| 604 | // to max length of the array of unicode strings, so the size of |
| 605 | // each unicode string is same. The size of each unicode character is |
| 606 | // 4, so the size of unicode string is 4 times of the length of |
| 607 | // unicode string. |
| 608 | auto unicode_len = itemsize / 4; |
| 609 | auto utf8_len = phi::strings::GetUTF8StrLen( |
| 610 | reinterpret_cast<const uint32_t *>(array.data()) + unicode_len * i, |
| 611 | unicode_len); |
| 612 | pstring pstr(utf8_len - 1, 0); |
| 613 | phi::strings::GetUTF8Str( |
| 614 | reinterpret_cast<const uint32_t *>(array.data()) + unicode_len * i, |
| 615 | pstr.mdata(), |
| 616 | unicode_len); |
| 617 | dst[i] = pstr; |
| 618 | } |
| 619 | } |
| 620 | } else { |
| 621 | PADDLE_THROW(common::errors::InvalidArgument( |
| 622 | "StringTensor only support CPUPlace now, but receive %s", |
| 623 | place.DebugString())); |
| 624 | } |
| 625 | } |
| 626 | |
| 627 | template <typename T> |
| 628 | void SetUVATensorFromPyArrayImpl( |
nothing calls this directly
no test coverage detected