MCPcopy Create free account
hub / github.com/Sally-SH/VSP-LLM / __Pyx_PyUnicode_Join

Function __Pyx_PyUnicode_Join

fairseq/fairseq/data/data_utils_fast.cpp:25510–25576  ·  view source on GitHub ↗

JoinPyUnicode */

Source from the content-addressed store, hash-verified

25508
25509/* JoinPyUnicode */
25510static PyObject* __Pyx_PyUnicode_Join(PyObject* value_tuple, Py_ssize_t value_count, Py_ssize_t result_ulength,
25511 Py_UCS4 max_char) {
25512#if CYTHON_USE_UNICODE_INTERNALS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS
25513 PyObject *result_uval;
25514 int result_ukind, kind_shift;
25515 Py_ssize_t i, char_pos;
25516 void *result_udata;
25517 CYTHON_MAYBE_UNUSED_VAR(max_char);
25518#if CYTHON_PEP393_ENABLED
25519 result_uval = PyUnicode_New(result_ulength, max_char);
25520 if (unlikely(!result_uval)) return NULL;
25521 result_ukind = (max_char <= 255) ? PyUnicode_1BYTE_KIND : (max_char <= 65535) ? PyUnicode_2BYTE_KIND : PyUnicode_4BYTE_KIND;
25522 kind_shift = (result_ukind == PyUnicode_4BYTE_KIND) ? 2 : result_ukind - 1;
25523 result_udata = PyUnicode_DATA(result_uval);
25524#else
25525 result_uval = PyUnicode_FromUnicode(NULL, result_ulength);
25526 if (unlikely(!result_uval)) return NULL;
25527 result_ukind = sizeof(Py_UNICODE);
25528 kind_shift = (result_ukind == 4) ? 2 : result_ukind - 1;
25529 result_udata = PyUnicode_AS_UNICODE(result_uval);
25530#endif
25531 assert(kind_shift == 2 || kind_shift == 1 || kind_shift == 0);
25532 char_pos = 0;
25533 for (i=0; i < value_count; i++) {
25534 int ukind;
25535 Py_ssize_t ulength;
25536 void *udata;
25537 PyObject *uval = PyTuple_GET_ITEM(value_tuple, i);
25538 if (unlikely(__Pyx_PyUnicode_READY(uval)))
25539 goto bad;
25540 ulength = __Pyx_PyUnicode_GET_LENGTH(uval);
25541 if (unlikely(!ulength))
25542 continue;
25543 if (unlikely((PY_SSIZE_T_MAX >> kind_shift) - ulength < char_pos))
25544 goto overflow;
25545 ukind = __Pyx_PyUnicode_KIND(uval);
25546 udata = __Pyx_PyUnicode_DATA(uval);
25547 if (!CYTHON_PEP393_ENABLED || ukind == result_ukind) {
25548 memcpy((char *)result_udata + (char_pos << kind_shift), udata, (size_t) (ulength << kind_shift));
25549 } else {
25550 #if PY_VERSION_HEX >= 0x030d0000
25551 if (unlikely(PyUnicode_CopyCharacters(result_uval, char_pos, uval, 0, ulength) < 0)) goto bad;
25552 #elif CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030300F0 || defined(_PyUnicode_FastCopyCharacters)
25553 _PyUnicode_FastCopyCharacters(result_uval, char_pos, uval, 0, ulength);
25554 #else
25555 Py_ssize_t j;
25556 for (j=0; j < ulength; j++) {
25557 Py_UCS4 uchar = __Pyx_PyUnicode_READ(ukind, udata, j);
25558 __Pyx_PyUnicode_WRITE(result_ukind, result_udata, char_pos+j, uchar);
25559 }
25560 #endif
25561 }
25562 char_pos += ulength;
25563 }
25564 return result_uval;
25565overflow:
25566 PyErr_SetString(PyExc_OverflowError, "join() result is too long for a Python string");
25567bad:

Callers 4

_unellipsifyFunction · 0.70
__pyx_pybuffer_indexFunction · 0.70

Calls 1

CYTHON_UNUSED_VARFunction · 0.70

Tested by

no test coverage detected