Function CopyTokenText

library/cpp/tokenizer/multitokenutil.cpp:69–73

@param len length of text to be copied (it can include suffix length)

    //! @param len    length of text to be copied (it can include suffix length)
    inline void CopyTokenText(wchar16* buffer, const wchar16* entry, size_t len, size_t& destpos, size_t& srcpos) {
        std::char_traits<wchar16>::copy(buffer + destpos, entry + srcpos, len); // srcpos - the beginning of source token
        srcpos += len;
        destpos += len;
    }

    inline bool CutSubtoken(TCharSpan& s, size_t maxLen) {
        if (s.EndPos() > maxLen) {