Function GlueTokens

library/cpp/tokenizer/multitokenutil.cpp:5–11

namespace {
    inline void GlueTokens(TCharSpan* desttok, const TCharSpan* srctok) {
        Y_ASSERT(srctok->PrefixLen == 0);
        desttok->Len += srctok->Len;
        desttok->SuffixLen = srctok->SuffixLen;
        desttok->Hyphen = srctok->Hyphen; // the next token can have hyphenation
        desttok->TokenDelim = srctok->TokenDelim;
    }

    inline void CopyToken(TCharSpan* desttok, const TCharSpan* srctok, size_t destpos) {
        Y_ASSERT(srctok->PrefixLen == 0);