Function ProcessHyphenation

library/cpp/tokenizer/multitokenutil.cpp:24–66

processes hyphenation between subtokens if any

    //! processes hyphenation between subtokens if any
    inline void ProcessHyphenation(wchar16* buffer, const wchar16* entry, TCharSpan*& desttok, const TCharSpan* srctok, size_t& destpos, size_t& srcpos) {
        // Check if the space contains hyphenation, and copy the separator
        if (desttok->Hyphen == HYPHEN_ORDINARY) {
            if (desttok->Type == TOKEN_WORD && srctok->Type == TOKEN_WORD) {
                // glue two adjacent tokens if both are TOKEN_WORD
                srcpos = srctok->Pos;
                GlueTokens(desttok, srctok);
            } else {
                // copy delimiter only - remove all following whitespace characters, for example: "sepa- \n rator" (see nlptok.rl, hyphenation/nlfasblank)
                wchar16* const dest = buffer + destpos;
                const wchar16* const src = entry + srcpos; // srcpos - the beginning of source delimiter
                *dest = *src;
                desttok->TokenDelim = TOKDELIM_MINUS;
                ++destpos;
                srcpos = srctok->Pos; // keep it before CopyToken()
                CopyToken(++desttok, srctok, destpos);
            }
        } else if (desttok->Hyphen == HYPHEN_SOFT) {
            if (desttok->Type == srctok->Type) {
                // glue two adjacent tokens if both have the same type
                srcpos = srctok->Pos;
                GlueTokens(desttok, srctok);
            } else {
                // delimiter isn't copied
                srcpos = srctok->Pos; // keep it before CopyToken()
                CopyToken(++desttok, srctok, destpos);
            }
        } else {
            Y_ASSERT(desttok->Hyphen == HYPHEN_NONE); // HYPHEN_HARD isn't processed yet
            Y_ASSERT(srctok->Pos >= srcpos);
            if (srctok->Pos > srcpos) {
                // copy the delimiter and create the next token
                const size_t seplen = srctok->Pos - srcpos; // srcpos - the beginning of source delimiter

                // copy suffix with delimiter
                std::char_traits<wchar16>::copy(buffer + destpos, entry + srcpos, seplen);
                destpos += seplen;

                srcpos = srctok->Pos; // keep it before CopyToken()
            }
            CopyToken(++desttok, srctok, destpos);
        }
    }

    //! @param len    length of text to be copied (it can include suffix length)
    inline void CopyTokenText(wchar16* buffer, const wchar16* entry, size_t len, size_t& destpos, size_t& srcpos) {