Method preprocess

src/SourceData.cpp:568–722

Prepare the linedata vector for every input line.*/

/** Prepare the linedata vector for every input line.*/
bool SourceData::FileData::preprocess(const QByteArray& encoding, bool removeComments)
{
    if(m_pBuf == nullptr)
        return true;

    QString line;
    QString curChar, prevChar;
    LineType lines = 0;
    qsizetype lastOffset = 0;
    std::unique_ptr<CommentParser> parser(new DefaultCommentParser());

    // detect line end style
    std::vector<e_LineEndStyle> vOrigDataLineEndStyle;
    m_eLineEndStyle = eLineEndStyleUndefined;

    QByteArray pCodec = detectEncoding(m_pBuf.get(), mDataSize).value_or(encoding);

    if(mDataSize > limits<qint32>::max())
    {
        reset();
        return false;
    }

    try
    {
        const QByteArray ba = QByteArray::fromRawData(m_pBuf.get(), (qsizetype)(mDataSize));
        EncodedDataStream ds(ba);

        ds.setEncoding(encoding);
        mHasBOM = ds.hasBOM();
        m_bIncompleteConversion = false;
        m_unicodeBuf->clear();

        assert(m_unicodeBuf->length() == 0);

        mHasEOLTermination = false;
        while(!ds.atEnd())
        {
            line.clear();
            if(lines >= limits<LineType>::max() - 5)
            {
                reset();
                return false;
            }

            prevChar = curChar;
            ds.readChar(curChar);

            qsizetype firstNonwhite = 0;
            bool foundNonWhite = false;

            while(curChar != '\n' && curChar != '\r')
            {
                if(curChar[0].isNull() || curChar[0].isNonCharacter())
                {
                    m_v->clear();
                    return true;
                }