Function join_by_col

NLP/Text2SQL-BASELINE/tools/evaluation/utils.py:124–139

(toks, cols)

        return new_toks
    
    def join_by_col(toks, cols):
        new_toks = []
        _len = len(toks)
        i = 0 
        while i < _len-1:
            merge = False
            for j in range(10):
                if ''.join(toks[i:i+j]) in cols:
                    new_toks.append(''.join(toks[i:i+j]))
                    i += j
                    merge = True
            if not merge:   
                new_toks.append(toks[i])
                i += 1
        new_toks.append(toks[-1])
        return new_toks

    tokens_tmp = _extract_value(string)