Function clean_docstring

src/codetext/clean/noise_removal.py:841–887

Clean docstring by removing special tag/url, characters, unrelevant information

(docstring: str, loosen_filter: bool = False)

def clean_docstring(docstring: str, loosen_filter: bool = False):
    """
    Clean docstring by removing special tag/url, characters, unrelevant information
    """
    cleaned_docstring = []
    if docstring == '' or docstring == None:
        return None
    _docstring = remove_comment_delimiters(docstring)
    if check_docstring_literal(_docstring):  # True is not pass
        return None #, [f"<check_docstring_literal> {docstring}"]

    # _docstring = '\n'.join(remove_comment_delimiters(docstring))
    docstring_paragraph_list = _docstring.strip().split('\n\n')
    
    for para in docstring_paragraph_list:
        docs = remove_unrelevant(para)
        docstring_list = re.split(r'(?<=.)[.!\?](?=\s+)', docs, flags=re.M)
        clean_line = []
        for line in docstring_list:
            try:
                line = remove_special_tag(line)
            except:
                print('Oops')
                return None
            
            # not_pass, res = check_docstring(line, loosen_filter)
            not_pass = check_docstring(line, loosen_filter)
            if not not_pass:
                clean_line.append(line)
            else:
                break
        
        if len(clean_line) < len(docstring_list):
            clean_line.append('')
        cleaned_docstring.append('.'.join(clean_line))
        

    cleaned_docstring = '\n\n'.join(cleaned_docstring)

    
    if check_docstring_length(cleaned_docstring):
        # if not res:
        #     return None #, [f"<check_docstring_length> {docstring}"]
        # else:
        return None #, res
    
    return cleaned_docstring #, res

if __name__ == '__main__':
    # test remove comment delimiters