Split python code to class, function and annotation.
(filepath: str, text: str, metadata: dict = {})
| 627 | return text_chunks + image_chunks |
| 628 | |
| 629 | def split_python_code(filepath: str, text: str, metadata: dict = {}): |
| 630 | """Split python code to class, function and annotation.""" |
| 631 | basename = os.path.basename(filepath) |
| 632 | texts = [] |
| 633 | texts.append(basename) |
| 634 | try: |
| 635 | node = ast.parse(text) |
| 636 | data = ast.get_docstring(node) |
| 637 | if data: |
| 638 | texts.append(data) |
| 639 | for child_node in ast.walk(node): |
| 640 | if isinstance( |
| 641 | child_node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) |
| 642 | ): |
| 643 | data = ast.get_docstring(child_node) |
| 644 | if data: |
| 645 | texts.append(f"{child_node.name} {data}") |
| 646 | except Exception as e: |
| 647 | logger.error('{} {}, continue'.format(filepath, str(e))) |
| 648 | chunks = [] |
| 649 | for text in texts: |
| 650 | chunks.append(Chunk(content_or_path=text, metadata=metadata)) |
| 651 | return chunks |
| 652 | |
| 653 | def clean_md(text: str): |
| 654 | """Remove parts of the markdown document that do not contain the key |