Code Parser Class.
| 17 | |
| 18 | |
| 19 | class CodeParser: |
| 20 | """Code Parser Class.""" |
| 21 | |
| 22 | def __init__(self, language: str, node_types: list[str], path_to_object_file: str): |
| 23 | self.node_types = node_types |
| 24 | self.language = language |
| 25 | try: |
| 26 | self.parser = tree_sitter.Parser() |
| 27 | self.parser.set_language( |
| 28 | tree_sitter.Language(f"{path_to_object_file}/my-languages.so", language) |
| 29 | ) |
| 30 | except Exception as e: |
| 31 | logger.exception("failed to build %s parser: ", e) |
| 32 | |
| 33 | def parse_file(self, content: str, filename: str): |
| 34 | """ |
| 35 | Parse code snippets from single code file. |
| 36 | |
| 37 | Args: |
| 38 | content: The content of the file. |
| 39 | filename: The name of the code file. |
| 40 | |
| 41 | Returns: |
| 42 | List of Parsed Snippets |
| 43 | """ |
| 44 | try: |
| 45 | tree = self.parser.parse(content) |
| 46 | except Exception as e: |
| 47 | logger.error(f"Failed to parse snippet: {filename} \n Error: {e}") |
| 48 | return |
| 49 | |
| 50 | cursor = tree.walk() |
| 51 | parsed_snippets = [] |
| 52 | |
| 53 | # Walking nodes from abstract syntax tree |
| 54 | while cursor.goto_first_child(): |
| 55 | if cursor.node.type in self.node_types: |
| 56 | parsed_snippets.append( |
| 57 | Snippet( |
| 58 | id=str(uuid.uuid4()), |
| 59 | snippet=cursor.node.text, |
| 60 | filename=filename, |
| 61 | language=self.language, |
| 62 | embedding=None, |
| 63 | ) |
| 64 | ) |
| 65 | |
| 66 | while cursor.goto_next_sibling(): |
| 67 | if cursor.node.type in self.node_types: |
| 68 | parsed_snippets.append( |
| 69 | Snippet( |
| 70 | id=str(uuid.uuid4()), |
| 71 | snippet=cursor.node.text, |
| 72 | filename=filename, |
| 73 | language=self.language, |
| 74 | embedding=None, |
| 75 | ) |
| 76 | ) |