Process text file with encoding handling
(file_path: str)
| 47 | |
| 48 | # Text Processing Function |
| 49 | async def process_text(file_path: str) -> str: |
| 50 | """Process text file with encoding handling""" |
| 51 | if not os.path.exists(file_path): |
| 52 | raise FileNotFoundError(f"Text file not found: {file_path}") |
| 53 | |
| 54 | try: |
| 55 | # Try UTF-8 first, fallback to other encodings |
| 56 | encodings = ['utf-8', 'ascii', 'iso-8859-1', 'cp1252'] |
| 57 | text = None |
| 58 | |
| 59 | for encoding in encodings: |
| 60 | try: |
| 61 | with open(file_path, "r", encoding=encoding) as f: |
| 62 | text = f.read() |
| 63 | break |
| 64 | except UnicodeDecodeError: |
| 65 | continue |
| 66 | |
| 67 | if text is None: |
| 68 | raise UnicodeError("Failed to decode file with any supported encoding") |
| 69 | |
| 70 | # Clean and normalize text |
| 71 | text = text.encode('ascii', 'ignore').decode('ascii') |
| 72 | return text.strip() |
| 73 | |
| 74 | except Exception as e: |
| 75 | logger.exception("An error occurred:", exc_info=True) |
| 76 | raise |
| 77 | |
| 78 | async def streaming_query_loop(rag: GraphRAG): |
| 79 | """Basic query loop for repeated questions""" |