Test document processing with Google embedder.
()
| 109 | return False |
| 110 | |
| 111 | def test_document_processing(): |
| 112 | """Test document processing with Google embedder.""" |
| 113 | logger.info("Testing document processing with Google embedder...") |
| 114 | |
| 115 | try: |
| 116 | from adalflow.core.types import Document |
| 117 | from adalflow.components.data_process import ToEmbeddings |
| 118 | from api.tools.embedder import get_embedder |
| 119 | |
| 120 | # Create some test documents |
| 121 | docs = [ |
| 122 | Document(text="This is a test document.", meta_data={"file_path": "test1.txt"}), |
| 123 | Document(text="Another test document here.", meta_data={"file_path": "test2.txt"}) |
| 124 | ] |
| 125 | |
| 126 | # Get the Google embedder |
| 127 | embedder = get_embedder(embedder_type='google') |
| 128 | logger.info(f"Embedder type: {type(embedder)}") |
| 129 | |
| 130 | # Process documents |
| 131 | embedder_transformer = ToEmbeddings(embedder=embedder, batch_size=100) |
| 132 | |
| 133 | # Transform documents |
| 134 | logger.info("Transforming documents...") |
| 135 | transformed_docs = embedder_transformer(docs) |
| 136 | |
| 137 | logger.info(f"Transformed docs type: {type(transformed_docs)}") |
| 138 | logger.info(f"Number of transformed docs: {len(transformed_docs)}") |
| 139 | |
| 140 | # Check the structure |
| 141 | for i, doc in enumerate(transformed_docs): |
| 142 | logger.info(f"Doc {i} type: {type(doc)}") |
| 143 | logger.info(f"Doc {i} attributes: {dir(doc)}") |
| 144 | if hasattr(doc, 'vector'): |
| 145 | logger.info(f"Doc {i} vector type: {type(doc.vector)}") |
| 146 | logger.info(f"Doc {i} vector length: {len(doc.vector) if doc.vector else 0}") |
| 147 | else: |
| 148 | logger.info(f"Doc {i} has no vector attribute") |
| 149 | |
| 150 | return transformed_docs |
| 151 | |
| 152 | except Exception as e: |
| 153 | logger.error(f"Error testing document processing: {e}") |
| 154 | import traceback |
| 155 | traceback.print_exc() |
| 156 | return False |
| 157 | |
| 158 | def main(): |
| 159 | """Main test function.""" |