Load data from a file with precomputed embeddings into the database. The file should have 3 columns per row, separated by tabs: 1. URL for the item 2. JSON for the item 3. Embedding for the item Args: file_path: Path to the input file (URL, JSON, embedding)
(file_path: str, site: str, batch_size: int = 100, delete_existing: bool = False, database: str | None = None)
| 555 | return [] |
| 556 | |
| 557 | async def loadJsonWithEmbeddingsToDB(file_path: str, site: str, batch_size: int = 100, delete_existing: bool = False, database: str | None = None): |
| 558 | """ |
| 559 | Load data from a file with precomputed embeddings into the database. |
| 560 | |
| 561 | The file should have 3 columns per row, separated by tabs: |
| 562 | 1. URL for the item |
| 563 | 2. JSON for the item |
| 564 | 3. Embedding for the item |
| 565 | |
| 566 | Args: |
| 567 | file_path: Path to the input file (URL, JSON, embedding) |
| 568 | site: Site identifier |
| 569 | batch_size: Number of documents to process and upload in each batch |
| 570 | delete_existing: Whether to delete existing entries for this site before loading |
| 571 | database: Specific database endpoint to use (if None, uses preferred endpoint) |
| 572 | """ |
| 573 | # Check if this is a URL |
| 574 | is_url_path = await is_url(file_path) |
| 575 | temp_path = None |
| 576 | |
| 577 | if is_url_path: |
| 578 | temp_path, _ = await save_url_content(file_path) |
| 579 | file_path = temp_path |
| 580 | print(f"file_path: {file_path}") |
| 581 | try: |
| 582 | resolved_path = None |
| 583 | |
| 584 | # First, check if the file exists at the given path |
| 585 | if os.path.exists(file_path): |
| 586 | # Check if this file actually contains embeddings |
| 587 | _file_type, has_embeddings = await detect_file_type(file_path) |
| 588 | if has_embeddings: |
| 589 | # If the file exists and has embeddings, use it directly |
| 590 | print(f"File exists and contains embeddings, using directly: {file_path}") |
| 591 | resolved_path = file_path |
| 592 | else: |
| 593 | print(f"File exists but doesn't contain embeddings: {file_path}") |
| 594 | |
| 595 | # If we haven't resolved a path yet, try standard resolution methods |
| 596 | if resolved_path is None: |
| 597 | embeddings_folder = CONFIG.nlweb.json_with_embeddings_folder |
| 598 | |
| 599 | # If the path already starts with the embeddings folder, use it directly |
| 600 | if file_path.startswith(embeddings_folder): |
| 601 | resolved_path = file_path |
| 602 | else: |
| 603 | # Otherwise, it might be just a filename or relative path, so resolve it |
| 604 | embeddings_path = get_embeddings_file_path(file_path) |
| 605 | |
| 606 | # Check if the resolved embeddings path exists |
| 607 | if os.path.exists(embeddings_path): |
| 608 | resolved_path = embeddings_path |
| 609 | else: |
| 610 | # Last resort: Check if the file exists in the base folder |
| 611 | if os.path.exists(file_path): |
| 612 | resolved_path = file_path |
| 613 | else: |
| 614 | # If still not found, use the standard embeddings path (which might not exist) |
no test coverage detected