Process an RSS/Atom feed into document objects. Args: file_path: Path to the RSS file or URL site: Site identifier Returns: List of document objects
(file_path: str, site: str)
| 498 | |
| 499 | |
| 500 | async def process_rss_feed(file_path: str, site: str) -> list[dict[str, Any]]: |
| 501 | """ |
| 502 | Process an RSS/Atom feed into document objects. |
| 503 | |
| 504 | Args: |
| 505 | file_path: Path to the RSS file or URL |
| 506 | site: Site identifier |
| 507 | |
| 508 | Returns: |
| 509 | List of document objects |
| 510 | """ |
| 511 | print(f"Processing RSS/Atom feed: {file_path}") |
| 512 | |
| 513 | try: |
| 514 | # Convert feed to schema.org format |
| 515 | podcast_episodes = rss2schema.feed_to_schema(file_path) |
| 516 | |
| 517 | documents = [] |
| 518 | |
| 519 | # Process each episode in the feed |
| 520 | for episode in podcast_episodes: |
| 521 | # Extract URL |
| 522 | url = episode.get("url") |
| 523 | |
| 524 | # Generate a synthetic URL if needed |
| 525 | if not url and "name" in episode: |
| 526 | url = f"synthetic:{site}:{episode['name']}" |
| 527 | episode["url"] = url |
| 528 | print(f"Generated synthetic URL for episode: {episode['name']}") |
| 529 | elif not url: |
| 530 | # Skip items without any identifiable information |
| 531 | continue |
| 532 | |
| 533 | # Convert to JSON - ensure no newlines in the JSON |
| 534 | json_data = json.dumps(episode, ensure_ascii=False).replace("\n", " ") |
| 535 | |
| 536 | # Extract name |
| 537 | name = episode.get("name", "Untitled Episode") |
| 538 | |
| 539 | # Create document |
| 540 | document = { |
| 541 | "id": str(hash(url) % (2**63)), # Create a stable ID from the URL |
| 542 | "schema_json": json_data, |
| 543 | "url": url, |
| 544 | "name": name, |
| 545 | "site": site |
| 546 | } |
| 547 | |
| 548 | documents.append(document) |
| 549 | |
| 550 | print(f"Processed {len(documents)} episodes from RSS/Atom feed") |
| 551 | return documents |
| 552 | except Exception as e: |
| 553 | print(f"Error processing RSS/Atom feed: {e!s}") |
| 554 | traceback.print_exc() |
| 555 | return [] |
| 556 | |
| 557 | async def loadJsonWithEmbeddingsToDB(file_path: str, site: str, batch_size: int = 100, delete_existing: bool = False, database: str | None = None): |
no test coverage detected