| 29 | |
| 30 | |
| 31 | class DatabaseOperationsTest: |
| 32 | def __init__(self): |
| 33 | self.test_rss_url = "https://feeds.npr.org/344098539/podcast.xml" |
| 34 | self.test_site = "test_npr_podcast" |
| 35 | self.test_query = "Tom Papa" |
| 36 | self.prod_query = "spicy crunchy snacks" |
| 37 | self.prod_endpoint = "nlweb_west" |
| 38 | |
| 39 | async def download_rss_feed(self) -> str: |
| 40 | """Download RSS feed content""" |
| 41 | print(f"\n📥 Downloading RSS feed from: {self.test_rss_url}") |
| 42 | async with aiohttp.ClientSession() as session, session.get(self.test_rss_url) as response: |
| 43 | content = await response.text() |
| 44 | print(f"✅ Downloaded RSS feed ({len(content)} bytes)") |
| 45 | return content |
| 46 | |
| 47 | async def parse_rss_to_documents_with_embeddings(self, rss_content: str) -> list[dict[str, Any]]: |
| 48 | """Parse RSS content and add embeddings""" |
| 49 | print("\n📄 Parsing RSS feed...") |
| 50 | |
| 51 | # Parse RSS using feedparser |
| 52 | feed = feedparser.parse(rss_content) |
| 53 | documents = [] |
| 54 | |
| 55 | for entry in feed.entries[:10]: # Limit to first 10 entries for testing |
| 56 | doc = { |
| 57 | "url": entry.get("link", ""), |
| 58 | "name": entry.get("title", ""), |
| 59 | "site": self.test_site, |
| 60 | "schema_json": { |
| 61 | "@type": "PodcastEpisode", |
| 62 | "name": entry.get("title", ""), |
| 63 | "description": entry.get("summary", ""), |
| 64 | "url": entry.get("link", ""), |
| 65 | "datePublished": entry.get("published", "") |
| 66 | } |
| 67 | } |
| 68 | documents.append(doc) |
| 69 | |
| 70 | print(f"✅ Parsed {len(documents)} episodes from RSS feed") |
| 71 | |
| 72 | # Generate embeddings |
| 73 | print(f"\n🔢 Generating embeddings for {len(documents)} documents...") |
| 74 | texts_to_embed = [] |
| 75 | for doc in documents: |
| 76 | # Create text from title and description |
| 77 | title = doc["schema_json"].get("name", "") |
| 78 | description = doc["schema_json"].get("description", "") |
| 79 | text = f"{title} {description}" |
| 80 | texts_to_embed.append(text) |
| 81 | |
| 82 | # Get embeddings in batch |
| 83 | embeddings = await batch_get_embeddings(texts_to_embed) |
| 84 | |
| 85 | # Add embeddings to documents |
| 86 | for i, doc in enumerate(documents): |
| 87 | if i < len(embeddings): |
| 88 | doc["embedding"] = embeddings[i] |