MCPcopy Create free account
hub / github.com/apify/crawlee-python / test_parent_sitemap

Function test_parent_sitemap

tests/unit/_utils/test_sitemap.py:185–213  ·  view source on GitHub ↗

Test loading a parent sitemap that references child sitemaps.

(server_url: URL, http_client: HttpClient)

Source from the content-addressed store, hash-verified

183
184
185async def test_parent_sitemap(server_url: URL, http_client: HttpClient) -> None:
186 """Test loading a parent sitemap that references child sitemaps."""
187 parent_sitemap = """
188<?xml version="1.0" encoding="UTF-8"?>
189<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
190<sitemap>
191<loc>{child_sitemap}</loc>
192<lastmod>2004-12-23</lastmod>
193</sitemap>
194<sitemap>
195<loc>{child_sitemap_2}</loc>
196<lastmod>2004-12-23</lastmod>
197</sitemap>
198</sitemapindex>
199""".strip()
200 child_sitemap = (server_url / 'sitemap.xml').with_query(
201 base64=encode_base64(get_basic_sitemap(url=server_url).encode())
202 )
203 child_sitemap_2 = (server_url / 'sitemap.xml.gz').with_query(
204 base64=encode_base64(compress_gzip(get_basic_sitemap(url=server_url)))
205 )
206 parent_sitemap_content = parent_sitemap.format(child_sitemap=child_sitemap, child_sitemap_2=child_sitemap_2)
207 encoded_parent_sitemap_content = encode_base64(parent_sitemap_content.encode())
208 parent_sitemap_url = (server_url / 'sitemap.xml').with_query(base64=encoded_parent_sitemap_content)
209
210 sitemap = await Sitemap.load(str(parent_sitemap_url), http_client=http_client)
211
212 assert len(sitemap.urls) == 10
213 assert set(sitemap.urls) == get_basic_results(server_url)
214
215
216async def test_non_sitemap_url(server_url: URL, http_client: HttpClient) -> None:

Callers

nothing calls this directly

Calls 6

get_basic_sitemapFunction · 0.90
get_basic_resultsFunction · 0.90
formatMethod · 0.80
encode_base64Function · 0.70
compress_gzipFunction · 0.70
loadMethod · 0.45

Tested by

no test coverage detected