()
| 72 | |
| 73 | |
| 74 | async def test_parse_from_content() -> None: |
| 75 | content = """User-agent: * |
| 76 | Disallow: *deny_all/ |
| 77 | crawl-delay: 10 |
| 78 | User-agent: Googlebot |
| 79 | Disallow: *deny_googlebot/""" |
| 80 | robots = await RobotsTxtFile.from_content('http://not-exists.com/robots.txt', content) |
| 81 | assert robots.is_allowed('http://not-exists.com/something/page.html') |
| 82 | assert robots.is_allowed('http://not-exists.com/deny_googlebot/page.html') |
| 83 | assert not robots.is_allowed('http://not-exists.com/deny_googlebot/page.html', 'Googlebot') |
| 84 | assert not robots.is_allowed('http://not-exists.com/deny_all/page.html') |
| 85 | |
| 86 | |
| 87 | async def test_bind_robots_txt_url() -> None: |
nothing calls this directly
no test coverage detected