Method crawl

csdn/selenium_csdn.py:41–65

(self)

        await asyncio.sleep(random.randint(5, 10))

    async def crawl(self):
        # 测试环境下 headless 设置为 False
        # 生产环境可以修改为无头浏览器
        self.browser = await launch({
            'headless': False,
            'userDataDir': cache_dir,
            'defaultViewport': {'width': 1440, 'height': 1000},
            'args': ['--no-sandbox']
        })
        self.page = await self.browser.newPage()
        await self.page.goto(self.url)

        # 伪造当前浏览状态 防止自动化工具检测
        codes = (
            "() =>{ Object.defineProperties(navigator,{ webdriver:"
            "{ get: () => false } }) }",
            "() =>{ window.navigator.chrome = { runtime: {},  }; }",
            "() =>{ Object.defineProperty(navigator, 'languages', "
            "{ get: () => ['en-US', 'en'] }); }",
            "() =>{ Object.defineProperty(navigator, 'plugins', { "
            "get: () => [1, 2, 3, 4, 5,6], }); }"
        )
        for code in codes:
            await self.page.evaluate(code)
        await self.send_key()


def main():