(self, task, tmp_dir)
| 1773 | browser.close() |
| 1774 | |
| 1775 | def _scrape_github(self, task, tmp_dir): |
| 1776 | opts = task['options'] |
| 1777 | tutto = opts.get('tutto', False) |
| 1778 | target = task['target'] |
| 1779 | |
| 1780 | with sync_playwright() as p: |
| 1781 | browser = p.chromium.launch(headless=True, args=["--disable-blink-features=AutomationControlled"]) |
| 1782 | context = browser.new_context(user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") |
| 1783 | page = context.new_page() |
| 1784 | |
| 1785 | page.goto(f"https://github.com/{target}", wait_until="domcontentloaded") |
| 1786 | task['progress'] = 20 |
| 1787 | time.sleep(3) |
| 1788 | |
| 1789 | if tutto or opts.get('dati', False): |
| 1790 | try: |
| 1791 | info = {"username": target} |
| 1792 | name_el = page.locator("span.p-name") |
| 1793 | if name_el.count() > 0: info['name'] = name_el.first.inner_text() |
| 1794 | |
| 1795 | bio_el = page.locator("div.p-note") |
| 1796 | if bio_el.count() > 0: info['bio'] = bio_el.first.inner_text() |
| 1797 | |
| 1798 | org_el = page.locator("span.p-org") |
| 1799 | if org_el.count() > 0: info['organization'] = org_el.first.inner_text() |
| 1800 | |
| 1801 | with open(os.path.join(tmp_dir, "info.json"), "w", encoding="utf-8") as f: |
| 1802 | safe_json_dump(info, f, indent=4, ensure_ascii=False) |
| 1803 | except: pass |
| 1804 | |
| 1805 | task['progress'] = 40 |
| 1806 | |
| 1807 | # Extract public emails from recent commits |
| 1808 | emails = set() |
| 1809 | try: |
| 1810 | import requests |
| 1811 | r = requests.get(f"https://api.github.com/users/{target}/events/public") |
| 1812 | if r.status_code == 200: |
| 1813 | events = r.json() |
| 1814 | for ev in events: |
| 1815 | if ev.get("type") == "PushEvent": |
| 1816 | commits = ev.get("payload", {}).get("commits", []) |
| 1817 | for c in commits: |
| 1818 | author = c.get("author", {}) |
| 1819 | email = author.get("email") |
| 1820 | if email and "noreply.github.com" not in email: |
| 1821 | emails.add(email) |
| 1822 | |
| 1823 | if emails: |
| 1824 | with open(os.path.join(tmp_dir, "extracted_emails.txt"), "w", encoding="utf-8") as f: |
| 1825 | for e in emails: f.write(e + "\n") |
| 1826 | except: pass |
| 1827 | |
| 1828 | task['progress'] = 60 |
| 1829 | |
| 1830 | if tutto or opts.get('media', False): |
| 1831 | os.makedirs(os.path.join(tmp_dir, "media"), exist_ok=True) |
| 1832 | try: |
no test coverage detected