(url)
| 12 | |
| 13 | @backoff.on_exception(backoff.expo,(Exception,), max_tries=3, max_value=10, jitter=None) |
| 14 | def langchain_parse_url(url): |
| 15 | |
| 16 | print(">" * 30 + "Running langchain_parse_url" + ">" * 30) |
| 17 | start = time.time() |
| 18 | loader = PlaywrightURLLoader(urls = [url], remove_selectors=["header", "footer"]) |
| 19 | data = loader.load() |
| 20 | if len(data) <= 0: |
| 21 | return None |
| 22 | content = data[0].page_content |
| 23 | |
| 24 | if "Please enable cookies" in content: |
| 25 | return None |
| 26 | if "Please verify you are a human" in content: |
| 27 | return None |
| 28 | if "Checking if the site connection is secure" in content: |
| 29 | return None |
| 30 | |
| 31 | print("url: {} | content: {}".format(url, content[:100].split("\n"))) |
| 32 | end = time.time() |
| 33 | print(">" * 30 + "Time elapsed: {}s".format(end - start) + ">" * 30) |
| 34 | print("<" * 30 + "Finish langchain_parse_url" + "<" * 30) |
| 35 | return content |
| 36 | |
| 37 | def my_rank(x): |
| 38 | return pd.Series(x).rank(pct=True).iloc[-1] |
nothing calls this directly
no outgoing calls
no test coverage detected