(*args, **kwargs)
| 44 | |
| 45 | @wraps(func) |
| 46 | def wrapper_requests(*args, **kwargs) -> Any: |
| 47 | print_running() |
| 48 | nonlocal parallel, data, cache, expires_in, cache_storage, beep, run_async, async_queue, metadata |
| 49 | nonlocal proxy, user_agent, close_on_crash, output, output_formats, max_retry, retry_wait, must_raise_exceptions, raise_exception, create_error_logs |
| 50 | |
| 51 | parallel = kwargs.get("parallel", parallel) |
| 52 | data = kwargs.get("data", data) |
| 53 | cache = kwargs.get("cache", cache) |
| 54 | expires_in = kwargs.get("expires_in", expires_in) |
| 55 | storage = kwargs.get("cache_storage", cache_storage) or FileCacheStorage |
| 56 | beep = kwargs.get("beep", beep) |
| 57 | run_async = kwargs.get("run_async", run_async) |
| 58 | metadata = kwargs.get("metadata", metadata) |
| 59 | async_queue = kwargs.get("async_queue", async_queue) |
| 60 | proxy = kwargs.get("proxy", proxy) |
| 61 | user_agent = kwargs.get("user_agent", user_agent) |
| 62 | close_on_crash = kwargs.get("close_on_crash", close_on_crash) |
| 63 | output = kwargs.get("output", output) |
| 64 | output_formats = kwargs.get("output_formats", output_formats) |
| 65 | max_retry = kwargs.get("max_retry", max_retry) |
| 66 | retry_wait = kwargs.get("retry_wait", retry_wait) |
| 67 | # A Special Option passed by botasaurus server which prevents caching at database level |
| 68 | return_dont_cache_as_is = kwargs.get("return_dont_cache_as_is", False) |
| 69 | must_raise_exceptions = kwargs.get( |
| 70 | "must_raise_exceptions", must_raise_exceptions |
| 71 | ) |
| 72 | create_error_logs = kwargs.get("create_error_logs", create_error_logs) |
| 73 | |
| 74 | raise_exception = kwargs.get("raise_exception", raise_exception) |
| 75 | |
| 76 | fn_name = func.__name__ |
| 77 | |
| 78 | if isinstance(proxy, list): |
| 79 | from itertools import cycle |
| 80 | cycled_proxy = cycle(proxy) |
| 81 | else: |
| 82 | cycled_proxy = None |
| 83 | def run_task( |
| 84 | data, |
| 85 | retry_attempt, |
| 86 | ) -> Any: |
| 87 | if cache is True: |
| 88 | # Returns {"data": value} or None |
| 89 | cached = storage.get(fn_name, data, expires_in) |
| 90 | if cached is not None: |
| 91 | return cached["data"] |
| 92 | |
| 93 | |
| 94 | if cycled_proxy: |
| 95 | evaluated_proxy = next(cycled_proxy) |
| 96 | else: |
| 97 | evaluated_proxy = evaluate_proxy(proxy(data) if callable(proxy) else proxy) |
| 98 | evaluated_user_agent = ( |
| 99 | user_agent(data) if callable(user_agent) else user_agent |
| 100 | ) |
| 101 | |
| 102 | reqs = create_request( |
| 103 | evaluated_proxy, evaluated_user_agent |
no test coverage detected