Method get

optimade/validator/utils.py:204–266

Makes the given request, with a number of retries if being rate limited. The request will be prepended with the `base_url` unless the request appears to be an absolute URL (i.e. starts with `http://` or `https://`). Parameters: request (str): the request to make

(self, request: str)

        self.read_timeout = read_timeout or DEFAULT_READ_TIMEOUT

    def get(self, request: str):
        """Makes the given request, with a number of retries if being rate limited. The
        request will be prepended with the `base_url` unless the request appears to be an
        absolute URL (i.e. starts with `http://` or `https://`).

        Parameters:
            request (str): the request to make against the base URL of this client.

        Returns:
            response (requests.models.Response): the response from the server.

        Raises:
            SystemExit: if there is no response from the server, or if the URL is invalid.
            ResponseError: if the server does not respond with a non-429 status code within
                the `MAX_RETRIES` attempts.

        """
        if urllib.parse.urlparse(request, allow_fragments=True).scheme:
            self.last_request = request
        else:
            if request and not request.startswith("/"):
                request = f"/{request}"
            self.last_request = f"{self.base_url}{request}"

        status_code = None
        retries = 0
        errors = []
        while retries < self.max_retries:
            retries += 1
            try:
                self.response = requests.get(
                    self.last_request,
                    headers=self.headers,
                    timeout=(self.timeout, self.read_timeout),
                )

                status_code = self.response.status_code
                # If we hit a 429 Too Many Requests status, then try again in 1 second
                if status_code != 429:
                    return self.response

            # If the connection times out, retry but cache the error
            except requests.exceptions.ConnectionError as exc:
                errors.append(str(exc))

            # Read timeouts should prevent further retries
            except requests.exceptions.ReadTimeout as exc:
                raise ResponseError(str(exc)) from exc

            except requests.exceptions.MissingSchema:
                sys.exit(
                    f"Unable to make request on {self.last_request}, did you mean http://{self.last_request}?"
                )

            # If the connection failed, or returned a 429, then wait 1 second before retrying
            time.sleep(1)

        else: