Method fetch

backend/src/http/page.rs:34–59

(&self)

    }

    pub fn fetch(&self) -> Result<Response> {
        if !self.should_fetch() {
            log::info!(
                "document not suitable for fetching, skipping fetch: {}",
                self.0
            );
            return Ok(Response {
                url: self.0.clone(),
                body: Html::parse_document("<title>Missing title [pdf]</title>"),
            });
        }

        log::info!("fetching page: {}", self.0);
        let text = ureq::get(&self.0.normalized)
            .timeout(Duration::from_secs(20_000))
            .set("User-Agent", USER_AGENT)
            .call()?
            .into_string()?;
        let body = Html::parse_fragment(text.as_ref());

        log::info!("page fetched: {}", self.0);
        Ok(Response {
            url: self.0.clone(),
            body,
        })
    }

    pub fn should_fetch(&self) -> bool {
        !self.0.is_pdf() && !self.0.avoid_requests()