Method load_from_url

core/src/document_loader.rs:189–335

Load document from URL

(
        &self,
        url: &str,
        document_type: &str,
    )

    /// Load document from URL
    async fn load_from_url(
        &self,
        url: &str,
        document_type: &str,
    ) -> GraphBitResult<DocumentContent> {
        // Validate URL format
        if !url.starts_with("http://") && !url.starts_with("https://") {
            return Err(GraphBitError::validation(
                "document_loader",
                format!("Invalid URL format: {url}"),
            ));
        }

        // Create HTTP client with timeout and user agent
        let client = reqwest::Client::builder()
            .timeout(std::time::Duration::from_secs(30))
            .user_agent("GraphBit Document Loader/1.0")
            .build()
            .map_err(|e| {
                GraphBitError::validation(
                    "document_loader",
                    format!("Failed to create HTTP client: {e}"),
                )
            })?;

        // Fetch the document
        let response = client.get(url).send().await.map_err(|e| {
            GraphBitError::validation("document_loader", format!("Failed to fetch URL {url}: {e}"))
        })?;

        // Check response status
        if !response.status().is_success() {
            return Err(GraphBitError::validation(
                "document_loader",
                format!("HTTP error {}: {url}", response.status()),
            ));
        }

        // Check content length
        if let Some(content_length) = response.content_length() {
            if content_length as usize > self.config.max_file_size {
                return Err(GraphBitError::validation(
                    "document_loader",
                    format!(
                        "Remote file size ({content_length} bytes) exceeds maximum allowed size ({} bytes)",
                        self.config.max_file_size
                    ),
                ));
            }
        }

        // Get content type from response headers
        let content_type = response
            .headers()
            .get("content-type")
            .and_then(|h| h.to_str().ok())
            .unwrap_or("")
            .to_lowercase();