Class SpiderLoader

packages/components/nodes/documentloaders/Spider/Spider.ts:18–75

}

class SpiderLoader extends BaseDocumentLoader {
    private apiKey: string
    private url: string
    private mode: 'crawl' | 'scrape'
    private limit?: number
    private additionalMetadata?: Record<string, unknown>
    private params?: Record<string, unknown>

    constructor(loaderParams: SpiderLoaderParameters) {
        super()
        const { apiKey, url, mode = 'crawl', limit, additionalMetadata, params } = loaderParams
        if (!apiKey) {
            throw new Error('Spider API key not set. You can set it as SPIDER_API_KEY in your .env file, or pass it to Spider.')
        }

        this.apiKey = apiKey
        this.url = url
        this.mode = mode
        this.limit = Number(limit)
        this.additionalMetadata = additionalMetadata
        this.params = params
    }

    public async load(): Promise<DocumentInterface[]> {
        const app = new SpiderApp({ apiKey: this.apiKey })
        let spiderDocs: any[]

        if (this.mode === 'scrape') {
            const response = await app.scrapeUrl(this.url, this.params)
            if (!response.success) {
                throw new Error(`Spider: Failed to scrape URL. Error: ${response.error}`)
            }
            spiderDocs = [response.data]
        } else if (this.mode === 'crawl') {
            if (this.params) {
                this.params.limit = this.limit
            }
            const response = await app.crawlUrl(this.url, this.params)
            if (!response.success) {
                throw new Error(`Spider: Failed to crawl URL. Error: ${response.error}`)
            }
            spiderDocs = response.data
        } else {
            throw new Error(`Unrecognized mode '${this.mode}'. Expected one of 'crawl', 'scrape'.`)
        }

        return spiderDocs.map(
            (doc) =>
                new Document({
                    pageContent: doc.content || '',
                    metadata: {
                        ...(this.additionalMetadata || {}),
                        source: doc.url
                    }
                })
        )
    }
}