Function readUrl

sdk/src/tools/read-url.ts:325–413

({
  url,
  max_chars = DEFAULT_MAX_CHARS,
  fetch: fetchImpl = globalThis.fetch,
}: {
  url: string
  max_chars?: number
  fetch?: FetchLike
})

}

export async function readUrl({
  url,
  max_chars = DEFAULT_MAX_CHARS,
  fetch: fetchImpl = globalThis.fetch,
}: {
  url: string
  max_chars?: number
  fetch?: FetchLike
}): Promise<ReadUrlOutput> {
  let parsedUrl: URL
  try {
    parsedUrl = new URL(url)
  } catch {
    return errorResult(url, 'Invalid URL')
  }

  if (!isAllowedUrl(parsedUrl)) {
    return errorResult(url, 'Only http:// and https:// URLs are supported')
  }

  const controller = new AbortController()
  const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS)

  try {
    const response = await fetchImpl(parsedUrl.toString(), {
      redirect: 'follow',
      signal: controller.signal,
      headers: {
        accept:
          'text/html,application/xhtml+xml,application/json,text/plain;q=0.9,*/*;q=0.8',
        'accept-language': 'en-US,en;q=0.9',
        'user-agent': USER_AGENT,
      },
    })

    if (!response.ok) {
      return errorResult(
        url,
        `Failed to fetch URL: ${response.status} ${response.statusText}`,
      )
    }

    const contentType = getHeader(response.headers, 'content-type') ?? ''
    if (contentType && !isSupportedContentType(contentType)) {
      return errorResult(
        url,
        `Unsupported content type: ${contentType || 'unknown'}`,
      )
    }

    const body = await readResponseBody(response, MAX_RESPONSE_BYTES)
    const extracted = extractTextByContentType(contentType, body)
    const truncated = truncateText(extracted.text, max_chars)

    if (!truncated.text) {
      return errorResult(url, 'No readable text found at URL')
    }