Function extractTextByContentType

sdk/src/tools/read-url.ts:268–307

(
  contentType: string,
  body: string,
)

}

function extractTextByContentType(
  contentType: string,
  body: string,
): {
  title?: string
  description?: string
  text: string
} {
  const lowerContentType = contentType.toLowerCase()

  if (
    lowerContentType.includes('text/html') ||
    lowerContentType.includes('application/xhtml')
  ) {
    return extractHtml(body)
  }

  if (isJsonContentType(lowerContentType)) {
    try {
      return { text: JSON.stringify(JSON.parse(body), null, 2) }
    } catch {
      return { text: normalizeText(body) }
    }
  }

  if (isMarkdownContentType(lowerContentType)) {
    return extractMarkdownFrontmatter(body)
  }

  if (
    lowerContentType.startsWith('text/') ||
    lowerContentType.includes('application/xml') ||
    lowerContentType.includes('application/rss+xml') ||
    lowerContentType.includes('application/atom+xml')
  ) {
    return { text: normalizeText(body) }
  }

  return { text: normalizeText(body) }
}

function truncateText(
  text: string,