Function extractHtml

sdk/src/tools/read-url.ts:177–221

(html: string)

}

function extractHtml(html: string): {
  title?: string
  description?: string
  text: string
} {
  const title = extractFirstMatch(html, /<title\b[^>]*>([\s\S]*?)<\/title>/i)
  const description =
    extractMetaContent(html, 'description') ??
    extractMetaContent(html, 'og:description')

  let readable = html
    .replace(/<!--[\s\S]*?-->/g, '\n')
    .replace(/<!doctype[^>]*>/gi, '\n')

  for (const tagName of [
    'script',
    'style',
    'svg',
    'canvas',
    'iframe',
    'noscript',
    'nav',
    'header',
    'footer',
    'form',
    'button',
    'select',
  ]) {
    readable = removeElement(readable, tagName)
  }

  readable = selectReadableHtml(readable)

  readable = readable
    .replace(/<br\s*\/?>/gi, '\n')
    .replace(
      /<\/(p|div|section|article|main|aside|li|tr|td|th|h[1-6]|blockquote|pre)>/gi,
      '\n',
    )
    .replace(/<(li|tr|h[1-6])\b[^>]*>/gi, '\n')
    .replace(/<[^>]*>/g, '')

  const text = normalizeText(decodeHtmlEntities(readable))
  return { title, description, text }
}

function extractMarkdownFrontmatter(body: string): {
  title?: string