Function htmlToMarkdown

src/tools/web/web-fetch.ts:107–135

Best-effort HTML→markdown. Captures headings, links, lists, code, paragraphs.

(html: string)

/** Best-effort HTML→markdown. Captures headings, links, lists, code, paragraphs. */
function htmlToMarkdown(html: string): string {
  html = html.replace(/<script[\s\S]*?<\/script>/gi, '');
  html = html.replace(/<style[\s\S]*?<\/style>/gi, '');
  html = html.replace(/<!--[\s\S]*?-->/g, '');
  // Headings
  for (let n = 1; n <= 6; n++) {
    const re = new RegExp(`<h${n}[^>]*>([\\s\\S]*?)</h${n}>`, 'gi');
    html = html.replace(re, (_m, txt) => `\n\n${'#'.repeat(n)} ${stripInner(txt).trim()}\n`);
  }
  // Code blocks
  html = html.replace(/<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, (_m, code) => `\n\n\`\`\`\n${decodeEntities(code)}\n\`\`\`\n`);
  html = html.replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, (_m, code) => `\`${decodeEntities(code)}\``);
  // Links
  html = html.replace(/<a [^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, (_m, href, txt) => `[${stripInner(txt).trim()}](${href})`);
  // Lists
  html = html.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, (_m, txt) => `- ${stripInner(txt).trim()}\n`);
  // Bold/italic
  html = html.replace(/<(?:b|strong)[^>]*>([\s\S]*?)<\/(?:b|strong)>/gi, '**$1**');
  html = html.replace(/<(?:i|em)[^>]*>([\s\S]*?)<\/(?:i|em)>/gi, '*$1*');
  // Paragraphs/divs/breaks
  html = html.replace(/<\/(p|div)[^>]*>/gi, '\n\n');
  html = html.replace(/<br[^>]*>/gi, '\n');
  // Strip remaining tags
  html = html.replace(/<[^>]+>/g, '');
  html = decodeEntities(html);
  // Cleanup whitespace
  html = html.replace(/[ \t]+/g, ' ').replace(/\n{3,}/g, '\n\n');
  return html.trim();
}

function stripInner(s: string): string {
  return s.replace(/<[^>]+>/g, '');