Method parseBuffer

apps/sim/lib/file-parsers/html-parser.ts:24–78

(buffer: Buffer)

  }

  async parseBuffer(buffer: Buffer): Promise<FileParseResult> {
    try {
      logger.info('Parsing HTML buffer, size:', buffer.length)

      const htmlContent = buffer.toString('utf-8')
      const $ = cheerio.load(htmlContent)

      // Extract meta information before removing tags
      const title = $('title').text().trim()
      const metaDescription = $('meta[name="description"]').attr('content') || ''

      $('script, style, noscript, meta, link, iframe, object, embed, svg').remove()

      $.root()
        .contents()
        .filter(function () {
          return this.type === 'comment'
        })
        .remove()

      const content = this.extractStructuredText($)

      const sanitizedContent = sanitizeTextForUTF8(content)

      const characterCount = sanitizedContent.length
      const wordCount = sanitizedContent.split(/\s+/).filter((word) => word.length > 0).length
      const estimatedTokenCount = Math.ceil(characterCount / 4)

      const headings = this.extractHeadings($)

      const links = this.extractLinks($)

      return {
        content: sanitizedContent,
        metadata: {
          title,
          metaDescription,
          characterCount,
          wordCount,
          tokenCount: estimatedTokenCount,
          headings,
          links: links.slice(0, 50),
          hasImages: $('img').length > 0,
          imageCount: $('img').length,
          hasTable: $('table').length > 0,
          tableCount: $('table').length,
          hasList: $('ul, ol').length > 0,
          listCount: $('ul, ol').length,
        },
      }
    } catch (error) {
      logger.error('HTML buffer parsing error:', error)
      throw new Error(`Failed to parse HTML buffer: ${(error as Error).message}`)
    }
  }

  /**
   * Extract structured text content preserving document hierarchy