Function fetchNewRssItems

apps/sim/lib/webhooks/polling/rss.ts:185–286

(
  config: RssWebhookConfig,
  requestId: string,
  logger: Logger
)

}

async function fetchNewRssItems(
  config: RssWebhookConfig,
  requestId: string,
  logger: Logger
): Promise<{ feed: RssFeed; items: RssItem[]; etag?: string; lastModified?: string }> {
  try {
    const urlValidation = await validateUrlWithDNS(config.feedUrl, 'feedUrl')
    if (!urlValidation.isValid) {
      logger.error(`[${requestId}] Invalid RSS feed URL: ${urlValidation.error}`)
      throw new Error(`Invalid RSS feed URL: ${urlValidation.error}`)
    }

    const headers: Record<string, string> = {
      'User-Agent': 'Sim/1.0 RSS Poller',
      Accept: 'application/rss+xml, application/xml, text/xml, */*',
    }
    if (config.etag) {
      headers['If-None-Match'] = config.etag
    }
    if (config.lastModified) {
      headers['If-Modified-Since'] = config.lastModified
    }

    const response = await secureFetchWithPinnedIP(config.feedUrl, urlValidation.resolvedIP!, {
      headers,
      timeout: 30000,
    })

    if (response.status === 304) {
      logger.info(`[${requestId}] RSS feed not modified (304) for ${config.feedUrl}`)
      return {
        feed: { items: [] } as RssFeed,
        items: [],
        etag: response.headers.get('etag') ?? config.etag,
        lastModified: response.headers.get('last-modified') ?? config.lastModified,
      }
    }

    if (!response.ok) {
      await response.text().catch(() => {})
      throw new Error(`Failed to fetch RSS feed: ${response.status} ${response.statusText}`)
    }

    const newEtag = response.headers.get('etag') ?? undefined
    const newLastModified = response.headers.get('last-modified') ?? undefined

    const xmlContent = await response.text()
    const feed = await parser.parseString(xmlContent)

    if (!feed.items || !feed.items.length) {
      return { feed: feed as RssFeed, items: [], etag: newEtag, lastModified: newLastModified }
    }

    const lastCheckedTime = config.lastCheckedTimestamp
      ? new Date(config.lastCheckedTimestamp)
      : null
    const lastSeenGuids = new Set(config.lastSeenGuids || [])