Function processRobotOutputFormats

server/src/utils/output-post-processor.ts:27–222

(
  params: ProcessRobotOutputParams
)

}

export async function processRobotOutputFormats(
  params: ProcessRobotOutputParams
): Promise<ProcessRobotOutputResult> {
  const {
    robotType,
    outputFormats,
    categorizedOutput,
    currentPage,
    initialBinaryOutput,
    llmConfig,
  } = params;

  const binaryOutput: Record<string, any> = {
    ...(initialBinaryOutput || {}),
  };

  const effectiveFormats = Array.isArray(outputFormats)
    ? (outputFormats.length > 0
      ? outputFormats
      : robotType === 'crawl'
        ? DEFAULT_OUTPUT_FORMATS
        : outputFormats) 
    : DEFAULT_OUTPUT_FORMATS;

  if (robotType !== 'crawl' && robotType !== 'search') {
    return { categorizedOutput, binaryOutput };
  }

  if (robotType === 'crawl' && Array.isArray((categorizedOutput.crawl as any)?.['Crawl Results'])) {
    const crawlResults: any[] = (categorizedOutput.crawl as any)['Crawl Results'];
    const includeVisibleScreenshot = effectiveFormats.includes('screenshot-visible');
    const includeFullpageScreenshot = effectiveFormats.includes('screenshot-fullpage');

    for (let pageIndex = 0; pageIndex < crawlResults.length; pageIndex++) {
      const pageResult = crawlResults[pageIndex];
      if (!pageResult.error) {
        let markdownConversionSucceeded = false;
        if (effectiveFormats.includes('markdown') && pageResult.html) {
          try {
            pageResult.markdown = await parseMarkdown(pageResult.html, pageResult.metadata?.url);
            markdownConversionSucceeded = true;
          } catch (e: any) {
            logger.log('warn', `Failed to convert crawl page to markdown: ${e.message}`);
          }
        }

        if (!effectiveFormats.includes('html') && markdownConversionSucceeded) {
          delete pageResult.html;
        }
        if (!effectiveFormats.includes('text')) delete pageResult.text;
        if (!effectiveFormats.includes('links')) delete pageResult.links;

        if (effectiveFormats.includes('summary')) {
          const pageText = (pageResult.markdown || pageResult.text || '').substring(0, 40000);
          if (pageText.trim()) {
            try {
              const { summarizeMarkdown } = require('../utils/summarizer');
              pageResult.summary = await summarizeMarkdown(pageText, llmConfig);