Function chunkReadme

src/api/utils/vectorStore.ts:269–501

(text: string, fileName?: string)

 * @returns Array of text chunks with preserved structure
 */
export function chunkReadme(text: string, fileName?: string): string[] {
  // Check if this appears to be a README format
  const hasMultipleHeadings = (text.match(/^#+\s+.+/gm) || []).length > 1;
  const hasCodeBlocks = text.includes("```");
  const isReadmeLike =
    hasMultipleHeadings &&
    (hasCodeBlocks || text.includes("* ") || text.includes("- "));

  // If not README-like, use the regular chunking
  if (!isReadmeLike) {
    return chunkText(text);
  }

  // Check if this is a special case file (like llms.txt) that needs list-item level chunking
  const isSpecialListFile = fileName?.toLowerCase().includes("llms.txt");

  // Track headers and their content
  interface HeaderSection {
    level: number;
    title: string;
    content: string;
    lineIndex: number;
  }

  const sections: HeaderSection[] = [];
  let currentSection: HeaderSection | null = null;
  let mainHeaderContent = "";
  let mainTitle = "";

  // Helper function to detect badge lines (markdown image links with badge URLs)
  function isBadgeLine(line: string): boolean {
    // Detect badge-specific patterns (shield.io, badge URLs, image links in a row)
    return (
      /!\[.*\]\(.*badge.*\)/.test(line) ||
      /!\[.*\]\(.*shield\.io.*\)/.test(line) ||
      (/\[!\[.*\]\(.*\)\]\(.*\)/.test(line) &&
        (line.includes("badge") || line.includes("shield"))) ||
      /img\.shields\.io/.test(line) ||
      (line.includes("<img") &&
        (line.includes("badge") || line.includes("shield")))
    );
  }

  // First pass: Extract headers and their content
  const lines = text.split("\n");
  let inBadgeSection = false;
  let badgeSectionEndLine = 0;
  let skipToLine = -1;

  // Detect the initial badge/logo section which often appears at the start of READMEs
  for (let i = 0; i < Math.min(20, lines.length); i++) {
    if (
      (lines[i].includes('<p align="center">') ||
        lines[i].includes('align="center"') ||
        lines[i].includes('<div align="center">')) &&
      i + 5 < lines.length
    ) {
      // Check if next few lines contain images, badges, or links