Function chunkStructuredDocs

src/api/utils/vectorStore.ts:932–1167

(text: string)

 * @returns Array of text chunks with preserved structure
 */
export function chunkStructuredDocs(text: string): string[] {
  const chunks: string[] = [];
  const lines = text.split("\n");

  // Step 1: Extract all headers and build a header hierarchy
  interface HeaderInfo {
    level: number;
    title: string;
    lineIndex: number;
  }

  const headers: HeaderInfo[] = [];

  lines.forEach((line, index) => {
    const headerMatch = line.match(/^(#{1,6})\s+(.*)/);
    if (headerMatch) {
      headers.push({
        level: headerMatch[1].length,
        title: headerMatch[2].trim(),
        lineIndex: index,
      });
    }
  });

  // If there's at least one header, create a chunk with the document title and description
  if (headers.length > 0) {
    const mainHeader = headers[0];
    let mainDescription = "";

    // Collect the main description until we hit another header or a blank line followed by a list item
    for (let i = mainHeader.lineIndex + 1; i < lines.length; i++) {
      const line = lines[i].trim();

      // Stop if we hit another header
      if (line.match(/^#{1,6}\s+/)) break;

      // Stop if we hit a blank line followed by a list item
      if (
        line === "" &&
        i + 1 < lines.length &&
        (lines[i + 1].trim().startsWith("- ") ||
          lines[i + 1].trim().startsWith("* "))
      )
        break;

      if (line !== "") {
        mainDescription += mainDescription ? "\n" + line : line;
      }
    }

    // Create a chunk with main title and description
    if (mainDescription) {
      chunks.push(`# ${mainHeader.title}\n\n${mainDescription}`);
    }
  }

  // Find the current section header for context
  const getCurrentHeader = (lineIndex: number): string => {