Function chunkDocumentation

src/api/utils/vectorStore.ts:510–551

(text: string, fileName?: string)

 * @returns Array of text chunks
 */
export function chunkDocumentation(text: string, fileName?: string): string[] {
  // First check if this is a structured document with list items (like llms.txt)
  if (fileName?.toLowerCase().includes("llms.txt")) {
    try {
      // For llms.txt files, each list item should be treated as its own chunk
      // with section header context
      const structuredChunks = chunkStructuredDocs(text);
      if (structuredChunks.length > 0) {
        return structuredChunks;
      }
    } catch (error) {
      console.warn(
        "Structured documentation chunking failed for llms.txt, trying README chunker",
      );
    }
  }

  // Then try README-specific chunking
  try {
    const readmeChunks = chunkReadme(text, fileName);
    if (readmeChunks.length > 0) {
      return readmeChunks;
    }
  } catch (error) {
    console.warn("README chunking failed, trying next chunker");
  }

  // Then try structured documentation chunking as fallback
  try {
    const structuredChunks = chunkStructuredDocs(text);
    if (structuredChunks.length > 0) {
      return structuredChunks;
    }
  } catch (error) {
    console.warn(
      "Structured documentation chunking failed, falling back to default chunker",
    );
  }

  // Fall back to the regular chunking algorithm
  return chunkText(text);
}

/**
 * Process documentation text into chunks for vector storage with improved boundaries