Function chunkText

src/api/utils/vectorStore.ts:561–642

(
  text: string,
  maxChunkSize: number = 1500,
  minChunkSize: number = 500,
)

 * @returns Array of text chunks
 */
export function chunkText(
  text: string,
  maxChunkSize: number = 1500,
  minChunkSize: number = 500,
): string[] {
  const chunks: string[] = [];

  // Split by markdown headings (## Heading)
  const headingPattern = /\n(#{1,6}\s+[^\n]+)\n/g;
  const sections = text.split(headingPattern);

  let currentChunk = "";
  let headingText = "";

  // Process each section
  for (let i = 0; i < sections.length; i++) {
    const section = sections[i];

    // Check if this is a heading
    if (i > 0 && i % 2 === 1) {
      headingText = section.trim();
      continue;
    }

    // This is content - process it with the preceding heading
    const contentWithHeading = headingText
      ? `${headingText}\n\n${section}`
      : section;

    // If content is short enough, add as single chunk
    if (contentWithHeading.length <= maxChunkSize) {
      if (contentWithHeading.trim().length > 0) {
        chunks.push(contentWithHeading.trim());
      }
      headingText = "";
      continue;
    }

    // If content is long, split by paragraphs
    const paragraphs = contentWithHeading.split(/\n\n+/);

    currentChunk = "";

    for (const paragraph of paragraphs) {
      const trimmedParagraph = paragraph.trim();

      // Skip empty paragraphs
      if (!trimmedParagraph) continue;

      // If adding this paragraph would exceed max size and we already have content
      if (
        currentChunk &&
        currentChunk.length + trimmedParagraph.length + 2 > maxChunkSize
      ) {
        // Only add the chunk if it meets minimum size
        if (currentChunk.length >= minChunkSize) {
          chunks.push(currentChunk.trim());
          currentChunk = trimmedParagraph;