Function breakdownArticleComponent

core/indexing/docs/article.ts:26–105

(
  url: string,
  article: ArticleComponent,
  subpath: string,
  max_chunk_size: number,
)

};

function breakdownArticleComponent(
  url: string,
  article: ArticleComponent,
  subpath: string,
  max_chunk_size: number,
): Chunk[] {
  const chunks: Chunk[] = [];
  const lines = article.body.split("\n");
  let startLine = 0;
  let endLine = 0;
  let content = "";
  let index = 0;

  const fullUrl = new URL(
    `${subpath}#${cleanFragment(article.title)}`,
    url,
  ).toString();

  const createChunk = (
    chunkContent: string,
    chunkStartLine: number,
    chunkEndLine: number,
  ) => {
    chunks.push({
      content: chunkContent.trim(),
      startLine: chunkStartLine,
      endLine: chunkEndLine,
      otherMetadata: {
        title: cleanHeader(article.title),
      },
      index: index++,
      filepath: fullUrl,
      digest: fullUrl,
    });
  };

  for (let i = 0; i < lines.length; i++) {
    const line = lines[i];

    // Handle oversized lines by splitting them
    if (line.length > max_chunk_size) {
      // First push any accumulated content
      if (content.trim().length > 0) {
        createChunk(content, startLine, endLine);
        content = "";
      }

      // Split the long line into chunks
      let remainingLine = line;
      let subLineStart = i;
      while (remainingLine.length > 0) {
        const chunk = remainingLine.slice(0, max_chunk_size);
        createChunk(chunk, subLineStart, i);
        remainingLine = remainingLine.slice(max_chunk_size);
      }
      startLine = i + 1;
      continue;
    }