Function splitOutCodeChunks

lib/embed-docs/utils.ts:105–152

(sectionText: string)

}

export function splitOutCodeChunks(sectionText: string): string[] {
  const chunks: string[] = [];
  const codeBlockPattern = /```(\w*)\n([\s\S]+?)```/g;
  // (\w*) to capture the language abbreviation
  // [\s\S]+? to capture the code content (non-greedy)
  // [\s\S] includes newlines
  let match;

  // Find initial index for cutting the string
  let currentIndex = 0;

  while ((match = codeBlockPattern.exec(sectionText))) {
    const nonCodeChunk = sectionText.slice(currentIndex, match.index);

    // If there is any non-code text between two code blocks
    if (nonCodeChunk.trim().length !== 0) {
      chunks.push(nonCodeChunk);
    }

    const languageAbbrev = match[1] ?? "";
    // Extract code content from a code block, trim additional redundant spaces
    let codeChunk = match[2].trim();
    if (isJson(codeChunk)) {
      const jsonChunk = getPossibleJsonChunk(codeChunk);
      // If the code chunk is JSON, reduce tokens by removing unnecessary prettification
      const prePost = codeChunk.split(jsonChunk);

      // Nothing to do if first line or not found (-1)
      codeChunk = prePost[0] + unprettifyJsonString(jsonChunk) + prePost[1];
    }

    // Re-add the code block markdown formatting, so it's obvious it's a code block
    chunks.push(`\`\`\`${languageAbbrev}\n${codeChunk}\n\`\`\``);

    // Move to the end of the current match
    currentIndex = codeBlockPattern.lastIndex;
  }

  // If there is a non-code section after the last code block
  if (currentIndex < sectionText.length - 1) {
    const finalChunk = sectionText.slice(currentIndex).trim();
    if (finalChunk.length !== 0) {
      chunks.push(finalChunk);
    }
  }

  return chunks;
}

export function getPossibleJsonChunk(text: string): string {
  // Regex to check if text contains [] or {}