Method splitDocuments

packages/llm/src/langchain.ts:207–217

(
    documents: Document[],
    chunkSize = env.CHUNK_SIZE
  )

  @measure
  private static async splitDocuments(
    documents: Document[],
    chunkSize = env.CHUNK_SIZE
  ) {
    const splitter = new TokenTextSplitter({
      encodingName: 'gpt2',
      chunkSize,
      chunkOverlap: 0,
    });
    return await splitter.splitDocuments(documents);
  }

  private static async getDocuments(url: string) {
    const dir = `.db/crawl/${StringUtils.clean(url)}`;