Function isTextWithSubstance

lib/embed-docs/utils.ts:4–15

(text: string)

import { getTokenCount, isDate, isEmail, isPhoneNumber, isUrl } from "../utils";

export function isTextWithSubstance(text: string): boolean {
  return (
    // Hack to include multi-line images or links
    ["[", "!["].includes(text) ||
    // If very short number of characters, likely it's not a useful chunk
    (text.length > 3 &&
      !isEmail(text) &&
      !isPhoneNumber(text) &&
      !isUrl(text) &&
      !isDate(text))
  );
}

export function splitTextByHeaders(
  markdownText: string,