Method getSpliter

embedding/embeddingHelper.py:346–395

(self, textSplit, fileName, autoSpliter=False, chunk_size=500, chunk_overlap=0)

        pass

    def getSpliter(self, textSplit, fileName, autoSpliter=False, chunk_size=500, chunk_overlap=0):
        self.checkClient()
        if textSplit is not None:
            return textSplit

        text_splitter = None
        if autoSpliter:
            # 根据文件的类型来实现spliter
            if str(fileName).endswith("txt"):
                text_splitter = CharacterTextSplitter(
                    separator="\n\n",
                    chunk_size=chunk_size,
                    chunk_overlap=chunk_overlap,
                    length_function=len,
                    is_separator_regex=False,
                )
            elif str(fileName).endswith("md"):
                headers_to_split_on = [
                    ("#", "Header 1"),
                    ("##", "Header 2"),
                    ("###", "Header 3"),
                ]
                text_splitter = MarkdownHeaderTextSplitter(headers_to_split_on=headers_to_split_on)
            elif str(fileName).endswith("py"):
                text_splitter = RecursiveCharacterTextSplitter.get_separators_for_language(Language.PYTHON)
            elif str(fileName).endswith("go"):
                text_splitter = RecursiveCharacterTextSplitter.get_separators_for_language(Language.GO)
            elif str(fileName).endswith("java"):
                text_splitter = RecursiveCharacterTextSplitter.get_separators_for_language(Language.JAVA)
            elif str(fileName).endswith("php"):
                text_splitter = RecursiveCharacterTextSplitter.get_separators_for_language(Language.PHP)
            elif str(fileName).endswith("js"):
                text_splitter = RecursiveCharacterTextSplitter.get_separators_for_language(Language.JS)
            elif str(fileName).endswith("proto"):
                text_splitter = RecursiveCharacterTextSplitter.get_separators_for_language(Language.PROTO)
            elif str(fileName).endswith("rst"):
                text_splitter = RecursiveCharacterTextSplitter.get_separators_for_language(Language.RST)
            elif str(fileName).endswith("ruby"):
                text_splitter = RecursiveCharacterTextSplitter.get_separators_for_language(Language.RUBY)
            elif str(fileName).endswith("scala"):
                text_splitter = RecursiveCharacterTextSplitter.get_separators_for_language(Language.SCALA)
            elif str(fileName).endswith("swift"):
                text_splitter = RecursiveCharacterTextSplitter.get_separators_for_language(Language.SWIFT)
            elif str(fileName).endswith("latex"):
                text_splitter = RecursiveCharacterTextSplitter.get_separators_for_language(Language.LATEX)
            elif str(fileName).endswith("html"):
                text_splitter = RecursiveCharacterTextSplitter.get_separators_for_language(Language.HTML)
            elif str(fileName).endswith("sol"):
                text_splitter = RecursiveCharacterTextSplitter.get_separators_for_language(Language.SOL)
        return text_splitter

    def processResult(self, query: list = None, query_vec: list = None, result=None):
        if self.need_redis and query is not None: