(self, fileName, filePath, chunk_size=500, chunk_overlap=0, text_splitter=None, autoSpliter=False,
csv_delimiter=",",
csv_quotechar='"', csv_fieldnames=[], json_jq_schema='.messages[].content', json_lines=True,
dir_glob="**/*.md",
dir_show_progress=True, dir_use_multithreading=False,
dir_text_loader_kwargs={'autodetect_encoding': True},
dir_silent_errors=True, dir_loader_cls=TextLoader)
| 262 | ) |
| 263 | |
| 264 | def split(self, fileName, filePath, chunk_size=500, chunk_overlap=0, text_splitter=None, autoSpliter=False, |
| 265 | csv_delimiter=",", |
| 266 | csv_quotechar='"', csv_fieldnames=[], json_jq_schema='.messages[].content', json_lines=True, |
| 267 | dir_glob="**/*.md", |
| 268 | dir_show_progress=True, dir_use_multithreading=False, |
| 269 | dir_text_loader_kwargs={'autodetect_encoding': True}, |
| 270 | dir_silent_errors=True, dir_loader_cls=TextLoader): |
| 271 | text_splitter = self.getSpliter( |
| 272 | textSplit=text_splitter, |
| 273 | fileName=fileName, |
| 274 | autoSpliter=autoSpliter, |
| 275 | chunk_size=chunk_size, |
| 276 | chunk_overlap=chunk_overlap |
| 277 | ) |
| 278 | if text_splitter is None: |
| 279 | text_splitter = TokenTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap) |
| 280 | fileName_withPath = filePath + '/' + fileName |
| 281 | if str(fileName).startswith('http') and not str(fileName).endswith('pdf'): |
| 282 | # TODO 进行文件的下载以及 |
| 283 | pass |
| 284 | if str(fileName_withPath).endswith("pdf"): |
| 285 | if str(fileName).startswith('http'): |
| 286 | pdf = OnlinePDFLoader(fileName).load() |
| 287 | else: |
| 288 | pdf = PyPDFLoader(fileName_withPath).load() |
| 289 | documents = text_splitter.split_documents(pdf) |
| 290 | elif str(fileName_withPath).endswith("txt"): |
| 291 | txt = TextLoader(fileName_withPath).load() |
| 292 | documents = text_splitter.split_documents(txt) |
| 293 | elif str(fileName_withPath).endswith("md"): |
| 294 | md = UnstructuredMarkdownLoader(fileName_withPath).load() |
| 295 | documents = text_splitter.split_documents(md) |
| 296 | elif str(fileName_withPath).endswith("csv"): |
| 297 | csv = CSVLoader( |
| 298 | file_path=fileName_withPath, |
| 299 | csv_args={ |
| 300 | 'delimiter': csv_delimiter, |
| 301 | 'quotechar': csv_quotechar, |
| 302 | 'fieldnames': csv_fieldnames |
| 303 | } |
| 304 | ).load() |
| 305 | documents = text_splitter.split_documents(csv) |
| 306 | elif str(fileName_withPath).endswith("json"): |
| 307 | jsonStr = JSONLoader( |
| 308 | fileName_withPath, |
| 309 | jq_schema=json_jq_schema, |
| 310 | json_lines=json_lines |
| 311 | ).load() |
| 312 | documents = text_splitter.split_documents(jsonStr) |
| 313 | elif str(fileName_withPath).endswith("html"): |
| 314 | pdf = UnstructuredHTMLLoader(fileName_withPath).load() |
| 315 | documents = text_splitter.split_documents(pdf) |
| 316 | elif str(fileName_withPath).endswith("/"): |
| 317 | if dir_glob.endswith('.py'): |
| 318 | dir_loader_cls = PythonLoader |
| 319 | dirs = DirectoryLoader( |
| 320 | fileName_withPath, |
| 321 | glob=dir_glob, |
no test coverage detected