| 224 | return result |
| 225 | |
| 226 | def saveFile(self, fileName, filePath, chunk_size=500, chunk_overlap=0, text_splitter=None, autoSpliter=False, |
| 227 | csv_delimiter=",", |
| 228 | csv_quotechar='"', csv_fieldnames=[], json_jq_schema='.messages[].content', json_lines=True, |
| 229 | dir_glob="**/*.md", |
| 230 | dir_show_progress=True, dir_use_multithreading=False, |
| 231 | dir_text_loader_kwargs={'autodetect_encoding': True}, |
| 232 | dir_silent_errors=True, dir_loader_cls=TextLoader): |
| 233 | documents = self.split( |
| 234 | fileName=fileName, |
| 235 | filePath=filePath, |
| 236 | chunk_size=chunk_size, |
| 237 | chunk_overlap=chunk_overlap, |
| 238 | text_splitter=text_splitter, |
| 239 | autoSpliter=autoSpliter, |
| 240 | csv_delimiter=csv_delimiter, |
| 241 | csv_quotechar=csv_quotechar, |
| 242 | csv_fieldnames=csv_fieldnames, |
| 243 | json_jq_schema=json_jq_schema, |
| 244 | json_lines=json_lines, |
| 245 | dir_glob=dir_glob, |
| 246 | dir_show_progress=dir_show_progress, |
| 247 | dir_use_multithreading=dir_use_multithreading, |
| 248 | dir_text_loader_kwargs=dir_text_loader_kwargs, |
| 249 | dir_silent_errors=dir_silent_errors, |
| 250 | dir_loader_cls=dir_loader_cls |
| 251 | ) |
| 252 | print(documents) |
| 253 | fileName_withPath = filePath + '/' + fileName |
| 254 | index = self.collection.count() |
| 255 | self.collection.add( |
| 256 | documents=[document.page_content for document in documents], |
| 257 | ids=['id_' + str(index + i) for i in range(documents.__len__())], |
| 258 | metadatas=[{ |
| 259 | 'fileName': fileName_withPath, |
| 260 | 'documentIndex': i |
| 261 | } for i in range(documents.__len__())] |
| 262 | ) |
| 263 | |
| 264 | def split(self, fileName, filePath, chunk_size=500, chunk_overlap=0, text_splitter=None, autoSpliter=False, |
| 265 | csv_delimiter=",", |