(directory)
| 275 | json.dump(infos_dict, f, indent=4) |
| 276 | |
| 277 | def get_file_paths_recursive(directory): |
| 278 | file_paths = [] |
| 279 | for root, dirs, files in os.walk(directory): |
| 280 | for file in files: |
| 281 | # Construct the file path |
| 282 | file_path = os.path.join(root, file) |
| 283 | file_paths.append(file_path) |
| 284 | return file_paths |
| 285 | |
| 286 | def truncate_tokens(string: str, encoding_name: str, max_length: int = 8192) -> str: |
| 287 | """Truncates a text string based on max number of tokens.""" |