(text, max_tokens=4096)
| 14 | TOTAL_TOKEN_COST = 0 |
| 15 | TOTAL_API_CALL_COST = 0 |
| 16 | def truncate_text(text, max_tokens=4096): |
| 17 | tokens = tokenizer.encode(text) |
| 18 | if len(tokens) > max_tokens: |
| 19 | tokens = tokens[:max_tokens] |
| 20 | truncated_text = tokenizer.decode(tokens) |
| 21 | return truncated_text |
| 22 | def create_if_not_exist(path): |
| 23 | if not os.path.exists(path): # 如果目录不存在，递归创建该目录 |
| 24 | os.makedirs(path, exist_ok=True) |
no outgoing calls
no test coverage detected