(structure, limit=110000)
| 504 | |
| 505 | |
| 506 | def check_token_limit(structure, limit=110000): |
| 507 | list = structure_to_list(structure) |
| 508 | for node in list: |
| 509 | num_tokens = count_tokens(node['text'], model=None) |
| 510 | if num_tokens > limit: |
| 511 | print(f"Node ID: {node['node_id']} has {num_tokens} tokens") |
| 512 | print("Start Index:", node['start_index']) |
| 513 | print("End Index:", node['end_index']) |
| 514 | print("Title:", node['title']) |
| 515 | print("\n") |
| 516 | |
| 517 | |
| 518 | def convert_physical_index_to_int(data): |
nothing calls this directly
no test coverage detected