| 1497 | |
| 1498 | |
| 1499 | def test_tokenize_nested_sequence_thread_safe(): |
| 1500 | nested_list = [] |
| 1501 | for ix in range(100): |
| 1502 | nested_list = [ix, nested_list] |
| 1503 | |
| 1504 | nested_dict = {} |
| 1505 | for ix in range(50): |
| 1506 | nested_dict = {ix: nested_dict} |
| 1507 | |
| 1508 | normalize_token(nested_list) |
| 1509 | with ThreadPoolExecutor() as pool: |
| 1510 | futures = [pool.submit(normalize_token, nested_list) for _ in range(1000)] |
| 1511 | assert len({f.result() for f in futures}) == 1 |
| 1512 | futures = [pool.submit(normalize_token, nested_dict) for _ in range(1000)] |
| 1513 | assert len({f.result() for f in futures}) == 1 |
| 1514 | |
| 1515 | |
| 1516 | def test_tokenize_range_index(): |