(page_list, list_result, start_index=1, N=None, model=None)
| 898 | |
| 899 | ################### verify toc ######################################################### |
| 900 | async def verify_toc(page_list, list_result, start_index=1, N=None, model=None): |
| 901 | print('start verify_toc') |
| 902 | # Find the last non-None physical_index |
| 903 | last_physical_index = None |
| 904 | for item in reversed(list_result): |
| 905 | if item.get('physical_index') is not None: |
| 906 | last_physical_index = item['physical_index'] |
| 907 | break |
| 908 | |
| 909 | # Early return if we don't have valid physical indices |
| 910 | if last_physical_index is None or last_physical_index < len(page_list)/2: |
| 911 | return 0, [] |
| 912 | |
| 913 | # Determine which items to check |
| 914 | if N is None: |
| 915 | print('check all items') |
| 916 | sample_indices = range(0, len(list_result)) |
| 917 | else: |
| 918 | N = min(N, len(list_result)) |
| 919 | print(f'check {N} items') |
| 920 | sample_indices = random.sample(range(0, len(list_result)), N) |
| 921 | |
| 922 | # Prepare items with their list indices |
| 923 | indexed_sample_list = [] |
| 924 | for idx in sample_indices: |
| 925 | item = list_result[idx] |
| 926 | # Skip items with None physical_index (these were invalidated by validate_and_truncate_physical_indices) |
| 927 | if item.get('physical_index') is not None: |
| 928 | item_with_index = item.copy() |
| 929 | item_with_index['list_index'] = idx # Add the original index in list_result |
| 930 | indexed_sample_list.append(item_with_index) |
| 931 | |
| 932 | # Run checks concurrently |
| 933 | tasks = [ |
| 934 | check_title_appearance(item, page_list, start_index, model) |
| 935 | for item in indexed_sample_list |
| 936 | ] |
| 937 | results = await asyncio.gather(*tasks) |
| 938 | |
| 939 | # Process results |
| 940 | correct_count = 0 |
| 941 | incorrect_results = [] |
| 942 | for result in results: |
| 943 | if result['answer'] == 'yes': |
| 944 | correct_count += 1 |
| 945 | else: |
| 946 | incorrect_results.append(result) |
| 947 | |
| 948 | # Calculate accuracy |
| 949 | checked_count = len(results) |
| 950 | accuracy = correct_count / checked_count if checked_count > 0 else 0 |
| 951 | print(f"accuracy: {accuracy*100:.2f}%") |
| 952 | return accuracy, incorrect_results |
| 953 | |
| 954 | |
| 955 |
no test coverage detected