| 56 | |
| 57 | |
| 58 | def get_adjacency(): |
| 59 | abs_adj = {item: set() for item in title_set} |
| 60 | full_adj = {item: set() for item in title_set} |
| 61 | |
| 62 | for item in tqdm(processed_data, desc="Generate abs adjacency dict"): |
| 63 | title = item["title"] |
| 64 | if title in title_set: # in corpus |
| 65 | for i in item["abs_hyperlink"]: |
| 66 | if i in title_set: |
| 67 | abs_adj[title].add(i) |
| 68 | abs_adj[i].add(title) |
| 69 | elif i.lower() in title_map: |
| 70 | abs_adj[title].add(title_map[i.lower()]) |
| 71 | abs_adj[title_map[i.lower()]].add(title) |
| 72 | |
| 73 | for item in tqdm(processed_data, desc="Generate full adjacency dict"): |
| 74 | title = item["title"] |
| 75 | if title in title_set: # in corpus |
| 76 | for i in item["full_hyperlink"]: |
| 77 | if i in doc_size: |
| 78 | full_adj[title].add(i) |
| 79 | full_adj[i].add(title) |
| 80 | elif i.lower() in title_map: |
| 81 | full_adj[title].add(title_map[i.lower()]) |
| 82 | full_adj[title_map[i.lower()]].add(title) |
| 83 | |
| 84 | return abs_adj, full_adj |
| 85 | |
| 86 | |
| 87 | def get_doc_size(): |