(text_a, text_b)
| 126 | |
| 127 | |
| 128 | def get_f1(text_a, text_b): |
| 129 | tokens_a = text_a.lower().split() |
| 130 | tokens_b = text_b.lower().split() |
| 131 | if len(tokens_a) == 0 or len(tokens_b) == 0: |
| 132 | return 1 if len(tokens_a) == len(tokens_b) else 0 |
| 133 | set_a = count_tokens(tokens_a) |
| 134 | set_b = count_tokens(tokens_b) |
| 135 | match = 0 |
| 136 | for token in set_a.keys(): |
| 137 | if token in set_b.keys(): |
| 138 | match += min(set_a[token], set_b[token]) |
| 139 | p = match / len(tokens_a) |
| 140 | r = match / len(tokens_b) |
| 141 | return 2.0 * p * r / (p + r + 1e-5) |
| 142 | |
| 143 | |
| 144 | def remove_duplicate(l_list, duplicate_rate): |
no test coverage detected