(l_list, duplicate_rate)
| 142 | |
| 143 | |
| 144 | def remove_duplicate(l_list, duplicate_rate): |
| 145 | tk_list = [l.lower().split() for l in l_list] |
| 146 | r_list = [] |
| 147 | history_set = set() |
| 148 | for i, w_list in enumerate(tk_list): |
| 149 | w_set = set(w_list) |
| 150 | if len(w_set & history_set) / len(w_set) <= duplicate_rate: |
| 151 | r_list.append(l_list[i]) |
| 152 | history_set |= w_set |
| 153 | return r_list |
| 154 | |
| 155 | |
| 156 | def rouge_metric(predictions, labels, examples, metric="rouge-1", duplicate_rate=0.7, dataset='cnn_dm'): |