Given a `texts` iterable of Text objects, group these objects when they have the same key. Yield a tuple of (Text object, count of its occurences).
(texts)
| 333 | |
| 334 | |
| 335 | def cluster(texts): |
| 336 | """ |
| 337 | Given a `texts` iterable of Text objects, group these objects when they have the |
| 338 | same key. Yield a tuple of (Text object, count of its occurences). |
| 339 | """ |
| 340 | clusters = defaultdict(list) |
| 341 | for text in texts: |
| 342 | clusters[text.key].append(text) |
| 343 | |
| 344 | for cluster_key, cluster_texts in clusters.items(): |
| 345 | try: |
| 346 | # keep the longest as the representative value for a cluster |
| 347 | cluster_texts.sort(key=lambda x:-len(x.key)) |
| 348 | representative = cluster_texts[0] |
| 349 | count = sum(t.count for t in cluster_texts) |
| 350 | if TRACE_DEEP: |
| 351 | logger_debug('cluster: representative, count', representative, count) |
| 352 | yield representative, count |
| 353 | except Exception as e: |
| 354 | msg = ( |
| 355 | f'Error in cluster(): cluster_key: {cluster_key!r}, ' |
| 356 | f'cluster_texts: {cluster_texts!r}\n' |
| 357 | ) |
| 358 | import traceback |
| 359 | msg += traceback.format_exc() |
| 360 | raise Exception(msg) from e |
| 361 | |
| 362 | |
| 363 | def clean(text): |
no test coverage detected