* The words of a sentence, lowercased, as a set. * * Used only to answer "which built sentence does this rewrite belong to". Not a * fact check and not a quality measure: a rewrite is supposed to change words, * and this is the coarsest possible reading of whether it changed which * sentence it
(text: string)
| 135 | * sentence it is. |
| 136 | */ |
| 137 | function wordSet(text: string): Set<string> { |
| 138 | return new Set(text |
| 139 | .toLowerCase() |
| 140 | .split(/[^a-z0-9_$./]+/i) |
| 141 | // A dot inside a word is part of a file name and stays; a dot at the |
| 142 | // end of one is the end of a sentence. Without this, `code.` and `code` |
| 143 | // are two different words and every rewrite that moved a full stop |
| 144 | // would read as a reordering. |
| 145 | .map((word) => word.replace(/^[./]+/, '').replace(/[./]+$/, '')) |
| 146 | .filter((word) => word.length > 0)); |
| 147 | } |
| 148 | |
| 149 | /** How much two sentences overlap, 0 to 1. */ |
| 150 | function overlap(one: Set<string>, other: Set<string>): number { |