| 55 | * ``` |
| 56 | */ |
| 57 | export function tokenize(string: string, wordDiff = false): string[] { |
| 58 | if (wordDiff) { |
| 59 | return string |
| 60 | .split(WHITESPACE_SYMBOLS_REGEXP) |
| 61 | .filter((token) => token); |
| 62 | } |
| 63 | const tokens: string[] = []; |
| 64 | const lines = string.split(/(\n|\r\n)/).filter((line) => line); |
| 65 | |
| 66 | for (const [i, line] of lines.entries()) { |
| 67 | if (i % 2) { |
| 68 | tokens[tokens.length - 1] += line; |
| 69 | } else { |
| 70 | tokens.push(line); |
| 71 | } |
| 72 | } |
| 73 | return tokens; |
| 74 | } |
| 75 | |
| 76 | /** |
| 77 | * Create details by filtering relevant word-diff for current line and merge |