Simple tokenization function.
(text)
| 32 | sentence_model = None |
| 33 | |
| 34 | def simple_tokenize(text): |
| 35 | """Simple tokenization function.""" |
| 36 | # Convert to string if not already |
| 37 | text = str(text) |
| 38 | return text.lower().replace('.', ' ').replace(',', ' ').replace('!', ' ').replace('?', ' ').split() |
| 39 | |
| 40 | def calculate_rouge_scores(prediction: str, reference: str) -> Dict[str, float]: |
| 41 | """Calculate ROUGE scores for prediction against reference.""" |