| 5 | |
| 6 | # Define the DocumentationEvaluator class |
| 7 | class DocumentationEvaluator: |
| 8 | def __init__(self, model_name='all-MiniLM-L6-v2'): |
| 9 | self.model = SentenceTransformer(model_name) |
| 10 | |
| 11 | def calculate_relevance(self, generated_response, reference_response): |
| 12 | gen_embedding = self.model.encode([generated_response]) |
| 13 | ref_embedding = self.model.encode([reference_response]) |
| 14 | similarity = cosine_similarity(gen_embedding, ref_embedding) |
| 15 | return similarity[0][0] |
| 16 | |
| 17 | def calculate_consistency(self, responses): |
| 18 | if not responses: |
| 19 | print("Warning: No responses provided for consistency calculation.") |
| 20 | return float('nan') |
| 21 | |
| 22 | embeddings = self.model.encode(responses) |
| 23 | mean_embedding = np.mean(embeddings, axis=0) |
| 24 | variances = np.var(embeddings - mean_embedding, axis=0) |
| 25 | consistency_score = np.mean(variances) |
| 26 | return consistency_score |
| 27 | |
| 28 | def calculate_readability(self, text): |
| 29 | if not text: |
| 30 | print("Warning: Empty text provided for readability calculation.") |
| 31 | return float('nan') |
| 32 | |
| 33 | flesch_reading_ease = textstat.flesch_reading_ease(text) |
| 34 | return flesch_reading_ease |
| 35 | |
| 36 | def evaluate_documentation(self, generated_doc, reference_doc=None): |
| 37 | scores = {} |
| 38 | scores['readability'] = self.calculate_readability(generated_doc) |
| 39 | responses = [section.strip() for section in generated_doc.split('\n') if section.strip()] |
| 40 | scores['consistency'] = self.calculate_consistency(responses) |
| 41 | |
| 42 | if reference_doc: |
| 43 | scores['relevance'] = self.calculate_relevance(generated_doc, reference_doc) |
| 44 | else: |
| 45 | scores['relevance'] = None |
| 46 | |
| 47 | return scores |
| 48 | |
| 49 | # Initialize the evaluator |
| 50 | evaluator = DocumentationEvaluator() |