Add the current step information to the history :param prompt: The current prompt :param task_description: The task description
(self, prompt: str, task_description: str)
| 102 | return f"####\n##Prompt:\n{sample['prompt']}\n{self.large_error_to_str(sample['errors'], num_errors_per_label)}####\n " |
| 103 | |
| 104 | def add_history(self, prompt: str, task_description: str): |
| 105 | """ |
| 106 | Add the current step information to the history |
| 107 | :param prompt: The current prompt |
| 108 | :param task_description: The task description |
| 109 | """ |
| 110 | conf_matrix = None |
| 111 | large_error_to_str = self.large_error_to_str(self.errors, self.num_errors) |
| 112 | prompt_input = {'task_description': task_description, 'accuracy': self.mean_score, 'prompt': prompt, |
| 113 | 'failure_cases': large_error_to_str} |
| 114 | if self.score_function_name == 'accuracy': |
| 115 | conf_matrix = confusion_matrix(self.dataset['annotation'], |
| 116 | self.dataset['prediction'], labels=self.label_schema) |
| 117 | conf_text = f"Confusion matrix columns:{self.label_schema} the matrix data:" |
| 118 | for i, row in enumerate(conf_matrix): |
| 119 | conf_text += f"\n{self.label_schema[i]}: {row}" |
| 120 | prompt_input['confusion_matrix'] = conf_text |
| 121 | elif self.score_function_name == 'ranking': |
| 122 | prompt_input['labels'] = self.label_schema |
| 123 | analysis = self.analyzer.invoke(prompt_input) |
| 124 | |
| 125 | self.history.append({'prompt': prompt, 'score': self.mean_score, |
| 126 | 'errors': self.errors, 'confusion_matrix': conf_matrix, 'analysis': analysis['text']}) |
| 127 | |
| 128 | def extract_errors(self) -> pd.DataFrame: |
| 129 | """ |
no test coverage detected