This is the main optimization process step.
(self)
| 222 | self.patient = state['patient'] |
| 223 | |
| 224 | def step(self): |
| 225 | """ |
| 226 | This is the main optimization process step. |
| 227 | """ |
| 228 | self.log_and_print(f'Starting step {self.batch_id}') |
| 229 | if len(self.dataset.records) == 0: |
| 230 | self.log_and_print('Dataset is empty generating initial samples') |
| 231 | self.generate_initial_samples() |
| 232 | if self.config.use_wandb: |
| 233 | cur_batch = self.dataset.get_leq(self.batch_id) |
| 234 | random_subset = cur_batch.sample(n=min(10, len(cur_batch)))[['text']] |
| 235 | self.wandb_run.log( |
| 236 | {"Prompt": wandb.Html(f"<p>{self.cur_prompt}</p>"), "Samples": wandb.Table(dataframe=random_subset)}, |
| 237 | step=self.batch_id) |
| 238 | |
| 239 | logging.info('Running annotator') |
| 240 | records = self.annotator.apply(self.dataset, self.batch_id) |
| 241 | self.dataset.update(records) |
| 242 | |
| 243 | self.predictor.cur_instruct = self.cur_prompt |
| 244 | logging.info('Running Predictor') |
| 245 | records = self.predictor.apply(self.dataset, self.batch_id, leq=True) |
| 246 | self.dataset.update(records) |
| 247 | |
| 248 | self.eval.dataset = self.dataset.get_leq(self.batch_id) |
| 249 | self.eval.eval_score() |
| 250 | logging.info('Calculating Score') |
| 251 | large_errors = self.eval.extract_errors() |
| 252 | self.eval.add_history(self.cur_prompt, self.task_description) |
| 253 | if self.config.use_wandb: |
| 254 | large_errors = large_errors.sample(n=min(6, len(large_errors))) |
| 255 | correct_samples = self.eval.extract_correct() |
| 256 | correct_samples = correct_samples.sample(n=min(6, len(correct_samples))) |
| 257 | vis_data = pd.concat([large_errors, correct_samples]) |
| 258 | self.wandb_run.log({"score": self.eval.history[-1]['score'], |
| 259 | "prediction_result": wandb.Table(dataframe=vis_data), |
| 260 | 'Total usage': self.calc_usage()}, step=self.batch_id) |
| 261 | if self.stop_criteria(): |
| 262 | self.log_and_print('Stop criteria reached') |
| 263 | return True |
| 264 | self.run_step_prompt() |
| 265 | self.save_state() |
| 266 | return False |
| 267 | |
| 268 | def run_pipeline(self, num_steps: int): |
| 269 | # Run the optimization pipeline for num_steps |
no test coverage detected