Build a set of Instances for a task, and store them in task.instances
(
self, limit=None, rank=None, world_size=None,
# additions
tokenizer = None, context_length = 1000, answer_length= 50, sequence_length = 2048, context_key = "context", cutting_context = False, answer_key = ["value"], skip_check_answer=False
)
| 432 | return instances, new_doc_set |
| 433 | |
| 434 | def build_all_requests( |
| 435 | self, limit=None, rank=None, world_size=None, |
| 436 | |
| 437 | # additions |
| 438 | tokenizer = None, context_length = 1000, answer_length= 50, sequence_length = 2048, context_key = "context", cutting_context = False, answer_key = ["value"], skip_check_answer=False |
| 439 | ) -> None: |
| 440 | """Build a set of Instances for a task, and store them in task.instances""" |
| 441 | if self.has_test_docs(): |
| 442 | docs = self.test_docs() |
| 443 | elif self.has_validation_docs(): |
| 444 | docs = self.validation_docs() |
| 445 | else: |
| 446 | assert False, f"Task dataset (path={self.DATASET_PATH}, name={self.DATASET_NAME}) must have valid or test docs!" |
| 447 | |
| 448 | eval_logger.info(f"Building contexts for {self.config.task} on rank {rank}...") |
| 449 | |
| 450 | instances = [] |
| 451 | new_doc_set = {} |
| 452 | |
| 453 | print("cutting_context:", cutting_context, context_length) |
| 454 | for doc_id, doc in utils.create_iterator( |
| 455 | enumerate(docs), rank, world_size, limit |
| 456 | ): |
| 457 | |
| 458 | if cutting_context: |
| 459 | new_instances, doc_set = self.truncate_context( |
| 460 | doc, doc_id, tokenizer = tokenizer, context_length = context_length, answer_length=answer_length, sequence_length = sequence_length, context_key = context_key, cutting_context = cutting_context, skip_check_answer=skip_check_answer, answer_key = answer_key, id_start=len(instances) |
| 461 | ) |
| 462 | instances += new_instances |
| 463 | new_doc_set.update(doc_set) |
| 464 | |
| 465 | else: |
| 466 | # sample fewshot context #TODO: need to offset doc_id by rank now! |
| 467 | fewshot_ctx = self.fewshot_context( |
| 468 | doc, |
| 469 | 0 if self.config.num_fewshot is None else self.config.num_fewshot, |
| 470 | ) |
| 471 | |
| 472 | # TODO: we should override self.config.repeats if doing greedy gen so users don't waste time+compute |
| 473 | inst = self.construct_requests( |
| 474 | doc=doc, |
| 475 | ctx=fewshot_ctx, |
| 476 | metadata=(self.config["task"], doc_id, self.config.repeats), |
| 477 | ) |
| 478 | |
| 479 | if not isinstance(inst, list): |
| 480 | inst = [inst] |
| 481 | |
| 482 | new_doc_set[doc_id] = doc |
| 483 | instances.extend(inst) |
| 484 | self._instances = instances |
| 485 | assert len(self._instances) != 0, "task.build_requests() did not find any docs!" |
| 486 | return new_doc_set |
| 487 | |
| 488 | |
| 489 | @abc.abstractmethod |
no test coverage detected