Method _create_dataset

lit_nlp/app.py:414–456

Create a dataset, updating and returning the metadata.

(
      self,
      data: types.JsonDict,
      dataset_name: Optional[str] = None,
      **unused_kw,
  )

    return list(self._datasets[dataset_name].indexed_examples)

  def _create_dataset(
      self,
      data: types.JsonDict,
      dataset_name: Optional[str] = None,
      **unused_kw,
  ):
    """Create a dataset, updating and returning the metadata."""
    if dataset_name is None:
      raise ValueError('No base dataset specified.')

    config: Optional[dict[str, Any]] = data.get('config')
    if config is None:
      raise ValueError('No config specified.')

    new_name: Optional[str] = config.pop('new_name', None)
    if new_name is None:
      raise ValueError('No name provided for the new dataset.')
    elif new_name in self._datasets:
      return (self._info, new_name)  # Return the existing dataset

    if (loader_info := self._dataset_loaders.get(dataset_name)) is None:
      raise ValueError(
          f'No loader information (Cls + init_spec) found for {dataset_name}'
      )

    dataset_cls, dataset_init_spec = loader_info

    if dataset_init_spec is not None:
      initializer_name = getattr(dataset_cls, '__name__', repr(dataset_cls))
      utils.validate_config_against_spec(
          config,
          dataset_init_spec,
          f'{dataset_name} ({initializer_name})',
          raise_for_unsupported=True,
      )

    new_dataset = dataset_cls(**config)
    annotated_dataset = self._run_annotators(new_dataset)
    self._datasets[new_name] = lit_dataset.IndexedDataset(
        base=annotated_dataset, id_fn=caching.input_hash
    )
    self._info = self._build_metadata()
    return (self._info, new_name)

  def _create_model(
      self, data: types.JsonDict, model_name: Optional[str] = None, **unused_kw