Method load

lit_nlp/api/dataset.py:361–409

Load and return additional previously-saved datapoints for this dataset. Args: path: The path to the persisted datapoint file. Returns: (IndexedDataset) A dataset containing the loaded data.

(self, path: str)

    return path

  def load(self, path: str):
    """Load and return additional previously-saved datapoints for this dataset.

    Args:
      path: The path to the persisted datapoint file.

    Returns:
      (IndexedDataset) A dataset containing the loaded data.

    """
    if not path.endswith(LIT_FILE_EXTENSION):
      # Try to load data using the base load method. If any data is
      # returned, then use that. Otherwise try loading the lit json extension
      # data format.
      base_dataset = self._base
      new_dataset = base_dataset.load(path) if base_dataset else None

      if new_dataset is not None:
        description = (f'{len(new_dataset)} examples from '
                       f'{path}\n{self._base.description()}')
        return IndexedDataset(
            base=new_dataset, id_fn=self.id_fn, description=description)

      path += LIT_FILE_EXTENSION

    with open(path, 'r') as fd:
      examples = [
          cast(IndexedInput, serialize.from_json(line))
          for line in fd.readlines()
      ]

    # Load the side-by-side spec if it exists on disk.
    spec_path = path + LIT_SPEC_EXTENSION
    if os.path.exists(spec_path):
      with open(spec_path, 'r') as fd:
        spec = serialize.from_json(fd.read())
    else:
      spec = None

    description = f'{len(examples)} examples from {path}'
    if self._base is not None:
      description += '\n' + self._base.description()

    return IndexedDataset(
        base=self._base,
        indexed_examples=examples,
        spec=spec,
        description=description,
        id_fn=self.id_fn)

  def __hash__(self):
    return hash(tuple([ex['id'] for ex in self._indexed_examples]))