Function _write_index_file

tensorflow_datasets/core/writer.py:159–178

Writes index file for records of a shard at given `sharded_index_path`. NOTE: Each record position (i.e shard_key in shard_keys) is stored as a string for better readability of the index files. The reader should parse the position from string using `RecordPosition.from_str()` method. Args:

(
    sharded_index_path: epath.PathLike, record_keys: Sequence[Any]
)

def _write_index_file(
    sharded_index_path: epath.PathLike, record_keys: Sequence[Any]
):
  """Writes index file for records of a shard at given `sharded_index_path`.

  NOTE: Each record position (i.e shard_key in shard_keys) is stored as a
  string for better readability of the index files. The reader should parse the
  position from string using `RecordPosition.from_str()` method.

  Args:
    sharded_index_path: Path to the sharded index path.
    record_keys: Sequence of keys/indices of the records in the shard.
  """
  # Store string representation of each record position.
  #
  # NOTE: This makes the index file more readable. Although the reader should
  # parse the record position from a string.
  index_info = {"index": [str(record_key) for record_key in record_keys]}
  epath.Path(sharded_index_path).write_text(json.dumps(index_info))
  logging.info("Wrote index file to %s", os.fspath(sharded_index_path))


class ExampleWriter: