Class StorageConfig

trinity/common/config.py:166–226

Storage config for both taskset and experience buffer. Not visible to users directly. Please use ExperienceBufferConfig or TasksetConfig instead.

@dataclass
class StorageConfig:
    """Storage config for both taskset and experience buffer.
    Not visible to users directly. Please use ExperienceBufferConfig or TasksetConfig instead.
    """

    name: str = ""
    storage_type: str = StorageType.FILE.value
    path: Optional[str] = None
    repeat_times: Optional[int] = None

    # For continuing training
    index: int = 0

    # used for StorageType.FILE
    split: str = "train"
    subset_name: Optional[str] = None
    format: FormatConfig = field(default_factory=FormatConfig)

    # used for StorageType.QUEUE
    capacity: int = 10000
    max_read_timeout: float = 1800
    replay_buffer: Optional[ReplayBufferConfig] = field(default_factory=ReplayBufferConfig)

    # used for StorageType.SQL
    max_retry_times: int = 3
    max_retry_interval: int = 1

    # used for rollout tasks
    default_workflow_type: Optional[str] = None
    default_reward_fn_type: Optional[str] = None
    rollout_args: GenerationConfig = field(default_factory=GenerationConfig)
    workflow_args: dict = field(default_factory=dict)
    reward_fn_args: dict = field(default_factory=dict)
    data_selector: DataSelectorConfig = field(default_factory=DataSelectorConfig)

    # enable progress bar (tqdm) for _HFBatchReader
    enable_progress_bar: Optional[bool] = False

    # get storage from existing experiment
    ray_namespace: Optional[str] = None

    # ! DO NOT SET except you know what you are doing
    wrap_in_ray: bool = True

    # ! DO NOT SET, automatically set
    schema_type: Optional[str] = None

    # ! DO NOT SET, automatically set from buffer.total_epochs
    total_epochs: int = 1  # automatically set

    # ! DO NOT SET, automatically set from buffer.total_steps
    total_steps: Optional[int] = None  # automatically set

    # ! DO NOT SET, automatically set from buffer.batch_size / train_batch_size
    batch_size: int = 0

    # ! DO NOT SET, automatically set from model.model_path
    tokenizer_path: Optional[str] = None