Class ExperienceBufferConfig

trinity/common/config.py:298–359

Storage Config for trainer input experience buffer.

@dataclass
class ExperienceBufferConfig:
    """Storage Config for trainer input experience buffer."""

    name: str = ""
    storage_type: str = StorageType.QUEUE.value
    path: Optional[str] = None

    # used for StorageType.QUEUE
    capacity: int = 10000
    max_read_timeout: float = 1800
    replay_buffer: Optional[ReplayBufferConfig] = field(default_factory=ReplayBufferConfig)

    # used for StorageType.SQL
    max_retry_times: int = 3
    max_retry_interval: int = 1

    # used for StorageType.FILE
    split: str = "train"
    subset_name: Optional[str] = None
    format: FormatConfig = field(default_factory=FormatConfig)
    enable_progress_bar: Optional[bool] = False
    data_selector: DataSelectorConfig = field(default_factory=DataSelectorConfig)

    # ! DO NOT SET, automatically set
    schema_type: Optional[str] = None
    # ! DO NOT SET
    index: int = 0
    # ! DO NOT SET, automatically set from buffer.batch_size
    batch_size: int = 0
    # ! DO NOT SET, automatically set from model.model_path
    tokenizer_path: Optional[str] = None
    # ! DO NOT SET, automatically set from buffer.total_epochs
    total_epochs: int = 1  # automatically set
    # ! DO NOT SET, automatically set from buffer.total_steps
    total_steps: Optional[int] = None  # automatically set
    # ! DO NOT SET, automatically set from ray_namespace
    ray_namespace: Optional[str] = None

    def to_storage_config(self) -> StorageConfig:
        storage_config = StorageConfig(
            name=self.name,
            storage_type=self.storage_type,
            path=self.path,
            data_selector=self.data_selector,
            capacity=self.capacity,
            max_read_timeout=self.max_read_timeout,
            replay_buffer=self.replay_buffer,
            max_retry_times=self.max_retry_times,
            max_retry_interval=self.max_retry_interval,
            split=self.split,
            subset_name=self.subset_name,
            format=self.format,
            enable_progress_bar=self.enable_progress_bar,
            schema_type=self.schema_type,
            index=self.index,
            batch_size=self.batch_size,
            tokenizer_path=self.tokenizer_path,
            total_epochs=self.total_epochs,