Initializes the configuration for language models. Args: unk_token: The unknown token. bos_token: The start of sentence token. eos_token: The end of sentence token. **kwargs: Additional configuration.
(
self,
unk_token: str = "<unk>",
bos_token: str = "<s>",
eos_token: str = "</s>",
**kwargs,
)
| 553 | """Configuration for language models.""" |
| 554 | |
| 555 | def __init__( |
| 556 | self, |
| 557 | unk_token: str = "<unk>", |
| 558 | bos_token: str = "<s>", |
| 559 | eos_token: str = "</s>", |
| 560 | **kwargs, |
| 561 | ): |
| 562 | """Initializes the configuration for language models. |
| 563 | |
| 564 | Args: |
| 565 | unk_token: The unknown token. |
| 566 | bos_token: The start of sentence token. |
| 567 | eos_token: The end of sentence token. |
| 568 | **kwargs: Additional configuration. |
| 569 | """ |
| 570 | super().__init__( |
| 571 | unk_token=unk_token, |
| 572 | bos_token=bos_token, |
| 573 | eos_token=eos_token, |
| 574 | **kwargs, |
| 575 | ) |
| 576 | |
| 577 | |
| 578 | class LanguageModelSpec(ModelSpec): |