| 56 | """ Example processor that converts a dictionary of texts into tokens. """ |
| 57 | @staticmethod |
| 58 | def get_default_config(updates=None): |
| 59 | config = ConfigDict() |
| 60 | config.fields_from_example = '' |
| 61 | config.fields = '' |
| 62 | config.subfield_separator = ' ' |
| 63 | config.add_bos_token = True |
| 64 | config.add_eos_token = True |
| 65 | config.prepend_text = '' |
| 66 | if updates is not None: |
| 67 | config.update(ConfigDict(updates).copy_and_resolve_references()) |
| 68 | return config |
| 69 | |
| 70 | def __init__(self, config, tokenizer): |
| 71 | self.config = self.get_default_config(config) |