(
self,
tokenizer,
dataset,
infinite=False,
seq_length=1024,
num_of_sequences=1024,
chars_per_token=3.6,
input_column_name="prompt",
output_column_name="completion"
)
| 140 | """ |
| 141 | |
| 142 | def __init__( |
| 143 | self, |
| 144 | tokenizer, |
| 145 | dataset, |
| 146 | infinite=False, |
| 147 | seq_length=1024, |
| 148 | num_of_sequences=1024, |
| 149 | chars_per_token=3.6, |
| 150 | input_column_name="prompt", |
| 151 | output_column_name="completion" |
| 152 | ): |
| 153 | self.tokenizer = tokenizer |
| 154 | self.concat_token_id = tokenizer.eos_token_id if tokenizer.eos_token_id is not None else args.eos_token_id |
| 155 | self.dataset = dataset |
| 156 | self.seq_length = seq_length |
| 157 | self.infinite = infinite |
| 158 | self.current_size = 0 |
| 159 | self.max_buffer_size = seq_length * chars_per_token * num_of_sequences |
| 160 | self.input_column_name = input_column_name |
| 161 | self.output_column_name = output_column_name |
| 162 | |
| 163 | def __iter__(self): |
| 164 | iterator = iter(self.dataset) |
nothing calls this directly
no outgoing calls
no test coverage detected