| 13 | TitleHistory2SidSFTDataset, PreferenceSFTDataset, UserPreference2sidSFTDataset |
| 14 | ) |
| 15 | class MockTokenizer: |
| 16 | def __init__(self): |
| 17 | self.pad_token_id = 0 |
| 18 | self.eos_token_id = 3 |
| 19 | self.bos_token_id = 2 |
| 20 | |
| 21 | def encode(self, text, bos=False, eos=False): |
| 22 | # Simple mock encoding - just return list of integers based on text length |
| 23 | tokens = list(range(10, 10 + min(len(text), 50))) # Limit to 50 tokens max |
| 24 | if bos: |
| 25 | tokens = [self.bos_token_id] + tokens |
| 26 | if eos: |
| 27 | tokens = tokens + [self.eos_token_id] |
| 28 | return tokens |
| 29 | |
| 30 | def create_minimal_csv(file_path, data): |
| 31 | """Helper to create minimal CSV files for testing""" |