Tokenize input dataset Args: sample (dict): Input data sample. sp_model (str): Path of tokenizer. Returns: tuple: dumped processed data sample and length of tokens.
(sample, sp_model)
| 47 | |
| 48 | |
| 49 | def tokenize(sample, sp_model): |
| 50 | """Tokenize input dataset |
| 51 | |
| 52 | Args: |
| 53 | sample (dict): Input data sample. |
| 54 | sp_model (str): Path of tokenizer. |
| 55 | |
| 56 | Returns: |
| 57 | tuple: dumped processed data sample and length of tokens. |
| 58 | """ |
| 59 | special_tokens_map = {"<eoh>": 103167, "<eoa>": 103166, "nl_id": 13} |
| 60 | token_ids = [sp_model.bos_id()] |
| 61 | human_s = sample["user"] |
| 62 | ass_s = sample["bot"] |
| 63 | |
| 64 | human_ids = sp_model.encode(human_s) + [special_tokens_map["<eoh>"], special_tokens_map["nl_id"]] |
| 65 | human_ids_ignore = [-token_id for token_id in human_ids] |
| 66 | |
| 67 | ass_template_ids = sp_model.encode("<|Bot|>:") |
| 68 | ass_template_ids_ignore = [-token_ids for token_ids in ass_template_ids] |
| 69 | ass_ids = ( |
| 70 | ass_template_ids_ignore |
| 71 | + sp_model.encode(ass_s[8:]) |
| 72 | + [special_tokens_map["<eoa>"], special_tokens_map["nl_id"]] |
| 73 | ) |
| 74 | |
| 75 | token_ids += human_ids_ignore + ass_ids |
| 76 | if len(token_ids) > 2047: |
| 77 | token_ids = token_ids[:2047] |
| 78 | token_ids += [sp_model.eos_id()] |
| 79 | line = str.encode(json.dumps({"tokens": token_ids}) + "\n") |
| 80 | return line, len(token_ids) |
| 81 | |
| 82 | |
| 83 | def dump_bin_meta_bin(samples, path, split_ratio=0.1): |