tokenization arguments.
(parser)
| 291 | |
| 292 | |
| 293 | def add_tokenization_args(parser): |
| 294 | """tokenization arguments.""" |
| 295 | |
| 296 | group = parser.add_argument_group('Tokenization', 'tokenization configurations') |
| 297 | group.add_argument('--tokenizer-type', type=str, default='fake', help='type name of tokenizer') |
| 298 | |
| 299 | # group.add_argument('--img-tokenizer-path', type=str, default=None, |
| 300 | # help='The checkpoint file path of image tokenizer.') |
| 301 | return parser |
| 302 | |
| 303 | def _adjust_vocab_size(args): |
| 304 | before = args.vocab_size |