| 742 | }; |
| 743 | |
| 744 | int main(int argc, char** argv) |
| 745 | { |
| 746 | try |
| 747 | { |
| 748 | // Setup interrupt handling for clean termination |
| 749 | setup_interrupt_handler(); |
| 750 | |
| 751 | command_line_parser parser; |
| 752 | parser.add_option("train", "Train a transformer model on enwiki"); |
| 753 | parser.add_option("generate", "Generate enwiki from a previously trained model"); |
| 754 | parser.add_option("verify", "Verify generated output against original data"); |
| 755 | parser.add_option("tokenize-only", "Only tokenize the input file and save tokens"); |
| 756 | parser.add_option("enwiki", "Path to the enwiki file (default: enwiki.txt)", 1); |
| 757 | parser.add_option("max-tokens", "Maximum number of tokens to load in memory", 1); |
| 758 | parser.add_option("max-bytes", "Maximum number of bytes to process from enwiki", 1); |
| 759 | parser.add_option("percent", "Percentage of enwiki to process (0-100)", 1); |
| 760 | parser.add_option("learning-rate", "Set the learning rate (default: 3e-4)", 1); |
| 761 | parser.add_option("batch-size", "Set the mini-batch size (default: 64)", 1); |
| 762 | parser.add_option("patience", "Iterations without progress before early stopping (default: 15000)", 1); |
| 763 | parser.add_option("max-epochs", "Maximum number of training epochs (default: 10)", 1); |
| 764 | parser.add_option("alpha", "Set the weight decay for Adam (default: 0.004)", 1); |
| 765 | parser.add_option("beta1", "Set Adam's first moment coefficient (default: 0.9)", 1); |
| 766 | parser.add_option("beta2", "Set Adam's second moment coefficient (default: 0.999)", 1); |
| 767 | parser.add_option("model-file", "Path for model (default: dlib_slm_enwiki_model.dat)", 1); |
| 768 | parser.add_option("output-file", "Path for output (default: enwiki_generated.txt)", 1); |
| 769 | parser.add_option("tokenizer", "Path to pre-trained tokenizer (default: enwiki_tokenizer.vocab)", 1); |
| 770 | parser.add_option("tokens-file", "Path to pre-tokenized tokens file (optional)", 1); |
| 771 | parser.add_option("force-tokenize", "Force tokenization even if tokens file exists"); |
| 772 | parser.parse(argc, argv); |
| 773 | |
| 774 | if (parser.number_of_arguments() == 0 && |
| 775 | !parser.option("train") && !parser.option("generate") && |
| 776 | !parser.option("verify") && !parser.option("tokenize-only")) |
| 777 | { |
| 778 | parser.print_options(); |
| 779 | return 0; |
| 780 | } |
| 781 | |
| 782 | // Default values |
| 783 | const double learning_rate = get_option(parser, "learning-rate", 3e-4); |
| 784 | const size_t batch_size = get_option(parser, "batch-size", 64); |
| 785 | const long patience = get_option(parser, "patience", 15000); |
| 786 | const size_t max_epochs = get_option(parser, "max-epochs", 10); |
| 787 | const double alpha = get_option(parser, "alpha", 0.004); |
| 788 | const double beta1 = get_option(parser, "beta1", 0.9); |
| 789 | const double beta2 = get_option(parser, "beta2", 0.999); |
| 790 | const std::string model_file = get_option(parser, "model-file", "dlib_slm_enwiki_model.dat"); |
| 791 | const std::string output_file = get_option(parser, "output-file", "enwiki_generated.txt"); |
| 792 | const std::string enwiki_path = get_option(parser, "enwiki", "enwiki.txt"); |
| 793 | const long max_seq_len = 30; |
| 794 | const long num_layers = 4; |
| 795 | const long num_heads = 6; |
| 796 | const long embedding_dim = 228; |
| 797 | const std::string tokenizer_path = get_option(parser, "tokenizer", "enwiki_tokenizer.vocab"); |
| 798 | // Default number of prompt tokens = input sequence length |
| 799 | const bool force_tokenize = parser.option("force-tokenize"); |
| 800 | const long num_tokens = 8000; |
| 801 |
nothing calls this directly
no test coverage detected