MCPcopy Create free account
hub / github.com/davisking/dlib / main

Function main

examples/text_classification_ex.cpp:224–357  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

222// ----------------------------------------------------------------------------------------
223
224int main(const int argc, const char** argv)
225try
226{
227 command_line_parser parser;
228 parser.add_option("test-fraction", "fraction of messages kept for testing (default: 0.2)", 1);
229 parser.set_group_name("Help Options");
230 parser.add_option("h", "alias for --help");
231 parser.add_option("help", "display this message and exit");
232 parser.parse(argc, argv);
233
234 if (parser.number_of_arguments() < 1 || parser.option("h") || parser.option("help"))
235 {
236 cout << "This example needs the SMS Spam Collection to run." << endl;
237 cout << "You can get it from https://archive.ics.uci.edu/dataset/228/sms+spam+collection" << endl;
238 cout << "Uncompress it and give the path to the SMSSpamCollection file to this program." << endl;
239 parser.print_options();
240 return EXIT_SUCCESS;
241 }
242 parser.check_option_arg_range("test-fraction", 0.01, 0.99);
243 const double test_fraction = get_option(parser, "test-fraction", 0.2);
244
245 std::vector<string> messages;
246 std::vector<double> labels;
247 load_sms_spam_collection(parser[0], messages, labels);
248 const auto num_spam = std::count(labels.begin(), labels.end(), +1);
249 cout << "# messages: " << messages.size() << " (" << num_spam << " spam)" << endl;
250
251 // Shuffle before splitting, since nothing says the file's order is random.
252 randomize_samples(messages, labels);
253 const size_t num_test = lround(messages.size() * test_fraction);
254 const std::vector<string> test_messages(messages.end() - num_test, messages.end());
255 const std::vector<double> test_labels(labels.end() - num_test, labels.end());
256 messages.resize(messages.size() - num_test);
257 labels.resize(labels.size() - num_test);
258
259 // Learn the TF-IDF weights from the training messages only. Document frequencies are
260 // information about the data set, and computing them on the test messages would make
261 // the test accuracy optimistic.
262 text_features features;
263 features.train(messages);
264 cout << features << endl;
265 std::vector<sample_type> samples;
266 for (const auto& message : messages)
267 samples.push_back(features(message));
268
269 // Now we train the classifier. auto_train_linear_classifier() trains a linear SVM and:
270 // - gives each class a C inversely proportional to its number of messages, so the
271 // 6 times more ham messages don't outweigh the spam ones,
272 // - picks C by 5-fold cross-validation,
273 // - turns the SVM's score into a probability of spam with Platt scaling.
274 // The TF-IDF weights have seen every fold, a small leak that is fine for picking C,
275 // which is why the final numbers come from the test set.
276 const auto df = auto_train_linear_classifier(samples, labels);
277
278 // A linear SVM is one weight per dimension, so we can see what it learned: the
279 // dimensions with the largest weights push a message toward spam, and those with the
280 // most negative weights toward ham. describe() tells us which n-grams they stand for.
281 std::vector<std::pair<double, unsigned long>> weights;

Callers

nothing calls this directly

Calls 15

get_optionFunction · 0.85
load_sms_spam_collectionFunction · 0.85
randomize_samplesFunction · 0.85
parseMethod · 0.80
number_of_argumentsMethod · 0.80
print_optionsMethod · 0.80
emplace_backMethod · 0.80
rbeginMethod · 0.80
serializeFunction · 0.70
deserializeFunction · 0.70
beginMethod · 0.45

Tested by

no test coverage detected