Function Extract

src/tools/PhraseExtract.cpp:28–53

using opencc::UTF8StringSlice;

void Extract(const std::vector<std::string>& inputFiles,
             const std::string& outputFile) {
  std::ostringstream buffer;
  for (const auto& inputFile : inputFiles) {
    std::ifstream ifs(inputFile);
    const std::string contents((std::istreambuf_iterator<char>(ifs)),
                               (std::istreambuf_iterator<char>()));
    buffer << contents;
  }
  const std::string& text = buffer.str();
  PhraseExtract extractor;
  extractor.SetWordMaxLength(2);
  extractor.SetPrefixSetLength(1);
  extractor.SetSuffixSetLength(1);
  extractor.Extract(text);
  std::ofstream ofs(outputFile);
  for (const auto& word : extractor.Words()) {
    const PhraseExtract::Signals& signals = extractor.Signal(word);
    const double entropy = signals.prefixEntropy + signals.suffixEntropy;
    const double logProbablity = extractor.LogProbability(word);
    ofs << word << " " << signals.frequency << " " << logProbablity << " "
        << signals.cohesion << " " << entropy << " " << signals.prefixEntropy
        << " " << signals.suffixEntropy << std::endl;
  }
  ofs.close();
}

int main(int argc, const char* argv[]) {
  try {