Method extractNgrams

src/python/handler/py_utils.cpp:447–490

}

py::UniqueObj CorpusObject::extractNgrams(size_t minCf, size_t minDf, size_t maxLen, size_t maxCand,
	float minScore, bool normalized, size_t workers) const
{
	if (!isIndependent())
		throw py::RuntimeError{ "Cannot modify the corpus bound to a topic model." };
	size_t vSize = vocab->vocabs->size();
	vector<size_t> cf(vSize),
		df(vSize),
		odf(vSize);
	for (auto& d : docs)
	{
		for (auto w : d.words)
		{
			if (w == tomoto::non_vocab_id) continue;
			odf[w] = 1;
			cf[w]++;
		}

		for (size_t i = 0; i < df.size(); ++i) df[i] += odf[i];
		fill(odf.begin(), odf.end(), 0);
	}

	auto tx = [](const tomoto::RawDoc& raw)
	{
		return RawDocWrapper{ raw };
	};
	auto docBegin = tomoto::makeTransformIter(docs.begin(), tx);
	auto docEnd = tomoto::makeTransformIter(docs.end(), tx);
	auto cands = tomoto::phraser::extractPMINgrams(docBegin, docEnd,
		cf, df,
		minCf, minDf, 2, maxLen, maxCand, minScore, normalized
	);

	auto ret = py::UniqueObj{ PyList_New(0) };
	for (auto& c : cands)
	{
		auto item = py::makeNewObject<CandidateObject>();
		item->corpus = py::UniqueCObj<CorpusObject>{ (CorpusObject*)this };
		Py_INCREF(this);
		item->cand = move(c);
		PyList_Append(ret.get(), (PyObject*)item.get());
	}
	return ret;
}

// TODO: It loses some ngram patterns. Fix me!
size_t CorpusObject::concatNgrams(PyObject* cands, const std::string& delimiter)