Method addDoc

src/python/handler/py_utils.cpp:249–356

}

size_t CorpusObject::addDoc(PyObject* words, PyObject* raw, PyObject* userData, PyObject* additionalKwargs)
{
	if (!isIndependent())
		throw py::RuntimeError{ "Cannot modify the corpus bound to a topic model." };
	tomoto::RawDoc doc;

	py::UniqueObj stopwords{ PyObject_GetAttrString((PyObject*)this, "_stopwords") };

	if (PyObject_HasAttrString((PyObject*)this, "_tokenizer")
		&& PyObject_IsTrue(py::UniqueObj{ PyObject_GetAttrString((PyObject*)this, "_tokenizer") }.get()))
	{
		if (words && words != Py_None) throw py::ValueError{ "only `raw` is required when `tokenizer` is provided." };
		if (!PyObject_IsTrue(raw)) return py::buildPyValue(-1);

		py::UniqueObj tokenizer{ PyObject_GetAttrString((PyObject*)this, "_tokenizer") };

		py::UniqueObj args{ PyTuple_New(1) };
		Py_INCREF(raw);
		PyTuple_SetItem(args.get(), 0, raw);
		py::UniqueObj kwargs{ PyDict_New() };
		PyDict_SetItemString(kwargs.get(), "user_data", userData);

		py::UniqueObj ret{ PyObject_Call(tokenizer.get(), args.get(), kwargs.get())};
		if (!ret) throw py::ExcPropagation{};
		py::foreach<PyObject*>(ret.get(), [&](PyObject* t)
		{
			if (PyUnicode_Check(t))
			{
				doc.words.emplace_back(vocab->vocabs->add(py::toCpp<std::string>(t)));
			}
			else if (t == Py_None)
			{
				doc.words.emplace_back(tomoto::non_vocab_id);
			}
			else if (PyTuple_Size(t) == 3)
			{
				PyObject* word = PyTuple_GetItem(t, 0);
				PyObject* pos = PyTuple_GetItem(t, 1);
				PyObject* len = PyTuple_GetItem(t, 2);
				if (!((PyUnicode_Check(word) || word == Py_None) && PyLong_Check(pos) && PyLong_Check(len))) throw py::ValueError{ "`tokenizer` must return an iterable of `str` or `tuple` of (`str`, `int`, `int`)." };
				bool isStopword = false;
				if (stopwords.get() != Py_None)
				{
					py::UniqueObj stopRet{ PyObject_CallObject(stopwords.get(), py::buildPyTuple(word).get())};
					if (!stopRet) throw py::ExcPropagation{};
					isStopword = PyObject_IsTrue(stopRet.get());
				}
				else if (word == Py_None)
				{
					isStopword = true;
				}
				doc.words.emplace_back(isStopword ? tomoto::non_vocab_id : vocab->vocabs->add(py::toCpp<std::string>(word)));
				doc.origWordPos.emplace_back(PyLong_AsLong(pos));
				doc.origWordLen.emplace_back(PyLong_AsLong(len));
			}
			else
			{
				throw py::ValueError{ "`tokenizer` must return an iterable of `str` or `tuple` of (`str`, `int`, `int`)." };