Method addDocs

src/python/handler/py_utils.cpp:358–445

}

size_t CorpusObject::addDocs(PyObject* tokenizedIter, PyObject* rawIter, PyObject* metadataIter)
{
	if (!isIndependent())
		throw py::RuntimeError{ "Cannot modify the corpus bound to a topic model." };

	size_t cnt = 0;
	py::UniqueObj stopwords{ PyObject_GetAttrString((PyObject*)this, "_stopwords") };

	py::foreach<PyObject*>(tokenizedIter, [&](PyObject* tokenized)
	{
		tomoto::RawDoc doc;
		py::foreach<PyObject*>(tokenized, [&](PyObject* t)
		{
			if (PyUnicode_Check(t))
			{
				doc.words.emplace_back(vocab->vocabs->add(py::toCpp<std::string>(t)));
			}
			else if (t == Py_None)
			{
				doc.words.emplace_back(tomoto::non_vocab_id);
			}
			else if (PyTuple_Size(t) == 3)
			{
				PyObject* word = PyTuple_GetItem(t, 0);
				PyObject* pos = PyTuple_GetItem(t, 1);
				PyObject* len = PyTuple_GetItem(t, 2);
				if (!((PyUnicode_Check(word) || word == Py_None) && PyLong_Check(pos) && PyLong_Check(len))) throw py::ValueError{ "`tokenizer` must return an iterable of `str` or `tuple` of (`str`, `int`, `int`)." };

				bool isStopword = false;
				if (stopwords.get() != Py_None)
				{
					py::UniqueObj stopRet{ PyObject_CallObject(stopwords.get(), py::buildPyTuple(word).get())};
					if (!stopRet) throw py::ExcPropagation{};
					isStopword = PyObject_IsTrue(stopRet.get());
				}
				else if (word == Py_None)
				{
					isStopword = true;
				}
				doc.words.emplace_back(isStopword ? tomoto::non_vocab_id : vocab->vocabs->add(py::toCpp<std::string>(word)));
				doc.origWordPos.emplace_back(PyLong_AsLong(pos));
				doc.origWordLen.emplace_back(PyLong_AsLong(len));
			}
			else
			{
				throw py::ValueError{ "`tokenizer` must return an iterable of `str` or `tuple` of (`str`, `int`, `int`)." };
			}
		}, "`tokenizer` must return an iterable of `str` or `tuple` of (`str`, `int`, `int`).");
		py::UniqueObj raw{ PyIter_Next(rawIter) };
		if (!raw) throw py::ExcPropagation{};
		py::UniqueObj metadata{ PyIter_Next(metadataIter) };
		if (!metadata) throw py::ExcPropagation{};

		doc.rawStr = tomoto::SharedString{ py::toCpp<std::string>(raw.get()) };

		PyObject* key, * value;
		Py_ssize_t p = 0;
		while (PyDict_Next(metadata.get(), &p, &key, &value))