Method python_reader

src/data_readers/data_reader_python.cpp:43–84

namespace lbann {

python_reader::python_reader(std::string module,
                             std::string module_dir,
                             std::string sample_function,
                             std::string num_samples_function,
                             std::string sample_dims_function,
                             bool shuffle)
  : generic_data_reader(shuffle)
{

  // Make sure Python is running and acquire GIL
  python::global_interpreter_lock gil;

  // Import Python module for data
  if (!module_dir.empty()) {
    auto path = PySys_GetObject("path"); // Borrowed reference
    PyList_Append(path, python::object(module_dir));
    python::check_error();
  }
  python::object data_module = PyImport_ImportModule(module.c_str());

  // Get number of samples
  python::object num_func =
    PyObject_GetAttrString(data_module, num_samples_function.c_str());
  python::object num = PyObject_CallObject(num_func, nullptr);
  m_num_samples = PyLong_AsLong(num);
  python::check_error();

  // Get sample dimensions
  python::object dims_func =
    PyObject_GetAttrString(data_module, sample_dims_function.c_str());
  python::object dims = PyObject_CallObject(dims_func, nullptr);
  dims = PyObject_GetIter(dims);
  for (auto d = PyIter_Next(dims); d != nullptr; d = PyIter_Next(dims)) {
    m_sample_dims.push_back(PyLong_AsLong(d));
    Py_DECREF(d);
  }
  python::check_error();

  // Get sample access function
  m_sample_function =
    PyObject_GetAttrString(data_module, sample_function.c_str());
}

python_reader::~python_reader()
{