| 111 | } |
| 112 | |
| 113 | bool python_reader::fetch_data_block( |
| 114 | std::map<data_field_type, CPUMat*>& input_buffers, |
| 115 | El::Int current_position_in_data_set, |
| 116 | El::Int block_offset, |
| 117 | El::Int block_stride, |
| 118 | El::Int sample_stride, |
| 119 | El::Int mb_size, |
| 120 | El::Matrix<El::Int>& indices_fetched, |
| 121 | const execution_mode mode) |
| 122 | { |
| 123 | |
| 124 | CPUMat& X = *(input_buffers[INPUT_DATA_TYPE_SAMPLES]); |
| 125 | // Acquire Python GIL on first IO thread |
| 126 | // Note: Do nothing on other IO threads. |
| 127 | if (block_offset != 0) { |
| 128 | return true; |
| 129 | } |
| 130 | python::global_interpreter_lock gil; |
| 131 | |
| 132 | // Check that shared memory array is large enough |
| 133 | const El::Int sample_size = get_linearized_data_size(); |
| 134 | const El::Int array_size = PyObject_Length(m_shared_memory_array); |
| 135 | if (array_size < sample_size * mb_size) { |
| 136 | std::stringstream err; |
| 137 | err << "Python data reader attempted to load " |
| 138 | << sample_size * mb_size * sizeof(DataType) << " B " |
| 139 | << "into shared memory array, but only " |
| 140 | << array_size * sizeof(DataType) << " B is available"; |
| 141 | LBANN_ERROR(err.str()); |
| 142 | } |
| 143 | |
| 144 | // Get arguments for sample access function |
| 145 | python::object args_list = PyList_New(0); |
| 146 | for (El::Int i = 0; i < mb_size; ++i) { |
| 147 | El::Int sample_index = |
| 148 | m_shuffled_indices[current_position_in_data_set + i * sample_stride]; |
| 149 | El::Int array_offset = sample_size * i; |
| 150 | PyList_Append( |
| 151 | args_list, |
| 152 | python::object(Py_BuildValue("(l,l)", sample_index, array_offset))); |
| 153 | indices_fetched.Set(i, 0, sample_index); |
| 154 | } |
| 155 | |
| 156 | // Get samples using Python process pool |
| 157 | python::object samples = PyObject_CallMethod(m_process_pool, |
| 158 | "starmap", |
| 159 | "(O,O)", |
| 160 | m_sample_function_wrapper.get(), |
| 161 | args_list.get()); |
| 162 | |
| 163 | // Copy data from shared memory to output matrix |
| 164 | CPUMat shared_memory_matrix(sample_size, |
| 165 | mb_size, |
| 166 | m_shared_memory_array_ptr, |
| 167 | sample_size); |
| 168 | El::Copy(shared_memory_matrix, X); |
| 169 | |
| 170 | return true; |