| 63 | } |
| 64 | |
| 65 | bool NumpyDecoder::SetupImpl(std::vector<OutputDesc> &output_desc, const Workspace &ws) { |
| 66 | const auto &input = ws.Input<CPUBackend>(0); |
| 67 | |
| 68 | headers_.clear(); |
| 69 | headers_.reserve(input.num_samples()); |
| 70 | for (int sampleIdx = 0; sampleIdx < input.num_samples(); sampleIdx++) { |
| 71 | const auto sampleData = std::string_view(static_cast<const char *>(input.raw_tensor(sampleIdx)), |
| 72 | volume(input.tensor_shape(sampleIdx))); |
| 73 | DALI_ENFORCE(!sampleData.empty(), "Input sample is empty. Expected a non-empty NPY file."); |
| 74 | headers_.emplace_back(ParseHeader(sampleData)); |
| 75 | } |
| 76 | |
| 77 | output_desc.resize(1); |
| 78 | if (dtype_override_.has_value()) { |
| 79 | output_desc[0].type = dtype_override_.value(); |
| 80 | } else { |
| 81 | if (!dtype_.has_value()) { |
| 82 | dtype_ = headers_.front().type(); |
| 83 | } |
| 84 | output_desc[0].type = dtype_.value(); |
| 85 | DALI_ENFORCE( |
| 86 | std::all_of( |
| 87 | headers_.begin(), headers_.end(), |
| 88 | [&](const numpy::HeaderData &header) { return header.type() == dtype_.value(); }), |
| 89 | "All samples in the dataset must have the same data type, but got differing types"); |
| 90 | } |
| 91 | |
| 92 | // Set the number of dimensions for the output shape based on the very first sample |
| 93 | if (!ndim_.has_value()) { |
| 94 | ndim_ = headers_.front().shape.sample_dim(); |
| 95 | } |
| 96 | TensorListShape<-1> output_shape(headers_.size(), ndim_.value()); |
| 97 | |
| 98 | for (int sampleIdx = 0; sampleIdx < input.num_samples(); ++sampleIdx) { |
| 99 | const auto &header = headers_[sampleIdx]; |
| 100 | if (header.shape.sample_dim() != output_shape.sample_dim()) { |
| 101 | DALI_FAIL( |
| 102 | make_string("All samples in the dataset must have the same number of dimensions, " |
| 103 | "but got differing sample dimensions: ", |
| 104 | output_shape.sample_dim(), " and ", header.shape.sample_dim())); |
| 105 | } |
| 106 | if (header.fortran_order) { |
| 107 | // Fortran order means the shape is transposed, we need to reverse |
| 108 | auto transposed_shape = TensorShape<-1>::empty_shape(output_shape.sample_dim()); |
| 109 | std::reverse_copy(header.shape.begin(), header.shape.end(), transposed_shape.begin()); |
| 110 | output_shape.set_tensor_shape(sampleIdx, transposed_shape); |
| 111 | } else { |
| 112 | output_shape.set_tensor_shape(sampleIdx, header.shape); |
| 113 | } |
| 114 | } |
| 115 | output_desc[0].shape = std::move(output_shape); |
| 116 | |
| 117 | return true; |
| 118 | } |
| 119 | |
| 120 | void RunDecoding(SampleView<CPUBackend> outputSample, ConstSampleView<CPUBackend> inputView, |
| 121 | const numpy::HeaderData &header) { |
nothing calls this directly
no test coverage detected