| 149 | } |
| 150 | |
| 151 | static Status ValidateArrowVsJson(const std::string& arrow_path, |
| 152 | const std::string& json_path) { |
| 153 | // Construct JSON reader |
| 154 | ARROW_ASSIGN_OR_RAISE(auto json_file, io::ReadableFile::Open(json_path)); |
| 155 | |
| 156 | ARROW_ASSIGN_OR_RAISE(int64_t file_size, json_file->GetSize()); |
| 157 | ARROW_ASSIGN_OR_RAISE(auto json_buffer, json_file->Read(file_size)); |
| 158 | |
| 159 | ARROW_ASSIGN_OR_RAISE(auto json_reader, IntegrationJsonReader::Open(json_buffer)); |
| 160 | |
| 161 | // Construct Arrow reader |
| 162 | ARROW_ASSIGN_OR_RAISE(auto arrow_file, io::ReadableFile::Open(arrow_path)); |
| 163 | |
| 164 | std::shared_ptr<ipc::RecordBatchFileReader> arrow_reader; |
| 165 | ARROW_ASSIGN_OR_RAISE(arrow_reader, ipc::RecordBatchFileReader::Open(arrow_file.get())); |
| 166 | |
| 167 | auto json_schema = json_reader->schema(); |
| 168 | auto arrow_schema = arrow_reader->schema(); |
| 169 | |
| 170 | if (!json_schema->Equals(*arrow_schema)) { |
| 171 | std::stringstream ss; |
| 172 | ss << "JSON schema: \n" |
| 173 | << json_schema->ToString(/* show_metadata = */ true) << "\n\n" |
| 174 | << "Arrow schema: \n" |
| 175 | << arrow_schema->ToString(/* show_metadata = */ true) << "\n"; |
| 176 | |
| 177 | if (FLAGS_verbose) { |
| 178 | std::cout << ss.str() << std::endl; |
| 179 | } |
| 180 | return Status::Invalid("Schemas did not match"); |
| 181 | } |
| 182 | |
| 183 | const int json_nbatches = json_reader->num_record_batches(); |
| 184 | const int arrow_nbatches = arrow_reader->num_record_batches(); |
| 185 | |
| 186 | if (json_nbatches != arrow_nbatches) { |
| 187 | return Status::Invalid("Different number of record batches: ", json_nbatches, |
| 188 | " (JSON) vs ", arrow_nbatches, " (Arrow)"); |
| 189 | } |
| 190 | |
| 191 | std::shared_ptr<RecordBatch> arrow_batch; |
| 192 | std::shared_ptr<RecordBatch> json_batch; |
| 193 | for (int i = 0; i < json_nbatches; ++i) { |
| 194 | ARROW_ASSIGN_OR_RAISE(json_batch, json_reader->ReadRecordBatch(i)); |
| 195 | ARROW_ASSIGN_OR_RAISE(arrow_batch, arrow_reader->ReadRecordBatch(i)); |
| 196 | Status valid_st = ValidateFull(*json_batch); |
| 197 | if (!valid_st.ok()) { |
| 198 | return Status::Invalid("JSON record batch ", i, " did not validate:\n", |
| 199 | valid_st.ToString()); |
| 200 | } |
| 201 | valid_st = ValidateFull(*arrow_batch); |
| 202 | if (!valid_st.ok()) { |
| 203 | return Status::Invalid("Arrow record batch ", i, " did not validate:\n", |
| 204 | valid_st.ToString()); |
| 205 | } |
| 206 | |
| 207 | if (!json_batch->ApproxEquals(*arrow_batch)) { |
| 208 | std::stringstream ss; |
no test coverage detected