| 150 | } |
| 151 | |
| 152 | static Status ValidateArrowVsJson(const std::string& arrow_path, |
| 153 | const std::string& json_path) { |
| 154 | // Construct JSON reader |
| 155 | ARROW_ASSIGN_OR_RAISE(auto json_file, io::ReadableFile::Open(json_path)); |
| 156 | |
| 157 | ARROW_ASSIGN_OR_RAISE(int64_t file_size, json_file->GetSize()); |
| 158 | ARROW_ASSIGN_OR_RAISE(auto json_buffer, json_file->Read(file_size)); |
| 159 | |
| 160 | ARROW_ASSIGN_OR_RAISE(auto json_reader, IntegrationJsonReader::Open(json_buffer)); |
| 161 | |
| 162 | // Construct Arrow reader |
| 163 | ARROW_ASSIGN_OR_RAISE(auto arrow_file, io::ReadableFile::Open(arrow_path)); |
| 164 | |
| 165 | std::shared_ptr<ipc::RecordBatchFileReader> arrow_reader; |
| 166 | ARROW_ASSIGN_OR_RAISE(arrow_reader, ipc::RecordBatchFileReader::Open(arrow_file.get())); |
| 167 | |
| 168 | auto json_schema = json_reader->schema(); |
| 169 | auto arrow_schema = arrow_reader->schema(); |
| 170 | |
| 171 | if (!json_schema->Equals(*arrow_schema)) { |
| 172 | std::stringstream ss; |
| 173 | ss << "JSON schema: \n" |
| 174 | << json_schema->ToString(/* show_metadata = */ true) << "\n\n" |
| 175 | << "Arrow schema: \n" |
| 176 | << arrow_schema->ToString(/* show_metadata = */ true) << "\n"; |
| 177 | |
| 178 | if (FLAGS_verbose) { |
| 179 | std::cout << ss.str() << std::endl; |
| 180 | } |
| 181 | return Status::Invalid("Schemas did not match"); |
| 182 | } |
| 183 | |
| 184 | const int json_nbatches = json_reader->num_record_batches(); |
| 185 | const int arrow_nbatches = arrow_reader->num_record_batches(); |
| 186 | |
| 187 | if (json_nbatches != arrow_nbatches) { |
| 188 | return Status::Invalid("Different number of record batches: ", json_nbatches, |
| 189 | " (JSON) vs ", arrow_nbatches, " (Arrow)"); |
| 190 | } |
| 191 | |
| 192 | std::shared_ptr<RecordBatch> arrow_batch; |
| 193 | std::shared_ptr<RecordBatch> json_batch; |
| 194 | for (int i = 0; i < json_nbatches; ++i) { |
| 195 | ARROW_ASSIGN_OR_RAISE(json_batch, json_reader->ReadRecordBatch(i)); |
| 196 | ARROW_ASSIGN_OR_RAISE(arrow_batch, arrow_reader->ReadRecordBatch(i)); |
| 197 | Status valid_st = ValidateFull(*json_batch); |
| 198 | if (!valid_st.ok()) { |
| 199 | return Status::Invalid("JSON record batch ", i, " did not validate:\n", |
| 200 | valid_st.ToString()); |
| 201 | } |
| 202 | valid_st = ValidateFull(*arrow_batch); |
| 203 | if (!valid_st.ok()) { |
| 204 | return Status::Invalid("Arrow record batch ", i, " did not validate:\n", |
| 205 | valid_st.ToString()); |
| 206 | } |
| 207 | |
| 208 | if (!json_batch->ApproxEquals(*arrow_batch)) { |
| 209 | std::stringstream ss; |
no test coverage detected