Return true if any of the arrays was changed (including descendents)
| 453 | |
| 454 | // Return true if any of the arrays was changed (including descendents) |
| 455 | Result<bool> Unify(std::shared_ptr<DataType> type, ArrayDataVector* chunks) { |
| 456 | DCHECK(!chunks->empty()); |
| 457 | bool changed = false; |
| 458 | std::shared_ptr<DataType> ext_type = nullptr; |
| 459 | |
| 460 | if (type->id() == Type::EXTENSION) { |
| 461 | ext_type = std::move(type); |
| 462 | type = checked_cast<const ExtensionType&>(*ext_type).storage_type(); |
| 463 | } |
| 464 | |
| 465 | // Unify all child dictionaries (if any) |
| 466 | if (type->num_fields() > 0) { |
| 467 | ArrayDataVector children(chunks->size()); |
| 468 | for (int i = 0; i < type->num_fields(); ++i) { |
| 469 | std::transform(chunks->begin(), chunks->end(), children.begin(), |
| 470 | [i](const std::shared_ptr<ArrayData>& array) { |
| 471 | return array->child_data[i]; |
| 472 | }); |
| 473 | ARROW_ASSIGN_OR_RAISE(bool child_changed, |
| 474 | Unify(type->field(i)->type(), &children)); |
| 475 | if (child_changed) { |
| 476 | // Only do this when unification actually occurred |
| 477 | for (size_t j = 0; j < chunks->size(); ++j) { |
| 478 | (*chunks)[j]->child_data[i] = std::move(children[j]); |
| 479 | } |
| 480 | changed = true; |
| 481 | } |
| 482 | } |
| 483 | } |
| 484 | |
| 485 | // Unify this dictionary |
| 486 | if (type->id() == Type::DICTIONARY) { |
| 487 | const auto& dict_type = checked_cast<const DictionaryType&>(*type); |
| 488 | // XXX Ideally, we should unify dictionaries nested in value_type first, |
| 489 | // but DictionaryUnifier doesn't supported nested dictionaries anyway, |
| 490 | // so this will fail. |
| 491 | ARROW_ASSIGN_OR_RAISE(auto unifier, |
| 492 | DictionaryUnifier::Make(dict_type.value_type(), this->pool)); |
| 493 | // Unify all dictionary array chunks |
| 494 | BufferVector transpose_maps(chunks->size()); |
| 495 | for (size_t j = 0; j < chunks->size(); ++j) { |
| 496 | DCHECK_NE((*chunks)[j]->dictionary, nullptr); |
| 497 | RETURN_NOT_OK( |
| 498 | unifier->Unify(*MakeArray((*chunks)[j]->dictionary), &transpose_maps[j])); |
| 499 | } |
| 500 | std::shared_ptr<Array> dictionary; |
| 501 | RETURN_NOT_OK(unifier->GetResultWithIndexType(dict_type.index_type(), &dictionary)); |
| 502 | for (size_t j = 0; j < chunks->size(); ++j) { |
| 503 | ARROW_ASSIGN_OR_RAISE( |
| 504 | (*chunks)[j], |
| 505 | TransposeDictIndices( |
| 506 | (*chunks)[j], type, type, dictionary->data(), |
| 507 | reinterpret_cast<const int32_t*>(transpose_maps[j]->data()), this->pool)); |
| 508 | if (ext_type) { |
| 509 | (*chunks)[j]->type = ext_type; |
| 510 | } |
| 511 | } |
| 512 | changed = true; |