Return true if any of the arrays was changed (including descendents)
| 451 | |
| 452 | // Return true if any of the arrays was changed (including descendents) |
| 453 | Result<bool> Unify(std::shared_ptr<DataType> type, ArrayDataVector* chunks) { |
| 454 | DCHECK(!chunks->empty()); |
| 455 | bool changed = false; |
| 456 | std::shared_ptr<DataType> ext_type = nullptr; |
| 457 | |
| 458 | if (type->id() == Type::EXTENSION) { |
| 459 | ext_type = std::move(type); |
| 460 | type = checked_cast<const ExtensionType&>(*ext_type).storage_type(); |
| 461 | } |
| 462 | |
| 463 | // Unify all child dictionaries (if any) |
| 464 | if (type->num_fields() > 0) { |
| 465 | ArrayDataVector children(chunks->size()); |
| 466 | for (int i = 0; i < type->num_fields(); ++i) { |
| 467 | std::transform(chunks->begin(), chunks->end(), children.begin(), |
| 468 | [i](const std::shared_ptr<ArrayData>& array) { |
| 469 | return array->child_data[i]; |
| 470 | }); |
| 471 | ARROW_ASSIGN_OR_RAISE(bool child_changed, |
| 472 | Unify(type->field(i)->type(), &children)); |
| 473 | if (child_changed) { |
| 474 | // Only do this when unification actually occurred |
| 475 | for (size_t j = 0; j < chunks->size(); ++j) { |
| 476 | (*chunks)[j]->child_data[i] = std::move(children[j]); |
| 477 | } |
| 478 | changed = true; |
| 479 | } |
| 480 | } |
| 481 | } |
| 482 | |
| 483 | // Unify this dictionary |
| 484 | if (type->id() == Type::DICTIONARY) { |
| 485 | const auto& dict_type = checked_cast<const DictionaryType&>(*type); |
| 486 | // XXX Ideally, we should unify dictionaries nested in value_type first, |
| 487 | // but DictionaryUnifier doesn't supported nested dictionaries anyway, |
| 488 | // so this will fail. |
| 489 | ARROW_ASSIGN_OR_RAISE(auto unifier, |
| 490 | DictionaryUnifier::Make(dict_type.value_type(), this->pool)); |
| 491 | // Unify all dictionary array chunks |
| 492 | BufferVector transpose_maps(chunks->size()); |
| 493 | for (size_t j = 0; j < chunks->size(); ++j) { |
| 494 | DCHECK_NE((*chunks)[j]->dictionary, nullptr); |
| 495 | RETURN_NOT_OK( |
| 496 | unifier->Unify(*MakeArray((*chunks)[j]->dictionary), &transpose_maps[j])); |
| 497 | } |
| 498 | std::shared_ptr<Array> dictionary; |
| 499 | RETURN_NOT_OK(unifier->GetResultWithIndexType(dict_type.index_type(), &dictionary)); |
| 500 | for (size_t j = 0; j < chunks->size(); ++j) { |
| 501 | ARROW_ASSIGN_OR_RAISE( |
| 502 | (*chunks)[j], |
| 503 | TransposeDictIndices( |
| 504 | (*chunks)[j], type, type, dictionary->data(), |
| 505 | reinterpret_cast<const int32_t*>(transpose_maps[j]->data()), this->pool)); |
| 506 | if (ext_type) { |
| 507 | (*chunks)[j]->type = ext_type; |
| 508 | } |
| 509 | } |
| 510 | changed = true; |