| 132 | } |
| 133 | |
| 134 | void HdfsTableSink::BuildPartitionDescMap() { |
| 135 | for (const HdfsTableDescriptor::PartitionIdToDescriptorMap::value_type& id_to_desc: |
| 136 | table_desc_->partition_descriptors()) { |
| 137 | // Build a map whose key is computed from the value of dynamic partition keys for a |
| 138 | // particular partition, and whose value is the descriptor for that partition. |
| 139 | |
| 140 | // True if this partition might be written to, false otherwise. |
| 141 | // A partition may be written to iff: |
| 142 | // For all partition key exprs e, either: |
| 143 | // 1. e is not constant |
| 144 | // 2. The value supplied by the query for this partition key is equal to e's |
| 145 | // constant value. |
| 146 | // Only relevant partitions are remembered in partition_descriptor_map_. |
| 147 | bool relevant_partition = true; |
| 148 | HdfsPartitionDescriptor* partition = id_to_desc.second; |
| 149 | DCHECK_EQ(partition->partition_key_value_evals().size(), |
| 150 | partition_key_expr_evals_.size()); |
| 151 | vector<ScalarExprEvaluator*> dynamic_partition_key_value_evals; |
| 152 | for (size_t i = 0; i < partition_key_expr_evals_.size(); ++i) { |
| 153 | // Remember non-constant partition key exprs for building hash table of Hdfs files |
| 154 | DCHECK(&partition_key_expr_evals_[i]->root() == partition_key_exprs_[i]); |
| 155 | if (!partition_key_exprs_[i]->is_constant()) { |
| 156 | dynamic_partition_key_value_evals.push_back( |
| 157 | partition->partition_key_value_evals()[i]); |
| 158 | } else { |
| 159 | // Deal with the following: one partition has (year=2009, month=3); another has |
| 160 | // (year=2010, month=3). |
| 161 | // A query like: INSERT INTO TABLE... PARTITION(year=2009) SELECT month FROM... |
| 162 | // would lead to both partitions having the same key modulo ignored constant |
| 163 | // partition keys. So only keep a reference to the partition which matches |
| 164 | // partition_key_values for constant values, since only that is written to. |
| 165 | void* table_partition_key_value = |
| 166 | partition->partition_key_value_evals()[i]->GetValue(nullptr); |
| 167 | void* target_partition_key_value = |
| 168 | partition_key_expr_evals_[i]->GetValue(nullptr); |
| 169 | if (table_partition_key_value == nullptr |
| 170 | && target_partition_key_value == nullptr) { |
| 171 | continue; |
| 172 | } |
| 173 | if (table_partition_key_value == nullptr |
| 174 | || target_partition_key_value == nullptr |
| 175 | || !RawValue::Eq(table_partition_key_value, target_partition_key_value, |
| 176 | partition_key_expr_evals_[i]->root().type())) { |
| 177 | relevant_partition = false; |
| 178 | break; |
| 179 | } |
| 180 | } |
| 181 | } |
| 182 | if (relevant_partition) { |
| 183 | string key; |
| 184 | // Pass nullptr as row, since all of these expressions are constant, and can |
| 185 | // therefore be evaluated without a valid row context. |
| 186 | GetHashTblKey(nullptr, dynamic_partition_key_value_evals, &key); |
| 187 | DCHECK(partition_descriptor_map_.find(key) == partition_descriptor_map_.end()) |
| 188 | << "Partitions with duplicate 'static' keys found during INSERT"; |
| 189 | partition_descriptor_map_[key] = partition; |
| 190 | } |
| 191 | } |