| 2201 | |
| 2202 | template <typename T> |
| 2203 | void appendNonNull( |
| 2204 | VectorStream* stream, |
| 2205 | const uint64_t* nulls, |
| 2206 | folly::Range<const vector_size_t*> rows, |
| 2207 | const T* values, |
| 2208 | Scratch& scratch) { |
| 2209 | auto numRows = rows.size(); |
| 2210 | ScratchPtr<int32_t, 64> nonNullHolder(scratch); |
| 2211 | const int32_t* nonNullIndices; |
| 2212 | int32_t numNonNull; |
| 2213 | if (LIKELY(numRows <= 8)) { |
| 2214 | // Short batches need extra optimization. The set bits are prematerialized. |
| 2215 | uint8_t nullsByte = *reinterpret_cast<const uint8_t*>(nulls); |
| 2216 | numNonNull = __builtin_popcount(nullsByte); |
| 2217 | nonNullIndices = |
| 2218 | numNonNull == numRows ? nullptr : simd::byteSetBits(nullsByte); |
| 2219 | } else { |
| 2220 | auto mutableIndices = nonNullHolder.get(numRows); |
| 2221 | // Convert null flags to indices. This is much faster than checking bits one |
| 2222 | // by one, several bits per clock specially if mostly null or non-null. Even |
| 2223 | // worst case of half nulls is more than one row per clock. |
| 2224 | numNonNull = simd::indicesOfSetBits(nulls, 0, numRows, mutableIndices); |
| 2225 | nonNullIndices = numNonNull == numRows ? nullptr : mutableIndices; |
| 2226 | } |
| 2227 | stream->appendNulls(nulls, 0, rows.size(), numNonNull); |
| 2228 | ByteOutputStream& out = stream->values(); |
| 2229 | |
| 2230 | if constexpr (sizeof(T) == 8) { |
| 2231 | AppendWindow<int64_t> window(out, scratch); |
| 2232 | int64_t* output = window.get(numNonNull); |
| 2233 | copyWordsWithRows( |
| 2234 | output, |
| 2235 | rows.data(), |
| 2236 | nonNullIndices, |
| 2237 | numNonNull, |
| 2238 | reinterpret_cast<const int64_t*>(values)); |
| 2239 | } else if constexpr (sizeof(T) == 4) { |
| 2240 | AppendWindow<int32_t> window(out, scratch); |
| 2241 | int32_t* output = window.get(numNonNull); |
| 2242 | copyWordsWithRows( |
| 2243 | output, |
| 2244 | rows.data(), |
| 2245 | nonNullIndices, |
| 2246 | numNonNull, |
| 2247 | reinterpret_cast<const int32_t*>(values)); |
| 2248 | } else { |
| 2249 | AppendWindow<T> window(out, scratch); |
| 2250 | T* output = window.get(numNonNull); |
| 2251 | copyWordsWithRows( |
| 2252 | output, |
| 2253 | rows.data(), |
| 2254 | nonNullIndices, |
| 2255 | numNonNull, |
| 2256 | values, |
| 2257 | stream->isLongDecimal()); |
| 2258 | } |
| 2259 | } |
| 2260 |
no test coverage detected