| 59 | int64_t count() const { return moments.count; } |
| 60 | |
| 61 | void Consume(const ArraySpan& array) { |
| 62 | constexpr bool kCanUseIntArithmetic = std::is_integral_v<CType> && sizeof(CType) <= 4; |
| 63 | |
| 64 | this->all_valid = array.GetNullCount() == 0; |
| 65 | int64_t valid_count = array.length - array.GetNullCount(); |
| 66 | if (valid_count == 0 || (!this->all_valid && !this->skip_nulls)) { |
| 67 | return; |
| 68 | } |
| 69 | |
| 70 | if constexpr (kCanUseIntArithmetic) { |
| 71 | if (level == 2) { |
| 72 | // int32/16/8: textbook one pass algorithm for M2 with integer arithmetic |
| 73 | |
| 74 | // max number of elements that sum will not overflow int64 (2Gi int32 elements) |
| 75 | // for uint32: 0 <= sum < 2^63 (int64 >= 0) |
| 76 | // for int32: -2^62 <= sum < 2^62 |
| 77 | constexpr int64_t kMaxChunkLength = 1ULL << (63 - sizeof(CType) * 8); |
| 78 | int64_t start_index = 0; |
| 79 | |
| 80 | ArraySpan slice = array; |
| 81 | while (valid_count > 0) { |
| 82 | // process in chunks that overflow will never happen |
| 83 | slice.SetSlice(start_index + array.offset, |
| 84 | std::min(kMaxChunkLength, array.length - start_index)); |
| 85 | const int64_t count = slice.length - slice.GetNullCount(); |
| 86 | start_index += slice.length; |
| 87 | valid_count -= count; |
| 88 | |
| 89 | if (count > 0) { |
| 90 | IntegerVarStd var_std; |
| 91 | const CType* values = slice.GetValues<CType>(1); |
| 92 | VisitSetBitRunsVoid(slice.buffers[0].data, slice.offset, slice.length, |
| 93 | [&](int64_t pos, int64_t len) { |
| 94 | for (int64_t i = 0; i < len; ++i) { |
| 95 | const auto value = values[pos + i]; |
| 96 | var_std.ConsumeOne(value); |
| 97 | } |
| 98 | }); |
| 99 | |
| 100 | // merge variance |
| 101 | auto slice_moments = Moments(var_std.count, var_std.mean(), var_std.m2()); |
| 102 | this->moments.MergeFrom(level, slice_moments); |
| 103 | } |
| 104 | } |
| 105 | return; |
| 106 | } |
| 107 | } |
| 108 | |
| 109 | // float/double/int64/decimal: calculate each moment in a separate pass. |
| 110 | // https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Two-pass_algorithm |
| 111 | SumType sum = internal::SumArray<CType, SumType, SimdLevel::NONE>(array); |
| 112 | |
| 113 | const double mean = ToDouble(sum) / valid_count; |
| 114 | const double m2 = internal::SumArray<CType, double, SimdLevel::NONE>( |
| 115 | array, [this, mean](CType value) { |
| 116 | const double v = ToDouble(value); |
| 117 | return (v - mean) * (v - mean); |
| 118 | }); |
no test coverage detected