| 127 | } |
| 128 | |
| 129 | static void compute_statistics(std::vector<tensor_statistics> & tstats, const std::string & name, const Stats & e) { |
| 130 | if (e.values.size() % e.counts.size() != 0) { |
| 131 | LOG_ERR("%s: activation size mismatch for tensor %s (%zu vs %zu)\n", __func__, name.c_str(), e.counts.size(), e.values.size()); |
| 132 | return; |
| 133 | } |
| 134 | if (e.counts.empty()) { |
| 135 | LOG_ERR("%s: there are no activations for tensor %s. The imatrix may be suboptimal\n", __func__, name.c_str()); |
| 136 | return; |
| 137 | } |
| 138 | |
| 139 | const int n_mat = e.counts.size(); |
| 140 | const int row_size = e.values.size() / n_mat; |
| 141 | |
| 142 | std::vector<float> activations; |
| 143 | activations.reserve(e.values.size()); |
| 144 | |
| 145 | for (int i = 0; i < n_mat; ++i) { |
| 146 | if (e.counts[i] == 0) { |
| 147 | LOG_DBG("%s: skipping tensor %s due to zero count at index %d\n", __func__, name.c_str(), i); |
| 148 | continue; |
| 149 | } |
| 150 | for (int j = 0; j < row_size; ++j) { |
| 151 | activations.push_back(e.values[i*row_size + j] / e.counts[i]); |
| 152 | } |
| 153 | } |
| 154 | |
| 155 | if (activations.empty()) { |
| 156 | LOG_ERR("%s: all counts are zero for tensor %s, skipping statistics computation\n", __func__, name.c_str()); |
| 157 | return; |
| 158 | } |
| 159 | |
| 160 | const float act_total = std::accumulate(activations.begin(), activations.end(), 0.0f); |
| 161 | const float act_max = *std::max_element(activations.begin(), activations.end()); |
| 162 | const float act_min = *std::min_element(activations.begin(), activations.end()); |
| 163 | const float act_mean = act_total / activations.size(); |
| 164 | const float act_sqr_total = std::inner_product(activations.begin(), activations.end(), activations.begin(), 0.0f); |
| 165 | const float act_var = (act_sqr_total / activations.size()) - (act_mean * act_mean); |
| 166 | const float act_dev = std::sqrt(std::max(0.0f, act_var)); |
| 167 | float threshold = 1e-5f; |
| 168 | const int inactive_count = std::count_if(activations.begin(), activations.end(), |
| 169 | [threshold](const float v) { return fabsf(v) <= threshold; }); |
| 170 | const float active_ratio = 1 - static_cast<float>(inactive_count) / activations.size(); |
| 171 | |
| 172 | float entropy = 0; |
| 173 | if (act_total > 0) { |
| 174 | for (const auto act : activations) { |
| 175 | if (const float p = act / act_total; p > 0) { |
| 176 | entropy -= p * std::log2(p); |
| 177 | } |
| 178 | } |
| 179 | } |
| 180 | |
| 181 | int z_score = 0; |
| 182 | if (act_dev > 0.0f) { |
| 183 | for (const auto act : activations) { |
| 184 | if (const float p = (act - act_mean) / act_dev; p > 1) { |
| 185 | z_score++; |
| 186 | } |