| 1263 | // ----------------------------------------------------------------------------------- |
| 1264 | |
| 1265 | void layer_normalize ( |
| 1266 | const double eps, |
| 1267 | resizable_tensor& dest, |
| 1268 | resizable_tensor& means, |
| 1269 | resizable_tensor& invstds, |
| 1270 | const tensor& src, |
| 1271 | const tensor& gamma, |
| 1272 | const tensor& beta |
| 1273 | ) |
| 1274 | { |
| 1275 | DLIB_CASSERT( |
| 1276 | have_same_dimensions(gamma, beta) && |
| 1277 | gamma.k() == src.k() && |
| 1278 | gamma.nr() == 1 && |
| 1279 | gamma.nc() == 1 && |
| 1280 | eps > 0, |
| 1281 | "\nsrc.k(): " << src.k() << |
| 1282 | "\ngamma.k(): " << gamma.k() << |
| 1283 | "\ngamma.nr(): " << gamma.nr() << |
| 1284 | "\ngamma.nc(): " << gamma.nc() << |
| 1285 | "\nbeta.k(): " << beta.k() << |
| 1286 | "\nbeta.nr(): " << beta.nr() << |
| 1287 | "\nbeta.nc(): " << beta.nc() << |
| 1288 | "\neps: " << eps |
| 1289 | ); |
| 1290 | |
| 1291 | dest.copy_size(src); |
| 1292 | means.set_size(src.num_samples()); |
| 1293 | invstds.set_size(src.num_samples()); |
| 1294 | |
| 1295 | // first compute means and invstds |
| 1296 | means = 0; |
| 1297 | invstds = 0; |
| 1298 | const float* p_src = src.host(); |
| 1299 | float* p_invstds = invstds.host(); |
| 1300 | float* p_means = means.host(); |
| 1301 | const long num = src.nr() * src.nc(); |
| 1302 | // compute means, and sum of squares |
| 1303 | for (long n = 0; n < src.num_samples(); ++n) |
| 1304 | { |
| 1305 | for (long k = 0; k < src.k(); ++k) |
| 1306 | { |
| 1307 | for (long i = 0; i < num; ++i) |
| 1308 | { |
| 1309 | p_means[n] += *p_src; |
| 1310 | p_invstds[n] += (*p_src) * (*p_src); |
| 1311 | ++p_src; |
| 1312 | } |
| 1313 | } |
| 1314 | } |
| 1315 | means /= src.k() * num; |
| 1316 | invstds /= src.k () * num; |
| 1317 | // copy data back to host |
| 1318 | invstds.host(); |
| 1319 | means.host(); |
| 1320 | |
| 1321 | // compute variances |
| 1322 | for (long n = 0; n < src.num_samples(); ++n) |