| 15 | return [clip_norm(g, c, norm) for g in gs] |
| 16 | |
| 17 | class Regularizer(object): |
| 18 | |
| 19 | def __init__(self, l1=0., l2=0., maxnorm=0., l2norm=False, frobnorm=False): |
| 20 | self.__dict__.update(locals()) |
| 21 | |
| 22 | def max_norm(self, p, maxnorm): |
| 23 | if maxnorm > 0: |
| 24 | norms = T.sqrt(T.sum(T.sqr(p), axis=0)) |
| 25 | desired = T.clip(norms, 0, maxnorm) |
| 26 | p = p * (desired/ (1e-7 + norms)) |
| 27 | return p |
| 28 | |
| 29 | def l2_norm(self, p): |
| 30 | return p/l2norm(p, axis=0) |
| 31 | |
| 32 | def frob_norm(self, p, nrows): |
| 33 | return (p/T.sqrt(T.sum(T.sqr(p))))*T.sqrt(nrows) |
| 34 | |
| 35 | def gradient_regularize(self, p, g): |
| 36 | g += p * self.l2 |
| 37 | g += T.sgn(p) * self.l1 |
| 38 | return g |
| 39 | |
| 40 | def weight_regularize(self, p): |
| 41 | p = self.max_norm(p, self.maxnorm) |
| 42 | if self.l2norm: |
| 43 | p = self.l2_norm(p) |
| 44 | if self.frobnorm > 0: |
| 45 | p = self.frob_norm(p, self.frobnorm) |
| 46 | return p |
| 47 | |
| 48 | |
| 49 | class Update(object): |