Basic block for channelwise fully-connected neural networks. Applies a dense linearity channelwise and a nonlinear activation function.
| 132 | return y |
| 133 | |
| 134 | class ChannelwiseFullyConnectedModule(Module): |
| 135 | """Basic block for channelwise fully-connected neural networks. |
| 136 | |
| 137 | Applies a dense linearity channelwise and a nonlinear activation function. |
| 138 | |
| 139 | """ |
| 140 | |
| 141 | global_count = 0 |
| 142 | |
| 143 | def __init__(self, |
| 144 | size, |
| 145 | bias=False, |
| 146 | weights=[], |
| 147 | activation=None, |
| 148 | transpose=False, |
| 149 | name=None, |
| 150 | parallel_strategy={}): |
| 151 | """Initalize channelwise fully connected module |
| 152 | |
| 153 | Args: |
| 154 | size (int or list): Dimension of the output tensor |
| 155 | bias (bool): Whether to apply bias after linearity. |
| 156 | transpose (bool): Whether to apply transpose of weights |
| 157 | matrix. |
| 158 | weights (`Weights` or iterator of `Weights`): Weights in |
| 159 | fully-connected layer. There are at most two: the |
| 160 | matrix and the bias. If weights are not provided, the |
| 161 | matrix will be initialized with He normal |
| 162 | initialization and the bias with zeros. |
| 163 | activation (type): Layer class for activation function. |
| 164 | name (str): Default name is in the form 'channelwisefc<index>'. |
| 165 | parallel_strategy (dict): Data partitioning scheme. |
| 166 | """ |
| 167 | super().__init__() |
| 168 | ChannelwiseFullyConnectedModule.global_count += 1 |
| 169 | self.instance = 0 |
| 170 | self.size = size |
| 171 | self.bias = bias |
| 172 | self.transpose = transpose |
| 173 | self.parallel_strategy = parallel_strategy |
| 174 | self.name = (name |
| 175 | if name |
| 176 | else 'channelwisefc{0}'.format(ChannelwiseFullyConnectedModule.global_count)) |
| 177 | self.data_layout = 'data_parallel' |
| 178 | |
| 179 | self.weights = list(make_iterable(weights)) |
| 180 | if len(self.weights) > 2: |
| 181 | raise ValueError('`FullyConnectedModule` has ' |
| 182 | 'at most two weights, ' |
| 183 | 'but got {0}'.format(len(self.weights))) |
| 184 | if len(self.weights) == 0: |
| 185 | self.weights.append( |
| 186 | lbann.Weights(initializer=lbann.HeNormalInitializer(), |
| 187 | name=self.name+'_matrix')) |
| 188 | if self.bias and len(self.weights) == 1: |
| 189 | self.weights.append( |
| 190 | lbann.Weights(initializer=lbann.ConstantInitializer(value=0.0), |
| 191 | name=self.name+'_bias')) |