droupout, AveragePool, Bug fix

amsukdu · amsukdu · commit fbbad68136ef · 2017-10-14T12:47:26.000+09:00
diff --git a/README.md b/README.md
@@ -10,6 +10,8 @@ This is Convolutional Neural Network only in python & numpy. It is simple and sl
 
 **Regulization :** Droupout(only on fc), L2
 
+**Pooling :** Max, Average
+
 **Loss Function :** Softmax, Logistic
 
 ## Prerequisites
@@ -22,62 +24,54 @@ AND gate and CIFAR-10 examples are included.
 
 ```python
 
-from classes.neural_net import NeuralNetwork
-
 lr = 1e-4
-l2_reg = 5e-6
-
-# 32 * 32 color image
-input_size = (3,32,32)
+l2_reg = 8e-6
 
-cnn = NeuralNetwork(input_size,
+cnn = NeuralNetwork(train_images.shape[1:],
                     [
                         {'type': 'conv', 'k': 16, 'u_type': 'nag', 'f': 5, 's': 1, 'p': 2},
-                        {'type': 'pool'},
+                        {'type': 'pool', 'method': 'average'},
                         {'type': 'conv', 'k': 20, 'u_type': 'nag', 'f': 5, 's': 1, 'p': 2},
-                        {'type': 'pool'},
+                        {'type': 'pool', 'method': 'average'},
                         {'type': 'conv', 'k': 20, 'u_type': 'nag', 'f': 5, 's': 1, 'p': 2},
-                        {'type': 'pool'},
+                        {'type': 'pool', 'method': 'average'},
                         {'type': 'output', 'k': len(le.classes_), 'u_type': 'adam'}
                     ]
                     , lr, l2_reg=l2_reg)
 
-
-for i in range(600000):
-    loss, acc = cnn.epoch(train_images, train_labels)
-    test_loss, test_acc = cnn.predict(train_images, train_labels)
-
 ```
 
 CIFAR-10 example gets ~72% test accuracy in 20 epoch.
 
 
 ## API Reference
 ```python
-classes.NeuralNetwork(input_shape, layer_list, lr, l2_reg=0, dropout_p=1, loss='softmax'):
+classes.NeuralNetwork(self, input_shape, layer_list, lr, l2_reg=0, loss='softmax'):
 ```
-
+<br />
 
 | Parameter | Description |
 | --- | --- |
 | input_shape | Data's numpy shape.  |
 | layer_list | List of layers you want to be networked. All of properties goes to **kwargs. |
 | lr | Learning rate. |
-| l2_reg | L2 regularization lambda. |
-| drouput_p | Dropout rate. 0 < dropout_p <= 1
+| l2_reg | L2 regularization|
 | loss | Loss function. 'softmax', 'logistic' |
 
 
 ```python
 # type fc, output
-classes.NeuralLayer(input_size, k, u_type='adam', a_type='relu')
+classes.NeuralLayer(input_size, k, f=3, s=1, p=1, u_type='adam', a_type='relu', dropout=1)
 
 # type pool
-classes.PoolLayer(input_size, f=2, s=2)
+classes.PoolLayer(input_size, f=2, s=2, method='max', dropout=1):
 
 # type conv
-classes.ConvLayer(input_size, k, f=3, s=1, p=1, u_type='adam', a_type='relu')
+classes.ConvLayer(input_size, k, f=3, s=1, p=1, u_type='adam', a_type='relu', dropout=1)
 ```
+<br />
+
+
 
 | Update Policy | u_type|
 | --- | --- |
@@ -92,6 +86,10 @@ classes.ConvLayer(input_size, k, f=3, s=1, p=1, u_type='adam', a_type='relu')
 | ReLU | 'relu' |
 | Sigmoid | 'sigmoid' |
 
+| Pooling |method|
+| --- | --- |
+| Max |'max' |
+|Avverage |'average'|
 
 ## License
 MIT
diff --git a/classes/conv_layer.py b/classes/conv_layer.py
@@ -5,7 +5,7 @@
 
 class ConvLayer(NeuralLayer):
 
-    def __init__(self, input_size, k, f=3, s=1, p=1, u_type='adam', a_type='relu'):
+    def __init__(self, input_size, k, f=3, s=1, p=1, u_type='adam', a_type='relu', dropout=1):
         self.image_size = 0
         self.w = input_size[2]
         self.h = input_size[1]
@@ -20,7 +20,7 @@ def __init__(self, input_size, k, f=3, s=1, p=1, u_type='adam', a_type='relu'):
         self.h2 = int((self.h - self.f + 2 * self.p) / self.s + 1)
         self.d2 = k
 
-        super(ConvLayer, self).__init__(f*f*self.d, k, u_type=u_type, a_type=a_type)
+        super(ConvLayer, self).__init__(f*f*self.d, k, u_type=u_type, a_type=a_type, dropout=dropout)
 
     def predict(self, batch):
         self.image_size = batch.shape[0]
@@ -55,7 +55,8 @@ def forward(self, batch):
             l2 += n.regularization()
 
         sum_weights = np.array(sum_weights)
-        strength = (sum_weights.dot(cols) + np.array(bias).reshape(sum_weights.shape[0], 1)).reshape(self.k, self.h2, self.w2, -1).transpose(3, 0, 1, 2)
+        strength = (sum_weights.dot(cols) + np.array(bias).reshape(sum_weights.shape[0], 1))
+        strength = strength.reshape(self.k, self.h2, self.w2, -1).transpose(3, 0, 1, 2)
 
         if self.activation:
             if self.a_type == 'sigmoid':
@@ -71,23 +72,32 @@ def backward(self, d, need_d=True):
         if d.ndim < 4:
             d = d.reshape(self.w2, self.h2, self.k, -1).T
 
-        delta = d * u.relu_d(self.forward_result)
-        padding = ((self.w - 1) * self.s + self.f - self.w2) // 2
-        cols = u.im2col_indices(delta, self.f, self.f, padding=padding, stride=self.s)
+        if self.activation:
+            if self.a_type == 'sigmoid':
+                delta = d * u.sigmoid_d(self.forward_result)
+            else:
+                delta = d * u.relu_d(self.forward_result)
+        else:
+            delta = d
+
         sum_weights = []
         for index, n in enumerate(self.neurons):
-            n.delta = delta[:, index, :, :].flatten()
+            n.delta = delta[:, index, :, :].transpose(1, 2, 0).flatten()
             if need_d:
-                rot = np.rot90(n.weights.reshape(self.d, self.f * self.f), 2).reshape(self.d, self.f, self.f)[::-1]
+                rot = np.rot90(n.weights.reshape(self.d, self.f, self.f), k=2, axes=(1, 2))
                 sum_weights.append(rot)
 
         if not need_d:
             return
 
-        sum_weights = np.array(sum_weights).transpose(1,0,2,3).reshape(self.d, -1)
+        padding = ((self.w - 1) * self.s + self.f - self.w2) // 2
+        cols = u.im2col_indices(delta, self.f, self.f, padding=padding, stride=self.s)
+
+        sum_weights = np.array(sum_weights).transpose(1, 0, 2, 3).reshape(self.d, -1)
 
         result = sum_weights.dot(cols)
         im = result.reshape(self.d, self.h, self.w, -1).transpose(3, 0, 1, 2)
+
         return im
 
     def output_size(self):
diff --git a/classes/layer.py b/classes/layer.py
@@ -7,6 +7,7 @@ class Layer(object):
     is_output = False
     activation = True
     u_type = 'adam'
+    dropout = 1
 
     @abstractmethod
     def forward(self, batch):
diff --git a/classes/neural_layer.py b/classes/neural_layer.py
@@ -6,10 +6,11 @@
 
 class NeuralLayer(Layer):
 
-    def __init__(self, input_size, k, u_type='adam', a_type='relu'):
+    def __init__(self, input_size, k, u_type='adam', a_type='relu', dropout=1):
         self.neurons = []
         self.forward_result = None
         self.k = k
+        self.dropout = dropout
 
         self.u_type = u_type
         self.a_type= a_type
@@ -19,14 +20,16 @@ def __init__(self, input_size, k, u_type='adam', a_type='relu'):
 
         for n in range(k):
             n = Neuron(input_size)
+
             if u_type == 'adam':
-                n.m_bias, n.v_bias, n.m, n.v = 0, 0, 0, 0
+                n.m, n.v = 0, 0
             elif u_type == 'm':
-                n.v_bias, n.v = 0, 0
+                n.v = 0, 0
             elif u_type == 'nag':
-                n.v, n.v_bias, n.v_prev, n.v_prev_bias = 0, 0, 0, 0
+                n.v, n.v_prev = 0, 0
             elif u_type == 'rmsprop':
-                n.cache, n.cache_bias, n.v, n.v = 0, 0, 0, 0
+                n.cache, n.v = 0, 0
+
             self.neurons.append(n)
 
     def predict(self, batch):
@@ -51,7 +54,6 @@ def forward(self, batch):
 
         if batch.ndim > 2:
             batch = batch.reshape(batch.shape[0], -1).T
-
         forward_result = []
         l2 = 0
         for n in self.neurons:
diff --git a/classes/neural_net.py b/classes/neural_net.py
@@ -5,13 +5,13 @@
 import numpy as np
 
 class NeuralNetwork(object):
-    def __init__(self, input_shape, layer_list, lr, l2_reg=0, dropout_p=1, loss='softmax'):
+    def __init__(self, input_shape, layer_list, lr, l2_reg=0, loss='softmax'):
         self.layers = []
         self.lr = np.float32(lr)
         self.l2_reg = np.float32(l2_reg)
         self.loss = loss
+        self.epoch_count = 0
 
-        self.dropout_p = dropout_p
         self.dropout_masks = []
         self.t = 0
 
@@ -48,9 +48,6 @@ def predict(self, batch, label):
         for index, layer in enumerate(self.layers):
             next_input = layer.predict(next_input)
 
-            if (self.dropout_p < 1) and (type(layer).__name__ == 'NeuralLayer') and not layer.is_output:
-                next_input *= self.dropout_p
-
         result = np.array(next_input)
         if self.loss == 'softmax':
             loss, delta = utils.softmax_loss(result, label)
@@ -64,39 +61,44 @@ def predict(self, batch, label):
 
     def epoch(self, batch, label):
         # forward
-        self.dropout_masks = []
         l2 = 0
         next_input = batch
         for index, layer in enumerate(self.layers):
             layer_result = layer.forward(next_input)
             next_input = layer_result[0]
             l2 += layer_result[1]
-            if (self.dropout_p < 1) and (type(layer).__name__ == 'NeuralLayer') and not layer.is_output:
-                dropout_mask = np.random.rand(*next_input.shape) < self.dropout_p
+            if layer.dropout < 1 and not layer.is_output:
+                dropout_mask = np.random.rand(*next_input.shape) < layer.dropout
+                next_input *= dropout_mask / layer.dropout
                 self.dropout_masks.append(dropout_mask)
-                next_input *= dropout_mask
+
 
         result = np.array(next_input)
         if self.loss == 'softmax':
             loss, delta = utils.softmax_loss(result, label)
         elif self.loss == 'logistic':
             loss, delta = utils.logistic_loss(result, label)
 
+        loss += 0.5 * self.l2_reg * l2
         max_result = np.argmax(result, axis=0)
         correct_count = np.sum(max_result == label)
 
         # backprop
         back_input = delta.T
         for index, layer in enumerate(reversed(self.layers)):
             is_input_layer = index < len(self.layers) - 1
-            back_input = layer.backward(back_input, is_input_layer)
 
-            if self.dropout_masks:
+            if layer.dropout < 1 and not layer.is_output and self.dropout_masks:
                 dropout_mask = self.dropout_masks.pop()
-                back_input *= dropout_mask
+                if dropout_mask.ndim > 2 and back_input.ndim == 2:
+                    back_input *= dropout_mask.T.reshape(-1, back_input.shape[1])
+                else:
+                    back_input *= dropout_mask
+
+            back_input = layer.backward(back_input, is_input_layer)
 
         # update
         for index, layer in enumerate(self.layers):
             layer.update(self.lr, l2_reg=self.l2_reg, t=self.t)
 
-        return loss + self.l2_reg * l2 / 2, correct_count / float(len(max_result)) * 100
+        return loss + self.l2_reg * l2 / 2, correct_count / float(len(max_result)) * 100
diff --git a/classes/neuron.py b/classes/neuron.py
@@ -3,14 +3,13 @@
 
 class Neuron(object):
     def __init__(self, input_size, bias=0.0):
-        limit = np.sqrt(2.0 / input_size)
-        self.weights = (np.random.randn(input_size) * limit).astype(np.float32)
+        self.weights = (np.random.randn(input_size) * np.sqrt(2.0 / input_size)).astype(np.float32)
         self.b = np.float32(bias)
         self.last_input = None
-        self.delta = 0
+        self.delta = None
 
     def strength(self, values):
         return np.dot(self.weights, values) + self.b
 
     def regularization(self):
-        return np.sum(np.square(self.weights))
+        return np.sum(np.square(self.weights))
diff --git a/classes/pool_layer.py b/classes/pool_layer.py
diff --git a/classes/utils.py b/classes/utils.py
diff --git a/example/cifar-10-batches-py/main.py b/example/cifar-10-batches-py/main.py
