more information added

README.md

@@ -6,8 +6,14 @@ These scrips are updated ones from the **neuralnetworksanddeeplearning.com** git
 
 The testing file (**test.py**) contains all three networks (network.py, network2.py, network3.py) from the book and it is the starting point to run (i.e. *train and evaluate*) them.
 
-## Just type at shell: **python3.5 test.py**
+## Google Colab
+The best way to run the code is to use the colab notebook provided in bellow link.
 
-In test.py there are examples of networks configurations with proper comments. I did that to relate with particular chapters from the book.
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1OAzGo1E5uKuKw1R2Cbce2KKxpvHPaLyY?usp=sharing)
 
+## Scripts
+To run the scripts, go to the chapter_codes folder, open the terminal, and type:
+```bash
+python chapter1-data.py
+```
 

chapter_codes/chapter1-data.py

@@ -0,0 +1,17 @@
+import sys
+sys.path.append('..')
+
+from elements import mnist_loader
+import numpy as np
+import matplotlib.pyplot as plt
+
+training_data, validation_data, test_data = mnist_loader.load_data_wrapper('../mnist.pkl.gz')
+training_data = list(training_data)
+test_data = list(test_data)
+
+plt.figure(figsize=(15,5))
+for i in range(1,17):
+    plt.subplot(2,8,i)
+    plt.imshow(training_data[i][0].reshape(28,28), cmap='Greys')
+    plt.xlabel(f'label: {np.argmax(training_data[i][1])}')
+plt.show()

chapter_codes/chapter1-network.py

@@ -0,0 +1,12 @@
+import sys
+sys.path.append('..')
+
+from elements import network
+from elements import mnist_loader
+
+training_data, validation_data, test_data = mnist_loader.load_data_wrapper('../mnist.pkl.gz')
+training_data = list(training_data)
+test_data = list(test_data)
+
+net = network.Network([784, 30, 10])
+net.SGD(training_data, 30, 10, 3.0, test_data=test_data)

chapter_codes/chapter2-network2.py

@@ -0,0 +1,14 @@
+import sys
+sys.path.append('..')
+
+from elements import network2
+from elements import mnist_loader
+
+training_data, validation_data, test_data = mnist_loader.load_data_wrapper('../mnist.pkl.gz')
+training_data = list(training_data)
+test_data = list(test_data)
+
+net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)
+#net.large_weight_initializer()
+evaluation_cost, evaluation_accuracy, training_cost, training_accuracy = \
+    net.SGD(training_data, 30, 10, 0.1, lmbda = 5.0, evaluation_data=validation_data, monitor_evaluation_accuracy=True)

chapter_codes/chapter3-earlystopping.py

@@ -0,0 +1,26 @@
+import sys
+sys.path.append('..')
+
+from elements import network2
+from elements import mnist_loader
+
+training_data, validation_data, test_data = mnist_loader.load_data_wrapper('../mnist.pkl.gz')
+training_data = list(training_data)
+test_data = list(test_data)
+
+net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)
+evaluation_cost, evaluation_accuracy, training_cost, training_accuracy = \
+    net.SGD(training_data[:1000], 30, 10, 0.5,
+    lmbda=5.0,
+    evaluation_data=validation_data,
+    monitor_evaluation_accuracy=True,
+    monitor_training_cost=True,
+    early_stopping_n=10)
+
+
+plt.figure(figsize=(6,5))
+plt.plot(np.array(evaluation_accuracy)/10000)
+plt.grid()
+plt.xlabel('Epoch')
+plt.legend(['Accuracy on the test data'])
+plt.show()

chapter_codes/chapter3-overfitting.py

@@ -0,0 +1,56 @@
+import sys
+sys.path.append('..')
+
+from elements import network2
+from elements import mnist_loader
+import matplotlib.pyplot as plt
+
+training_data, validation_data, test_data = mnist_loader.load_data_wrapper('../mnist.pkl.gz')
+training_data = list(training_data)
+test_data = list(test_data)
+
+net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)
+net.large_weight_initializer()
+evaluation_cost, evaluation_accuracy, training_cost, training_accuracy = \
+    net.SGD(training_data[:1000], 400, 10, 0.5, evaluation_data=test_data,
+    monitor_evaluation_cost=True,
+    monitor_evaluation_accuracy=True,
+    monitor_training_cost=True,
+    monitor_training_accuracy=True)
+
+
+plt.figure(figsize=(6,5))
+plt.plot(training_cost[200:])
+plt.grid()
+plt.title('Cost on the training data')
+plt.xlabel('Epoch')
+plt.show()
+
+plt.figure(figsize=(6,5))
+plt.plot(evaluation_accuracy[200:])
+plt.grid()
+plt.title('Accuracy (%) on the test data')
+plt.xlabel('Epoch')
+plt.show()
+
+plt.figure(figsize=(6,5))
+plt.plot(evaluation_cost)
+plt.grid()
+plt.title('Cost on the test data')
+plt.xlabel('Epoch')
+plt.show()
+
+plt.figure(figsize=(6,5))
+plt.plot(training_accuracy)
+plt.grid()
+plt.title('Accuracy (%) on the training data')
+plt.xlabel('Epoch')
+plt.show()
+
+plt.figure(figsize=(6,5))
+plt.plot(np.array(training_accuracy[:30])/1000)
+plt.plot(np.array(evaluation_accuracy[:30])/10000)
+plt.grid()
+plt.xlabel('Epoch')
+plt.legend(['Accuracy on the training data','Accuracy on the test data'])
+plt.show()

chapter_codes/chapter3-regularization.py

@@ -0,0 +1,44 @@
+import sys
+sys.path.append('..')
+
+from elements import network2
+from elements import mnist_loader
+import matplotlib.pyplot as plt
+
+training_data, validation_data, test_data = mnist_loader.load_data_wrapper('../mnist.pkl.gz')
+training_data = list(training_data)
+test_data = list(test_data)
+
+net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)
+net.large_weight_initializer()
+evaluation_cost, evaluation_accuracy, training_cost, training_accuracy = \
+    net.SGD(training_data[:1000], 400, 10, 0.5,
+    evaluation_data=test_data,
+    lmbda = 0.1, # this is a regularization parameter
+    monitor_evaluation_cost=True,
+    monitor_evaluation_accuracy=True,
+    monitor_training_cost=True,
+    monitor_training_accuracy=True)
+
+
+plt.figure(figsize=(6,5))
+plt.plot(np.array(training_cost[200:])/1000)
+plt.grid()
+plt.title('Cost on the training data')
+plt.xlabel('Epoch')
+plt.show()
+
+plt.figure(figsize=(6,5))
+plt.plot(np.array(evaluation_accuracy[200:])/1000)
+plt.grid()
+plt.title('Accuracy (%) on the test data')
+plt.xlabel('Epoch')
+plt.show()
+
+plt.figure(figsize=(6,5))
+plt.plot(np.array(training_accuracy[:30])/1000)
+plt.plot(np.array(evaluation_accuracy[:30])/10000)
+plt.grid()
+plt.xlabel('Epoch')
+plt.legend(['Accuracy on the training data','Accuracy on the test data'])
+plt.show()

chapter_codes/chapter5-vanishing.py

@@ -0,0 +1,16 @@
+import sys
+sys.path.append('..')
+
+from elements import network2
+from elements import mnist_loader
+
+training_data, validation_data, test_data = mnist_loader.load_data_wrapper('../mnist.pkl.gz')
+training_data = list(training_data)
+test_data = list(test_data)
+
+net = network2.Network([784, 30, 30, 30, 30, 10], cost=network2.CrossEntropyCost)
+evaluation_cost, evaluation_accuracy, training_cost, training_accuracy = \
+    net.SGD(training_data, 30, 10, 0.1,
+    lmbda=5.0,
+    evaluation_data=validation_data,
+    monitor_evaluation_accuracy=True)

chapter_codes/chapter6-1conv.py

@@ -0,0 +1,21 @@
+import sys
+sys.path.append('..')
+
+from elements import network3
+from elements.network3 import Network
+from elements.network3 import ConvPoolLayer, FullyConnectedLayer, SoftmaxLayer
+
+# read data:
+training_data, validation_data, test_data = network3.load_data_shared('../mnist.pkl.gz')
+
+# mini-batch size:
+mini_batch_size = 10
+
+net = Network([
+    ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
+                  filter_shape=(20, 1, 5, 5),
+                  poolsize=(2, 2)),
+    FullyConnectedLayer(n_in=20*12*12, n_out=100),
+    SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)
+
+net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data)

chapter_codes/chapter6-2conv.py

@@ -0,0 +1,27 @@
+import sys
+sys.path.append('..')
+
+import warnings
+warnings.filterwarnings('ignore') 
+
+from elements import network3
+from elements.network3 import Network
+from elements.network3 import ConvPoolLayer, FullyConnectedLayer, SoftmaxLayer
+
+# read data:
+training_data, validation_data, test_data = network3.load_data_shared('../mnist.pkl.gz')
+
+# mini-batch size:
+mini_batch_size = 10
+
+net = Network([
+    ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
+                  filter_shape=(20, 1, 5, 5),
+                  poolsize=(2, 2)),
+    ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
+                  filter_shape=(40, 20, 5, 5),
+                  poolsize=(2, 2)),
+    FullyConnectedLayer(n_in=40*4*4, n_out=100),
+    SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)
+
+net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data)

chapter_codes/chapter6-relu.py

@@ -0,0 +1,29 @@
+import sys
+sys.path.append('..')
+
+import warnings
+warnings.filterwarnings('ignore')
+
+from elements import network3
+from elements.network3 import Network
+from elements.network3 import ConvPoolLayer, FullyConnectedLayer, SoftmaxLayer, ReLU
+
+# read data:
+training_data, validation_data, test_data = network3.load_data_shared('../mnist.pkl.gz')
+
+# mini-batch size:
+mini_batch_size = 10
+
+net = Network([
+    ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
+                  filter_shape=(20, 1, 5, 5),
+                  poolsize=(2, 2),
+                  activation_fn=ReLU),
+    ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
+                  filter_shape=(40, 20, 5, 5),
+                  poolsize=(2, 2),
+                  activation_fn=ReLU),
+    FullyConnectedLayer(n_in=40*4*4, n_out=100, activation_fn=ReLU),
+    SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)
+
+net.SGD(training_data, 60, mini_batch_size, 0.03, validation_data, test_data, lmbda=0.1)

chapter_codes/chapter6-shallow.py

@@ -0,0 +1,18 @@
+import sys
+sys.path.append('..')
+
+from elements import network3
+from elements.network3 import Network
+from elements.network3 import ConvPoolLayer, FullyConnectedLayer, SoftmaxLayer
+
+# read data:
+training_data, validation_data, test_data = network3.load_data_shared('../mnist.pkl.gz')
+
+# mini-batch size:
+mini_batch_size = 10
+
+net = Network([
+    FullyConnectedLayer(n_in=784, n_out=100),
+    SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)
+
+net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data)