diff --git a/deep_learning4e.py b/deep_learning4e.py index 18c41f54e..87b33546a 100644 --- a/deep_learning4e.py +++ b/deep_learning4e.py @@ -187,7 +187,7 @@ def gradient_descent(dataset, net, loss, epochs=1000, l_rate=0.01, batch_size=1, Gradient descent algorithm to update the learnable parameters of a network. :return: the updated network """ - examples = dataset.examples # init data + examples = dataset.examples # init data for e in range(epochs): total_loss = 0 @@ -209,7 +209,7 @@ def gradient_descent(dataset, net, loss, epochs=1000, l_rate=0.01, batch_size=1, if verbose and (e + 1) % verbose == 0: print("epoch:{}, total_loss:{}".format(e + 1, total_loss)) - + return net @@ -238,10 +238,10 @@ def adam_optimizer(dataset, net, loss, epochs=1000, rho=(0.9, 0.999), delta=1 / for batch in get_batch(examples, batch_size): t += 1 inputs, targets = init_examples(batch, dataset.inputs, dataset.target, len(net[-1].nodes)) - + # compute gradients of weights gs, batch_loss = BackPropagation(inputs, targets, weights, net, loss) - + # update s,r,s_hat and r_gat s = vector_add(scalar_vector_product(rho[0], s), scalar_vector_product((1 - rho[0]), gs)) @@ -249,15 +249,15 @@ def adam_optimizer(dataset, net, loss, epochs=1000, rho=(0.9, 0.999), delta=1 / scalar_vector_product((1 - rho[1]), element_wise_product(gs, gs))) s_hat = scalar_vector_product(1 / (1 - rho[0] ** t), s) r_hat = scalar_vector_product(1 / (1 - rho[1] ** t), r) - + # rescale r_hat r_hat = map_vector(lambda x: 1 / (math.sqrt(x) + delta), r_hat) - + # delta weights delta_theta = scalar_vector_product(-l_rate, element_wise_product(s_hat, r_hat)) weights = vector_add(weights, delta_theta) total_loss += batch_loss - + # update the weights of network each batch for i in range(len(net)): if weights[i]: @@ -266,7 +266,7 @@ def adam_optimizer(dataset, net, loss, epochs=1000, rho=(0.9, 0.999), delta=1 / if verbose and (e + 1) % verbose == 0: print("epoch:{}, total_loss:{}".format(e + 1, total_loss)) - + return net @@ -405,7 +405,7 @@ def PerceptronLearner(dataset, learning_rate=0.01, epochs=100, verbose=None): # initialize the network, add dense layer raw_net = [InputLayer(input_size), DenseLayer(input_size, output_size)] - + # update the network learned_net = gradient_descent(dataset, raw_net, mse_loss, epochs, l_rate=learning_rate, verbose=verbose) @@ -478,7 +478,7 @@ def AutoencoderLearner(inputs, encoding_size, epochs=200): model.add(Dense(encoding_size, input_dim=input_size, activation='relu', kernel_initializer='random_uniform', bias_initializer='ones')) model.add(Dense(input_size, activation='relu', kernel_initializer='random_uniform', bias_initializer='ones')) - + # update model with sgd sgd = optimizers.SGD(lr=0.01) model.compile(loss='mean_squared_error', optimizer=sgd, metrics=['accuracy']) diff --git a/notebooks/chapter19/Learners.ipynb b/notebooks/chapter19/Learners.ipynb index 60c50cd1d..9997cfbcc 100644 --- a/notebooks/chapter19/Learners.ipynb +++ b/notebooks/chapter19/Learners.ipynb @@ -35,7 +35,7 @@ "source": [ "import os, sys\n", "sys.path = [os.path.abspath(\"../../\")] + sys.path\n", - "from DeepNeuralNet4e import *\n", + "from deep_learning4e import *\n", "from notebook4e import *\n", "from learning4e import *" ] @@ -482,13 +482,6 @@ "source": [ "After the model converging, the model's error ratio on the training set is still high. We will introduce the convolutional network in the following chapters to see how it helps improve accuracy on learning this dataset." ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/notebooks/chapter19/Loss Functions and Layers.ipynb b/notebooks/chapter19/Loss Functions and Layers.ipynb index eda7529ab..cccad7a88 100644 --- a/notebooks/chapter19/Loss Functions and Layers.ipynb +++ b/notebooks/chapter19/Loss Functions and Layers.ipynb @@ -116,7 +116,7 @@ "source": [ "import os, sys\n", "sys.path = [os.path.abspath(\"../../\")] + sys.path\n", - "from DeepNeuralNet4e import *\n", + "from deep_learning4e import *\n", "from notebook4e import *" ] }, @@ -372,13 +372,6 @@ "source": [ "We can see that each time kernel picks up the maximum value in its region." ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/notebooks/chapter19/Optimizer and Backpropagation.ipynb b/notebooks/chapter19/Optimizer and Backpropagation.ipynb index faa459ac5..e1c0a4db7 100644 --- a/notebooks/chapter19/Optimizer and Backpropagation.ipynb +++ b/notebooks/chapter19/Optimizer and Backpropagation.ipynb @@ -47,7 +47,7 @@ "source": [ "import os, sys\n", "sys.path = [os.path.abspath(\"../../\")] + sys.path\n", - "from DeepNeuralNet4e import *\n", + "from deep_learning4e import *\n", "from notebook4e import *" ] }, @@ -285,13 +285,6 @@ "source": [ "The demonstration of optimizers and back-propagation algorithm will be made together with neural network learners." ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/notebooks/chapter19/RNN.ipynb b/notebooks/chapter19/RNN.ipynb index 2b06b83a2..1383529fb 100644 --- a/notebooks/chapter19/RNN.ipynb +++ b/notebooks/chapter19/RNN.ipynb @@ -60,7 +60,7 @@ "source": [ "import os, sys\n", "sys.path = [os.path.abspath(\"../../\")] + sys.path\n", - "from DeepNeuralNet4e import *\n", + "from deep_learning4e import *\n", "from notebook4e import *" ] }, @@ -440,13 +440,6 @@ "source": [ "It shows we added two dense layers to the network structures." ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": {