various updates

aymericdamien · aymericdamien · commit f55ff0eb6f94 · 2017-05-29T03:35:45.000+01:00
diff --git a/examples/2_BasicModels/random_forest.py b/examples/2_BasicModels/random_forest.py
@@ -0,0 +1,69 @@
+""" Random Forest.
+
+Implement Random Forest algorithm with TensorFlow, and apply it to classify 
+handwritten digit images. This example is using the MNIST database of 
+handwritten digits as training samples (http://yann.lecun.com/exdb/mnist/).
+
+Author: Aymeric Damien
+Project: https://github.com/aymericdamien/TensorFlow-Examples/
+"""
+
+from __future__ import print_function
+
+import tensorflow as tf
+from tensorflow.contrib.tensor_forest.python import tensor_forest
+
+# Ignore all GPUs, tf random forest does not benefit from it.
+import os
+os.environ["CUDA_VISIBLE_DEVICES"] = ""
+
+# Import MNIST data
+from tensorflow.examples.tutorials.mnist import input_data
+mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
+
+# Parameters
+num_steps = 10000 # Total steps to train
+batch_size = 512 # The number of samples per batch
+num_classes = 10 # The 10 digits
+num_features = 784 # Each image is 28x28 pixels
+num_trees = 100
+max_nodes = 1000
+
+# Input and Target data
+X = tf.placeholder(tf.float32, shape=[None, num_features])
+Y = tf.placeholder(tf.float32, shape=[None, num_classes])
+
+# Random Forest Parameters
+hparams = tensor_forest.ForestHParams(num_classes=num_classes,
+                                      num_features=num_features,
+                                      num_trees=num_trees,
+                                      max_nodes=max_nodes).fill()
+# Build the Random Forest
+forest_graph = tensor_forest.RandomForestGraphs(hparams)
+# Get training graph and loss
+train_op = forest_graph.training_graph(X, Y)
+loss_op = forest_graph.training_loss(X, Y)
+
+# Measure the accuracy
+infer_op = forest_graph.inference_graph(X)
+correct_prediction = tf.equal(tf.argmax(infer_op, 1), tf.argmax(Y, 1))
+accuracy_op = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
+
+# Initialize all variables
+init = tf.global_variables_initializer()
+
+with tf.Session() as sess:
+    sess.run(init)
+
+    # Training
+    for i in range(1, num_steps + 1):
+        # Prepare Data
+        # Get the next batch of MNIST data (only images are needed, not labels)
+        batch_x, batch_y = mnist.train.next_batch(batch_size)
+        _, l = sess.run([train_op, loss_op], feed_dict={X: batch_x, Y: batch_y})
+        if i % 100 == 0 or i == 1:
+            acc = sess.run(accuracy_op, feed_dict={X: batch_x, Y: batch_y})
+            print('Step %i, Loss: %f, Acc: %f' % (i, l, acc))
+
+
+
diff --git a/examples/3_NeuralNetworks/dcgan.py b/examples/3_NeuralNetworks/dcgan.py
@@ -26,7 +26,7 @@
 mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
 
 # Training Params
-n_steps = 20000
+num_steps = 20000
 batch_size = 32
 
 # Network Params
@@ -124,7 +124,7 @@ def discriminator(x, reuse=False):
     sess.run(init)
 
     # Training
-    for i in range(1, n_steps+1):
+    for i in range(1, num_steps+1):
 
         # Prepare Input Data
         # Get the next batch of MNIST data (only images are needed, not labels)
diff --git a/examples/3_NeuralNetworks/gan.py b/examples/3_NeuralNetworks/gan.py
@@ -30,7 +30,7 @@
 mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
 
 # Training Params
-n_steps = 30000
+num_steps = 30000
 batch_size = 32
 
 # Network Params
@@ -39,7 +39,7 @@
 disc_hidden_dim = 256
 noise_dim = 100 # Noise data points
 
-# A custom initialization (see Xavier glorot init)
+# A custom initialization (see Xavier Glorot init)
 def glorot_init(shape):
     return tf.random_normal(shape=shape, stddev=1. / tf.sqrt(shape[0] / 2.))
 
@@ -120,7 +120,7 @@ def discriminator(x):
     sess.run(init)
 
     # Training
-    for i in range(1, n_steps+1):
+    for i in range(1, num_steps+1):
         # Prepare Data
         # Get the next batch of MNIST data (only images are needed, not labels)
         batch_x, _ = mnist.train.next_batch(batch_size)
diff --git a/examples/3_NeuralNetworks/variational_autoencoder.py b/examples/3_NeuralNetworks/variational_autoencoder.py
@@ -6,12 +6,15 @@
 References:
     - Auto-Encoding Variational Bayes The International Conference on Learning
     Representations (ICLR), Banff, 2014. D.P. Kingma, M. Welling
+    - Understanding the difficulty of training deep feedforward neural networks.
+    X Glorot, Y Bengio. Aistats 9, 249-256
     - Y. LeCun, L. Bottou, Y. Bengio, and P. Haffner. "Gradient-based
     learning applied to document recognition." Proceedings of the IEEE,
     86(11):2278-2324, November 1998.
 
 Links:
     - [VAE Paper] https://arxiv.org/abs/1312.6114
+    - [Xavier Glorot Init](www.cs.cmu.edu/~bhiksha/courses/deeplearning/Fall.../AISTATS2010_Glorot.pdf).
     - [MNIST Dataset] http://yann.lecun.com/exdb/mnist/
 
 Author: Aymeric Damien
@@ -30,15 +33,15 @@
 
 # Parameters
 learning_rate = 0.001
-n_steps = 20000
-batch_size = 32
+num_steps = 30000
+batch_size = 64
 
 # Network Parameters
 image_dim = 784 # MNIST images are 28x28 pixels
-hidden_dim = 256
+hidden_dim = 512
 latent_dim = 2
 
-# A custom initialization (see Xavier glorot init)
+# A custom initialization (see Xavier Glorot init)
 def glorot_init(shape):
     return tf.random_normal(shape=shape, stddev=1. / tf.sqrt(shape[0] / 2.))
 
@@ -61,7 +64,7 @@ def glorot_init(shape):
 # Building the encoder
 input_image = tf.placeholder(tf.float32, shape=[None, image_dim])
 encoder = tf.matmul(input_image, weights['encoder_h1']) + biases['encoder_b1']
-encoder = tf.nn.relu(encoder)
+encoder = tf.nn.tanh(encoder)
 z_mean = tf.matmul(encoder, weights['z_mean']) + biases['z_mean']
 z_std = tf.matmul(encoder, weights['z_std']) + biases['z_std']
 
@@ -72,7 +75,7 @@ def glorot_init(shape):
 
 # Building the decoder (with scope to re-use these layers later)
 decoder = tf.matmul(z, weights['decoder_h1']) + biases['decoder_b1']
-decoder = tf.nn.relu(decoder)
+decoder = tf.nn.tanh(decoder)
 decoder = tf.matmul(decoder, weights['decoder_out']) + biases['decoder_out']
 decoder = tf.nn.sigmoid(decoder)
 
@@ -99,7 +102,7 @@ def vae_loss(x_reconstructed, x_true):
     sess.run(init)
 
     # Training
-    for i in range(1, n_steps+1):
+    for i in range(1, num_steps+1):
         # Prepare Data
         # Get the next batch of MNIST data (only images are needed, not labels)
         batch_x, _ = mnist.train.next_batch(batch_size)
@@ -115,24 +118,24 @@ def vae_loss(x_reconstructed, x_true):
     noise_input = tf.placeholder(tf.float32, shape=[None, latent_dim])
     # Rebuild the decoder to create image from noise
     decoder = tf.matmul(noise_input, weights['decoder_h1']) + biases['decoder_b1']
-    decoder = tf.nn.relu(decoder)
+    decoder = tf.nn.tanh(decoder)
     decoder = tf.matmul(decoder, weights['decoder_out']) + biases['decoder_out']
     decoder = tf.nn.sigmoid(decoder)
 
     # Building a manifold of generated digits
-    n = 15  # Figure row size
-    figure = np.zeros((28 * n, 28 * n))
-    # Random normal distributions to feed network with
-    x_axis = norm.ppf(np.linspace(0., 1., n))
-    y_axis = norm.ppf(np.linspace(0., 1., n))
-
-    for i, x in enumerate(x_axis):
-        for j, y in enumerate(y_axis):
-            samples = np.array([[x, y]])
-            x_reconstructed = sess.run(decoder, feed_dict={noise_input: samples})
-            digit = np.array(x_reconstructed[0]).reshape(28, 28)
-            figure[i * 28: (i + 1) * 28, j * 28: (j + 1) * 28] = digit
-
-    plt.figure(figsize=(10, 10))
-    plt.imshow(figure, cmap='Greys_r')
+    n = 20
+    x_axis = np.linspace(-3, 3, n)
+    y_axis = np.linspace(-3, 3, n)
+
+    canvas = np.empty((28 * n, 28 * n))
+    for i, yi in enumerate(x_axis):
+        for j, xi in enumerate(y_axis):
+            z_mu = np.array([[xi, yi]] * batch_size)
+            x_mean = sess.run(decoder, feed_dict={noise_input: z_mu})
+            canvas[(n - i - 1) * 28:(n - i) * 28, j * 28:(j + 1) * 28] = \
+            x_mean[0].reshape(28, 28)
+
+    plt.figure(figsize=(8, 10))
+    Xi, Yi = np.meshgrid(x_axis, y_axis)
+    plt.imshow(canvas, origin="upper", cmap="gray")
     plt.show()
