Merge branch 'master' of https://github.com/PanDav2/Clockwork-RNN

PanDav2 · PanDav2 · commit fb3047ce3fe4 · 2017-01-19T02:42:28.000+01:00
diff --git a/main.py b/main.py
@@ -17,14 +17,14 @@
         'hidden_dim': 9,
         'output_dim': 1,
         'periods': [1, 2, 4, 8, 16, 32, 64, 128, 256],
-        'num_steps': 320,
+        'num_steps': 10,
 
-        'learning_rate': 3e-4,
-        'learning_rate_step': 200,
-        'learning_rate_decay': 0.95,
+        'learning_rate': 1e-2,
+        'learning_rate_step': 50,
+        'learning_rate_decay': 0.9,
+        'optimizer': 'rmsprop',
         'momentum': 0.95,
-        'max_gradient': 10,
-        'max_epochs': 5000
+        'max_epochs': 1000
     }
 
     ### Create the model ###
@@ -75,7 +75,6 @@
         outputs = outputs.reshape(-1)
         ground_truth = targets.reshape(-1)
 
-        print(outputs)
         # Final result
         print('')
         print('')
diff --git a/models/clockwork_rnn.py b/models/clockwork_rnn.py
@@ -19,12 +19,12 @@ def __init__(self, config):
                 - Learning rate: learning_rate
                 - Learning rate: learning_rate_step
                 - Learning rate: learning_rate_decay
+                - Optimized used (momentum, rmsprop): optimizer
                 - Momentum: momentum
-                - Gradient clipping (by norm): max_gradient
         """
 
         # Check config
-        for v in ['input_dim', 'hidden_dim', 'output_dim', 'periods', 'num_steps', 'learning_rate', 'learning_rate_step', 'learning_rate_decay', 'momentum', 'max_gradient']:
+        for v in ['input_dim', 'hidden_dim', 'output_dim', 'periods', 'num_steps', 'learning_rate', 'learning_rate_step', 'learning_rate_decay', 'optimizer', 'momentum']:
             if v not in config:
                 print('Missing config[\'{}\']'.format(v))
                 exit(1)
@@ -137,7 +137,13 @@ def create_optimizer(self):
                                 self.config['learning_rate_decay'], # Decay rate.
                                 staircase = True)
 
-        self.optimizer = tf.train.MomentumOptimizer(self.learning_rate, self.config['momentum'], use_nesterov = True)
+        if self.config['optimizer'] == 'momentum':
+            self.optimizer = tf.train.MomentumOptimizer(self.learning_rate, self.config['momentum'], use_nesterov = True)
+        elif self.config['optimizer'] == 'rmsprop':
+            self.optimizer = tf.train.RMSPropOptimizer(self.learning_rate)
+        else:
+            print('Unknown optimizer {}'.format(self.config['optimizer']))
+            exit(1)
 
         self.train_step = self.optimizer.minimize(self.loss, global_step = self.global_step)
 
