fixed ddpg determinstic act

krfricke · krfricke · commit 6c9847dd7cfc · 2018-02-19T20:31:03.000+01:00
diff --git a/examples/configs/mlp2_normalized_network.json b/examples/configs/mlp2_normalized_network.json
@@ -28,7 +28,7 @@
 
     {
         "type": "dense",
-        "size": 1,
-        "activation": "tanh"
+        "size": 64,
+        "activation": null
     }
 ]
diff --git a/tensorforce/agents/ddpg_agent.py b/tensorforce/agents/ddpg_agent.py
@@ -190,7 +190,3 @@ def initialize_model(self):
             target_sync_frequency=self.target_sync_frequency,
             target_update_weight=self.target_update_weight
         )
-
-    def act(self, states, deterministic=False, fetch_tensors=None):
-        # Always return deterministic action from model, add exploration separately
-        return super(DDPGAgent, self).act(states=states, deterministic=True, fetch_tensors=fetch_tensors)
diff --git a/tensorforce/models/dpg_target_model.py b/tensorforce/models/dpg_target_model.py
@@ -82,6 +82,9 @@ def tf_apply(self, x, internals, update, return_internals=False):
 
         out = self.t2d.apply(x=out, update=update)
 
+        # Remove last dimension because we only return Q values for one state and action
+        out = tf.squeeze(out)
+
         if return_internals:
             # Todo: Internals management
             return out, None
@@ -249,6 +252,7 @@ def tf_optimization(self, states, internals, actions, terminal, reward, next_sta
             states=next_states, internals=next_internals, actions=next_target_actions, terminal=terminal,
             reward=reward, update=update
         )
+
         predicted_q = tf.stop_gradient(input=predicted_q)
 
         real_q = self.critic.apply(dict(states=states, actions=actions), internals=internals, update=update)

Original file line number	Diff line number	Diff line change
`@@ -28,7 +28,7 @@`
`28`	`28`
`29`	`29`	`{`
`30`	`30`	`"type": "dense",`
`31`		`- "size": 1,`
`32`		`- "activation": "tanh"`
	`31`	`+ "size": 64,`
	`32`	`+ "activation": null`
`33`	`33`	`}`
`34`	`34`	`]`
Original file line number	Diff line number	Diff line change
`@@ -190,7 +190,3 @@ def initialize_model(self):`
`190`	`190`	`target_sync_frequency=self.target_sync_frequency,`
`191`	`191`	`target_update_weight=self.target_update_weight`
`192`	`192`	`)`
`193`		`-`
`194`		`- def act(self, states, deterministic=False, fetch_tensors=None):`
`195`		`- # Always return deterministic action from model, add exploration separately`
`196`		`- return super(DDPGAgent, self).act(states=states, deterministic=True, fetch_tensors=fetch_tensors)`