added target baseline to ddpg

krfricke · krfricke · commit 38dd34eb6c30 · 2018-01-27T13:58:06.000+01:00
diff --git a/examples/configs/ddpg.json b/examples/configs/ddpg.json
@@ -4,7 +4,7 @@
     "update_mode": {
         "unit": "timesteps",
         "batch_size": 200,
-        "frequency": 1
+        "frequency": 200
     },
     "memory": {
         "type": "replay",
diff --git a/examples/configs/vpg.json b/examples/configs/vpg.json
@@ -4,7 +4,7 @@
     "update_mode": {
         "unit": "episodes",
         "batch_size": 20,
-        "frequency": 1
+        "frequency": 20
     },
     "memory": {
         "type": "latest",
diff --git a/tensorforce/agents/ddpg_agent.py b/tensorforce/agents/ddpg_agent.py
@@ -23,8 +23,8 @@
 
 class DDPGAgent(LearningAgent):
     """
-    Deep Deterministic Policy Gradient agent as described by [Silver et al. (2014)]
-    (http://proceedings.mlr.press/v32/silver14.pdf).
+    Deep Deterministic Policy Gradient agent as described by [Lillicrap et al. (2016)]
+    (https://arxiv.org/pdf/1509.02971.pdf).
 
     """
 
diff --git a/tensorforce/models/pg_log_prob_target_model.py b/tensorforce/models/pg_log_prob_target_model.py
@@ -22,6 +22,7 @@
 from tensorforce import util
 from tensorforce.models import PGLogProbModel
 
+from tensorforce.core.baselines import Baseline, AggregatedBaseline
 from tensorforce.core.networks import Network
 from tensorforce.core.optimizers import Synchronization
 
@@ -67,6 +68,8 @@ def __init__(
         self.target_network = None
         self.target_optimizer = None
         self.target_distributions = None
+        self.target_baseline = None
+        self.target_baseline_optimizer = None
 
         super(PGLogProbModel, self).__init__(
             states=states,
@@ -112,18 +115,73 @@ def initialize(self, custom_getter):
         # Target network distributions
         self.target_distributions = self.create_distributions()
 
-    def tf_pg_loss_per_instance(self, states, internals, actions, terminal, reward, next_states, next_internals, update):
-        embedding = self.target_network.apply(x=states, internals=internals, update=update)
-        log_probs = list()
+        # Target baseline
+        if self.baseline_mode:
+            if all(name in self.states_spec for name in self.baseline_spec):
+                # Implies AggregatedBaseline.
+                assert self.baseline_mode == 'states'
+                self.target_baseline = AggregatedBaseline(baselines=self.baseline_spec)
+            else:
+                self.target_baseline = Baseline.from_spec(
+                    spec=self.baseline_spec,
+                    kwargs=dict(
+                        summary_labels=self.summary_labels,
+                        scope='target_baseline'
+                    )
+                )
+
+            # Target baseline optimizer
+            self.target_baseline_optimizer = Synchronization(
+                sync_frequency=self.target_sync_frequency,
+                update_weight=self.target_update_weight
+            )
+
+    def tf_reward_estimation(self, states, internals, terminal, reward, update):
+        if self.baseline_mode is None:
+            reward = self.fn_discounted_cumulative_reward(terminal=terminal, reward=reward, discount=self.discount)
 
-        for name, distribution in self.target_distributions.items():
-            distr_params = distribution.parameterize(x=embedding)
-            log_prob = distribution.log_probability(distr_params=distr_params, action=actions[name])
-            collapsed_size = util.prod(util.shape(log_prob)[1:])
-            log_prob = tf.reshape(tensor=log_prob, shape=(-1, collapsed_size))
-            log_probs.append(log_prob)
-        log_prob = tf.reduce_mean(input_tensor=tf.concat(values=log_probs, axis=1), axis=1)
-        return -log_prob * reward
+        else:
+            assert self.target_baseline
+            if self.baseline_mode == 'states':
+                state_value = self.target_baseline.predict(
+                    states=states,
+                    internals=internals,
+                    update=update
+                )
+
+            elif self.baseline_mode == 'network':
+                embedding = self.target_network.apply(
+                    x=states,
+                    internals=internals,
+                    update=update
+                )
+                state_value = self.target_baseline.predict(
+                    states=embedding,
+                    internals=internals,
+                    update=update
+                )
+
+            if self.gae_lambda is None:
+                reward = self.fn_discounted_cumulative_reward(
+                    terminal=terminal,
+                    reward=reward,
+                    discount=self.discount
+                )
+                reward -= state_value
+
+            else:
+                next_state_value = tf.concat(values=(state_value[1:], (0.0,)), axis=0)
+                zeros = tf.zeros_like(tensor=next_state_value)
+                next_state_value = tf.where(condition=terminal, x=zeros, y=next_state_value)
+                td_residual = reward + self.discount * next_state_value - state_value
+                gae_discount = self.discount * self.gae_lambda
+                reward = self.fn_discounted_cumulative_reward(
+                    terminal=terminal,
+                    reward=td_residual,
+                    discount=gae_discount
+                )
+
+        return reward
 
     def tf_optimization(self, states, internals, actions, terminal, reward, next_states=None, next_internals=None):
         optimization = super(PGLogProbModel, self).tf_optimization(
@@ -145,6 +203,14 @@ def tf_optimization(self, states, internals, actions, terminal, reward, next_sta
             source_variables=self.network.get_variables() + network_distributions_variables
         )
 
+        if self.target_baseline:
+            target_baseline_optimization = self.target_baseline_optimizer.minimize(
+                time=self.timestep,
+                variables=self.target_baseline.get_variables(),
+                source_variables=self.baseline.get_variables()
+            )
+            return tf.group(optimization, target_optimization, target_baseline_optimization)
+
         return tf.group(optimization, target_optimization)
 
     def get_variables(self, include_non_trainable=False):
@@ -156,11 +222,16 @@ def get_variables(self, include_non_trainable=False):
             target_distributions_variables = self.get_distributions_variables(self.target_distributions)
             target_optimizer_variables = self.target_optimizer.get_variables()
 
+            if self.target_baseline:
+                target_baseline_variables = self.target_baseline.get_variables()
+                return model_variables + target_variables + target_optimizer_variables + \
+                    target_distributions_variables + target_baseline_variables
+
             return model_variables + target_variables + target_optimizer_variables + target_distributions_variables
         else:
             return model_variables
 
     def get_summaries(self):
         target_distributions_summaries = self.get_distributions_summaries(self.target_distributions)
         return super(PGLogProbModel, self).get_summaries() + self.target_network.get_summaries() \
-               + target_distributions_summaries
+            + target_distributions_summaries
