# Copyright 2017 reinforce.io. All Rights Reserved.

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

# ==============================================================================

from __future__ import absolute_import

from __future__ import division

from __future__ import print_function

import tensorflow as tf

from tensorforce import util

from tensorforce.models import QModel

class QNstepModel(QModel):

"""

Deep Q network using n-step rewards as described in Asynchronous Methods for Reinforcement Learning.

"""

def tf_q_delta(self, q_value, next_q_value, terminal, reward):

for _ in range(util.rank(q_value) - 1):

terminal = tf.expand_dims(input=terminal, axis=1)

reward = tf.expand_dims(input=reward, axis=1)

multiples = (1,) + util.shape(q_value)[1:]

terminal = tf.tile(input=terminal, multiples=multiples)

reward = tf.tile(input=reward, multiples=multiples)

reward = self.fn_discounted_cumulative_reward(

terminal=terminal,

reward=reward,

discount=self.discount,

final_reward=next_q_value[-1]

)

return reward - q_value