Skip to content

Commit 4e8c908

Browse files
Mazecreatormichaelschaarschmidt
Mazecreator
authored and
michaelschaarschmidt
committed
Tensorboard and Gradient Summary
1 parent 4e8ea44 commit 4e8c908

7 files changed

Lines changed: 328 additions & 10 deletions

File tree

‎tensorforce/agents/agent.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,11 @@
2020
from copy import deepcopy
2121

2222
import numpy as np
23+
import inspect
2324

2425
from tensorforce import util, TensorForceError
2526
import tensorforce.agents
27+
from tensorforce.meta_parameter_recorder import MetaParameterRecorder
2628

2729

2830
class Agent(object):
@@ -94,6 +96,27 @@ def __init__(
9496
if isinstance(action['shape'], int):
9597
action['shape'] = (action['shape'],)
9698

99+
# TensorFlow summaries & Configuration Meta Parameter Recorder options
100+
if self.summary_spec is None:
101+
self.summary_labels = set()
102+
else:
103+
self.summary_labels = set(self.summary_spec.get('labels', ()))
104+
105+
self.meta_param_recorder = None
106+
107+
if 'configuration' in self.summary_labels or 'print_configuration' in self.summary_labels:
108+
self.meta_param_recorder = MetaParameterRecorder(inspect.currentframe())
109+
if 'meta_dict' in self.summary_spec:
110+
# Custom Meta Dictionary passed
111+
self.meta_param_recorder.merge_custom(self.summary_spec['meta_dict'])
112+
if 'configuration' in self.summary_labels:
113+
# Setup for TensorBoard population
114+
self.summary_spec['meta_param_recorder_class'] = self.meta_param_recorder
115+
if 'print_configuration' in self.summary_labels:
116+
# Print to STDOUT (TADO: optimize output)
117+
self.meta_param_recorder.text_output(format_type=1)
118+
119+
# Init Model, this must follow the Summary Configuration section above to cary meta_param_recorder
97120
self.model = self.initialize_model()
98121

99122
# Batched observe for better performance with Python.

‎tensorforce/core/optimizers/meta_optimizer.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,15 @@ class MetaOptimizer(Optimizer):
2727
more optimal step size.
2828
"""
2929

30-
def __init__(self, optimizer):
30+
def __init__(self, optimizer, **kwargs):
3131
"""
3232
Creates a new meta optimizer instance.
3333
3434
Args:
3535
optimizer: The optimizer which is modified by this meta optimizer.
3636
"""
37-
super(MetaOptimizer, self).__init__()
38-
self.optimizer = Optimizer.from_spec(spec=optimizer)
37+
super(MetaOptimizer, self).__init__(**kwargs)
38+
self.optimizer = Optimizer.from_spec(spec=optimizer, kwargs=kwargs)
3939

4040
def get_variables(self):
4141
return super(MetaOptimizer, self).get_variables() + self.optimizer.get_variables()

‎tensorforce/core/optimizers/multi_step.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,15 @@ class MultiStep(MetaOptimizer):
2929
optimizer a number of times.
3030
"""
3131

32-
def __init__(self, optimizer, num_steps=5):
32+
def __init__(self, optimizer, num_steps=5, **kwargs):
3333
"""
3434
Creates a new multi-step meta optimizer instance.
3535
3636
Args:
3737
optimizer: The optimizer which is modified by this meta optimizer.
3838
num_steps: Number of optimization steps to perform.
3939
"""
40-
super(MultiStep, self).__init__(optimizer=optimizer)
40+
super(MultiStep, self).__init__(optimizer=optimizer, **kwargs)
4141

4242
assert isinstance(num_steps, int) and num_steps > 0
4343
self.num_steps = num_steps

‎tensorforce/core/optimizers/optimizer.py‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,20 @@ class Optimizer(object):
3030
updating a set of variables.
3131
"""
3232

33-
def __init__(self):
33+
def __init__(self, **kwargs):
3434
"""
3535
Creates a new optimizer instance.
3636
"""
3737
self.variables = dict()
38+
if 'summaries' in kwargs:
39+
self.summaries = kwargs['summaries']
40+
del kwargs['summaries']
41+
if 'summary_labels' in kwargs:
42+
self.summary_labels = kwargs['summary_labels']
43+
del kwargs['summary_labels']
44+
else:
45+
self.summary_labels = dict()
46+
3847

3948
def custom_getter(getter, name, registered=False, **kwargs):
4049
variable = getter(name=name, registered=True, **kwargs)
@@ -90,6 +99,32 @@ def minimize(self, time, variables, **kwargs):
9099
Returns:
91100
The optimization operation.
92101
"""
102+
# Add training variable gradient histograms/scalars to summary output
103+
if 'gradients' in self.summary_labels:
104+
valid = True
105+
if isinstance(self, tensorforce.core.optimizers.TFOptimizer):
106+
gradients = self.optimizer.compute_gradients(kwargs['fn_loss']())
107+
elif isinstance(self.optimizer, tensorforce.core.optimizers.TFOptimizer):
108+
## This section handles "Multi_step" and may handle others
109+
# if failure is found, add another elif to handle that case
110+
gradients = self.optimizer.optimizer.compute_gradients(kwargs['fn_loss']())
111+
else:
112+
# Didn't find proper gradient information
113+
valid = False
114+
115+
# valid gradient data found, create summary data items
116+
if valid:
117+
for grad, var in gradients:
118+
if grad is not None:
119+
axes = list(range(len(grad.shape)))
120+
mean, var = tf.nn.moments(grad,axes)
121+
summary = tf.summary.scalar(name='gradients/'+var.name+"/mean", tensor=mean)
122+
self.summaries.append(summary)
123+
summary = tf.summary.scalar(name='gradients/'+var.name+"/variance", tensor=var)
124+
self.summaries.append(summary)
125+
summary = tf.summary.histogram(name='gradients/'+var.name, values=grad)
126+
self.summaries.append(summary)
127+
93128
deltas = self.step(time=time, variables=variables, **kwargs)
94129
with tf.control_dependencies(control_inputs=deltas):
95130
return tf.no_op()

‎tensorforce/core/optimizers/tf_optimizer.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,12 @@ def __init__(self, optimizer, **kwargs):
6262
'gradient_descent', 'momentum', 'rmsprop'.
6363
**kwargs: Additional arguments passed on to the TensorFlow optimizer constructor.
6464
"""
65-
super(TFOptimizer, self).__init__()
65+
super(TFOptimizer, self).__init__(**kwargs)
66+
# need to remove these from kwargs or error generated by TFOptimizer.tf_optimizers[optimizer](**kwargs)
67+
if 'summaries' in kwargs:
68+
del kwargs['summaries']
69+
if 'summary_labels' in kwargs:
70+
del kwargs['summary_labels']
6671

6772
self.name = optimizer
6873
self.optimizer = TFOptimizer.tf_optimizers[optimizer](**kwargs)

‎tensorforce/meta_parameter_recorder.py‎

Lines changed: 241 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,241 @@
1+
# Copyright 2017 reinforce.io. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
# ==============================================================================
15+
16+
import inspect
17+
import os
18+
import numpy as np
19+
import tensorflow as tf
20+
from tensorforce import TensorForceError
21+
22+
class MetaParameterRecorder(object):
23+
"""
24+
Class to record MetaParameters as well as Summary/Description for TensorBoard (TEXT & FILE will come later)
25+
26+
#### General:
27+
28+
* format_type: used to configure data convertion for TensorBoard=0, TEXT & JSON (not Implemented), etc
29+
"""
30+
31+
def __init__(self, current_frame):
32+
"""
33+
Init the MetaPrameterRecord with "Agent" parameters by passing inspect.currentframe() from Agent Class
34+
35+
The Init will search back to find the parent class to capture all passed parameters and store
36+
them in "self.meta_params".
37+
38+
NOTE: Currently only optimized for TensorBoard output
39+
40+
TODO: Add JSON Export, TEXT EXPORT
41+
42+
Args:
43+
current_frame: frame value from class to obtain metaparameters[= inspect.currentframe()]
44+
45+
"""
46+
self.ignore_unknown_dtypes = False
47+
self.meta_params = dict()
48+
self.method_calling = inspect.getframeinfo(current_frame)[2]
49+
50+
_, _, __, self.vals_current = inspect.getargvalues(current_frame)
51+
#self is the class name of the frame involved
52+
if 'self' in self.vals_current:
53+
self.recorded_class_type = self.vals_current['self']
54+
# Add explicit AgentName item so class can be deleted
55+
self.meta_params['AgentName'] = str(self.vals_current['self'])
56+
57+
frame_list = inspect.getouterframes(current_frame)
58+
59+
for frame in frame_list:
60+
#Rather than frame.frame (named tuple), use [0] for python2
61+
args, varargs, keywords, vals =inspect.getargvalues(frame[0])
62+
if 'self' in vals:
63+
if self.recorded_class_type == vals['self']:
64+
for i in args:
65+
self.meta_params[i] = vals[i]
66+
# Remove the "CLASS" from the dictionary, has no value "AgentName" contains STR of Class
67+
del self.meta_params['self']
68+
69+
70+
def merge_custom(self, custom_dict):
71+
if type(custom_dict) is not dict:
72+
raise TensorForceError("Error: MetaParameterRecorder 'meta_dict' must be passed a dictionary but was passed a type {} which is not supported.".format(str(type(custom_dictdata))))
73+
for key in custom_dict:
74+
if key in self.meta_params:
75+
raise TensorForceError("Error: MetaParameterRecorder 'meta_dict' key {} conflicts with internal key, please change passed key.".format(str(key)))
76+
self.meta_params[key] = custom_dict[key]
77+
# This line assumes the merge data came from summary_spec['meta_dict'], remove this from summary_spec
78+
del self.meta_params['summary_spec']['meta_dict']
79+
80+
def text_output(self, format_type=1):
81+
print('======================= ' + self.meta_params['AgentName'] + ' ====================================')
82+
for key in self.meta_params:
83+
print (" ", key, type(self.meta_params[key]),"=", self.convert_data_to_string(self.meta_params[key], format_type=format_type))
84+
print('======================= ' + self.meta_params['AgentName'] + ' ====================================')
85+
86+
def convert_dictionary_to_string(self, data, indent=0, format_type=0, seperator=None, eol=None):
87+
data_string = ""
88+
add_seperator = ""
89+
if eol is None:
90+
eol = os.linesep
91+
if seperator is None:
92+
seperator = ", "
93+
94+
#This should not ever occur but here as a catch
95+
if type(data) is not dict:
96+
raise TensorForceError("Error: MetaParameterRecorder Dictionary conversion was passed a type {} not supported.".format(str(type(data))))
97+
98+
if format_type == 0: # TensorBoard
99+
label=""
100+
div=""
101+
if indent>0:
102+
label= " | "
103+
div = "--- | "
104+
data_string += label + "Key | Value" + eol + div + "--- | ----" + eol
105+
106+
for key in data:
107+
key_txt = key
108+
if format_type == 0: # TensorBoard
109+
key_txt = "**" + key + "**"
110+
key_value_sep = ' | '
111+
if indent>0:
112+
key_txt=" | " + key_txt
113+
114+
data_string += add_seperator + key_txt + key_value_sep + self.convert_data_to_string(data[key], seperator=seperator, indent=indent+1) + eol
115+
116+
return data_string
117+
118+
def convert_list_to_string(self, data, indent=0, format_type=0, eol=None, count=True):
119+
data_string =""
120+
if eol is None:
121+
eol = os.linesep
122+
123+
#This should not ever occur but here as a catch
124+
if type(data) is not list:
125+
raise TensorForceError("Error: MetaParameterRecorder List conversion was passed a type {} not supported.".format(str(type(data))))
126+
127+
for index,line in enumerate(data):
128+
data_string_prefix = ""
129+
if count and indent==0:
130+
data_string_prefix = str(index+1)+". "
131+
if format_type == 0: # TensorBoard
132+
#Only add indent for 2nd item and beyond as this is likely a dictionary entry
133+
if indent>0 and index>0:
134+
data_string_prefix = " | "+data_string_prefix
135+
if index==(len(data)-1):
136+
append_eol = ""
137+
else:
138+
append_eol = eol
139+
data_string += data_string_prefix + self.convert_data_to_string(line, indent=indent+1) + append_eol
140+
141+
return data_string
142+
143+
def convert_ndarray_to_md(self, data, format_type=0, eol=None):
144+
data_string =""
145+
data_string1 = "|Row|"
146+
data_string2 = "|:---:|"
147+
if eol is None:
148+
eol = os.linesep
149+
150+
#This should not ever occur but here as a catch
151+
if type(data) is not np.ndarray:
152+
raise TensorForceError("Error: MetaParameterRecorder ndarray conversion was passed a type {} not supported.".format(str(type(data))))
153+
154+
shape = data.shape
155+
rank = data.ndim
156+
157+
if rank == 2:
158+
for col in range(shape[1]):
159+
data_string1 += "Col-" + str(col) + "|"
160+
data_string2 += ":----:|"
161+
data_string += data_string1 + eol + data_string2 + eol
162+
163+
for row in range(shape[0]):
164+
data_string += "|" + str(row) + "|"
165+
for col in range(shape[1]):
166+
data_string += str(data[row,col]) + "|"
167+
168+
if row!=(shape[0]-1):
169+
data_string += eol
170+
171+
elif rank == 1:
172+
data_string += "|Row|Col-0|" + eol + "|:----:|:----:|" + eol
173+
174+
for row in range(shape[0]):
175+
data_string += str(row) + "|" + str(data[row]) +"|" + eol
176+
177+
return data_string
178+
179+
def convert_data_to_string(self, data, indent=0, format_type=0, seperator=None, eol=None):
180+
data_string =""
181+
if type(data) is int:
182+
data_string = str(data)
183+
elif type(data) is float:
184+
data_string = str(data)
185+
elif type(data) is str:
186+
data_string = data
187+
elif type(data) is tuple:
188+
data_string = str(data)
189+
elif type(data) is list:
190+
data_string = self.convert_list_to_string(data, indent=indent, eol=eol)
191+
elif type(data) is bool:
192+
data_string = str(data)
193+
elif type(data) is dict:
194+
data_string = self.convert_dictionary_to_string(data, indent=indent, seperator=seperator)
195+
elif type(data) is np.ndarray:
196+
if format_type==0: # TensorBoard
197+
data_string = self.convert_ndarray_to_md(data)
198+
else:
199+
data_string = str(data)
200+
elif data is None:
201+
data_string = "None"
202+
else:
203+
if not self.ignore_unknown_dtypes:
204+
data_string ="Error: MetaParameterRecorder Type conversion from type {} not supported.".format(str(type(data)))
205+
data_string += " ("+str(data)+") "
206+
else:
207+
if format_type == 0: # TensorBoard
208+
data_string = "**?**"
209+
210+
return data_string
211+
212+
def build_metagraph_list(self):
213+
"""
214+
Convert MetaParams into TF Summary Format and create summary_op
215+
216+
Args:
217+
None
218+
219+
Returns:
220+
Merged TF Op for TEXT summary elements, should only be executed once to reduce data duplication
221+
222+
"""
223+
ops = []
224+
225+
self.ignore_unknown_dtypes = True
226+
for key in sorted(self.meta_params):
227+
value=self.convert_data_to_string(self.meta_params[key])
228+
229+
if len(value) == 0:
230+
continue
231+
if isinstance(value,str):
232+
ops.append(tf.summary.text(key, tf.convert_to_tensor(str(value))))
233+
else:
234+
ops.append(tf.summary.text(key, tf.as_string(tf.convert_to_tensor(value))))
235+
236+
with tf.control_dependencies(tf.tuple(ops)):
237+
self.summary_merged = tf.summary.merge_all()
238+
239+
return self.summary_merged
240+
241+

0 commit comments

Comments
 (0)