Skip to content

Commit d84ee13

Browse files
Oceania2018Oceania2018
committed
keras.layers.LayerNormalization SciSharp#875 
1 parent 3f0b1f9 commit d84ee13

10 files changed

Lines changed: 94 additions & 39 deletions

File tree

‎src/TensorFlowNET.Core/APIs/tf.nn.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -124,10 +124,10 @@ public Tensor relu(Tensor features, string name = null)
124124
=> gen_nn_ops.relu(features, name);
125125

126126
public Tensor[] fused_batch_norm(Tensor x,
127-
IVariableV1 scale,
128-
IVariableV1 offset,
129-
IVariableV1 mean = null,
130-
IVariableV1 variance = null,
127+
Tensor scale,
128+
Tensor offset,
129+
Tensor mean = null,
130+
Tensor variance = null,
131131
float epsilon = 0.001f,
132132
string data_format = "NHWC",
133133
bool is_training = true,

‎src/TensorFlowNET.Core/Gradients/ops.gradient_function_mapping.cs‎

Lines changed: 42 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ limitations under the License.
1919
using System.Linq;
2020
using System.Reflection;
2121
using Tensorflow.Gradients;
22-
using static Tensorflow.Binding;
2322

2423
namespace Tensorflow
2524
{
@@ -49,14 +48,48 @@ public static void RegisterFromAssembly()
4948
RegisterGradientFunction(m.GetCustomAttribute<RegisterGradient>().Name,
5049
(oper, out_grads) =>
5150
{
52-
tf.Logger.Debug($"Caculate Gradient: {oper.name} {m.Name}");
53-
var results = g.InvokeMember(m.Name,
54-
BindingFlags.InvokeMethod,
55-
null,
56-
null,
57-
args: new object[] { oper, out_grads }) as Tensor[];
58-
foreach (var result in results.Where(x => x != null))
59-
tf.Logger.Debug($"Gradient: {result.name} {result.shape}");
51+
// tf.Logger.Debug($"Caculate Gradient: {oper.name} {m.Name}");
52+
53+
var results = m.Name switch
54+
{
55+
"_AddGrad" => math_grad._AddGrad(oper, out_grads),
56+
"_AddV2Grad" => math_grad._AddV2Grad(oper, out_grads),
57+
"_BiasAddGrad" => nn_grad._BiasAddGrad(oper, out_grads),
58+
"_CastGrad" => math_grad._CastGrad(oper, out_grads),
59+
"_ConcatGradV2" => array_grad._ConcatGradV2(oper, out_grads),
60+
"_Conv2DGrad" => nn_grad._Conv2DGrad(oper, out_grads),
61+
"_ExpandDimsGrad" => array_grad._ExpandDimsGrad(oper, out_grads),
62+
"_ExpGrad" => math_grad._ExpGrad(oper, out_grads),
63+
"_FusedBatchNormV3Grad" => nn_grad._FusedBatchNormV3Grad(oper, out_grads),
64+
"_IdGrad" => math_grad._IdGrad(oper, out_grads),
65+
"_LeakyReluGrad" => nn_grad._LeakyReluGrad(oper, out_grads),
66+
"_Log1pGrad" => math_grad._Log1pGrad(oper, out_grads),
67+
"_MaximumGrad" => math_grad._MaximumGrad(oper, out_grads),
68+
"_MeanGrad" => math_grad._MeanGrad(oper, out_grads),
69+
"_MinimumGrad" => math_grad._MinimumGrad(oper, out_grads),
70+
"_MulGrad" => math_grad._MulGrad(oper, out_grads),
71+
"_NegGrad" => math_grad._NegGrad(oper, out_grads),
72+
"_PadGrad" => array_grad._PadGrad(oper, out_grads),
73+
"_PowGrad" => math_grad._PowGrad(oper, out_grads),
74+
"_RealDivGrad" => math_grad._RealDivGrad(oper, out_grads),
75+
"_ReadGrad" => resource_variable_grad._ReadGrad(oper, out_grads),
76+
"_ReshapeGrad" => array_grad._ReshapeGrad(oper, out_grads),
77+
"_ResizeNearestNeighborGrad" => image_grad._ResizeNearestNeighborGrad(oper, out_grads),
78+
"_SelectGrad" => math_grad._SelectGrad(oper, out_grads),
79+
"_SigmoidGrad" => math_grad._SigmoidGrad(oper, out_grads),
80+
"_SumGrad" => math_grad._SumGrad(oper, out_grads),
81+
"_SubGrad" => math_grad._SubGrad(oper, out_grads),
82+
"_StridedSliceGrad" => array_grad._StridedSliceGrad(oper, out_grads),
83+
_ => g.InvokeMember(m.Name,
84+
BindingFlags.InvokeMethod,
85+
null,
86+
null,
87+
args: new object[] { oper, out_grads }) as Tensor[]
88+
};
89+
90+
// foreach (var result in results.Where(x => x != null))
91+
// tf.Logger.Debug($"Gradient: {result.name} {result.shape}");
92+
6093
return results;
6194
}
6295
);

‎src/TensorFlowNET.Core/NumPy/NDArray.Equal.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@ public override bool Equals(object obj)
1717
float val => GetAtIndex<float>(0) == val,
1818
double val => GetAtIndex<double>(0) == val,
1919
string val => StringData(0) == val,
20+
int[] val => ToArray<int>().SequenceEqual(val),
21+
long[] val => ToArray<long>().SequenceEqual(val),
22+
float[] val => ToArray<float>().SequenceEqual(val),
23+
double[] val => ToArray<double>().SequenceEqual(val),
2024
NDArray val => Equals(this, val),
2125
_ => base.Equals(obj)
2226
};

‎src/TensorFlowNET.Core/Operations/NnOps/gen_nn_ops.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -191,10 +191,10 @@ public static Tensor[] fused_batch_norm(Tensor x,
191191
}
192192

193193
public static Tensors fused_batch_norm_v3(Tensor x,
194-
IVariableV1 scale,
195-
IVariableV1 offset,
196-
IVariableV1 mean,
197-
IVariableV1 variance,
194+
Tensor scale,
195+
Tensor offset,
196+
Tensor mean,
197+
Tensor variance,
198198
float epsilon = 0.0001f,
199199
float exponential_avg_factor = 1.0f,
200200
string data_format = "NHWC",

‎src/TensorFlowNET.Core/Operations/nn_impl.py.cs‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -150,20 +150,18 @@ public static Tensor batch_normalization(Tensor x,
150150
/// <param name="name"></param>
151151
/// <returns></returns>
152152
public static Tensor[] fused_batch_norm(Tensor x,
153-
IVariableV1 scale,
154-
IVariableV1 offset,
155-
IVariableV1 mean,
156-
IVariableV1 variance,
153+
Tensor scale,
154+
Tensor offset,
155+
Tensor mean = null,
156+
Tensor variance = null,
157157
float epsilon = 0.001f,
158158
string data_format = "NHWC",
159159
bool is_training = true,
160160
string name = null,
161161
float exponential_avg_factor = 1.0f)
162162
{
163-
/*if (mean == null)
164-
mean = constant_op.constant(new float[0]);
165-
if (variance == null)
166-
variance = constant_op.constant(new float[0]);*/
163+
mean = mean ?? constant_op.constant(new float[0]);
164+
variance = variance ?? constant_op.constant(new float[0]);
167165
var min_epsilon = 1.001e-5f;
168166
epsilon = epsilon > min_epsilon ? epsilon : min_epsilon;
169167

‎src/TensorFlowNET.Core/Tensors/Tensors.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ public Tensor this[int index]
2929
{
3030
get
3131
{
32+
if (Length == 1)
33+
return items[0][index];
3234
return items[index];
3335
}
3436

‎src/TensorFlowNET.Keras/Layers/Normalization/BatchNormalization.cs‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -214,10 +214,10 @@ private Tensor _fused_batch_norm(Tensor inputs, Tensor training)
214214
{
215215
return tf.nn.fused_batch_norm(
216216
inputs,
217-
gamma,
218-
beta,
219-
mean: moving_mean,
220-
variance: moving_variance,
217+
gamma.AsTensor(),
218+
beta.AsTensor(),
219+
mean: moving_mean.AsTensor(),
220+
variance: moving_variance.AsTensor(),
221221
epsilon: epsilon,
222222
is_training: true,
223223
data_format: _data_format,
@@ -228,10 +228,10 @@ private Tensor _fused_batch_norm(Tensor inputs, Tensor training)
228228
{
229229
return tf.nn.fused_batch_norm(
230230
inputs,
231-
gamma,
232-
beta,
233-
mean: moving_mean,
234-
variance: moving_variance,
231+
gamma.AsTensor(),
232+
beta.AsTensor(),
233+
mean: moving_mean.AsTensor(),
234+
variance: moving_variance.AsTensor(),
235235
epsilon: epsilon,
236236
is_training: false,
237237
data_format: _data_format);

‎src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,14 +101,21 @@ public override Shape ComputeOutputShape(Shape input_shape)
101101

102102
protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null)
103103
{
104-
Tensors outputs = null;
104+
Tensor outputs = null;
105105
var inputs_dtype = inputs.dtype.as_base_dtype();
106106
var input_shape = inputs.shape;
107107
var ndims = len(input_shape);
108108
var broadcast_shape = range(ndims).Select(x => 1).ToArray();
109109
foreach (var dim in axis)
110110
broadcast_shape[dim] = input_shape.as_int_list()[dim];
111111

112+
Func<IVariableV1, Tensor> _broadcast = v =>
113+
{
114+
if (v.shape.ndim != ndims && !axis.SequenceEqual(new int[] { ndims - 1 }))
115+
return tf.reshape(v.AsTensor(), broadcast_shape);
116+
return v.AsTensor();
117+
};
118+
112119
if (_fused)
113120
{
114121
var tensor_shape = tf.shape(inputs);
@@ -127,18 +134,28 @@ protected override Tensors Call(Tensors inputs, Tensor state = null, bool? train
127134
var scale = tf.ones(new Shape((int)pre_dim), dtype: DType);
128135
var offset = tf.zeros(new Shape((int)pre_dim), dtype: DType);
129136

130-
/*outputs = tf.nn.fused_batch_norm(
137+
outputs = tf.nn.fused_batch_norm(
131138
inputs,
132139
scale: scale,
133140
offset: offset,
134141
epsilon: epsilon,
135-
data_format: "NCHW");*/
142+
data_format: "NCHW")[0];
143+
144+
outputs = tf.reshape(outputs, tensor_shape);
145+
146+
(scale, offset) = (_broadcast(gamma), _broadcast(beta));
147+
148+
outputs = outputs * tf.cast(scale, outputs.dtype);
149+
outputs = outputs + tf.cast(offset, outputs.dtype);
136150
}
137151
else
138152
{
139153

140154
}
141155

156+
// If some components of the shape got lost due to adjustments, fix that.
157+
outputs.shape = input_shape;
158+
142159
return outputs;
143160
}
144161
}

‎test/TensorFlowNET.Graph.UnitTest/MultithreadingTests.cs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,6 @@ public void SessionRun()
152152
//the core method
153153
void Core(int tid)
154154
{
155-
Assert.IsNull(tf.peak_default_graph());
156155
//graph is created automatically to perform create these operations
157156
var a1 = tf.constant(new[] { 2f }, shape: new[] { 1 });
158157
var a2 = tf.constant(new[] { 3f }, shape: new[] { 1 });

‎test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using Tensorflow.Keras;
66
using static Tensorflow.Binding;
77
using static Tensorflow.KerasApi;
8+
using System.Linq;
89

910
namespace TensorFlowNET.Keras.UnitTest
1011
{
@@ -86,7 +87,7 @@ public void Embedding_Simple()
8687
var emb = keras.layers.Embedding(256, 12, input_length: 4);
8788
var input_array = np.arange(12).reshape((3, 4)).astype(np.float32);
8889
var output = emb.Apply(input_array);
89-
Assert.AreEqual(new Shape(3, 4, 12), output.shape);
90+
Assert.AreEqual((3, 4, 12), output.shape);
9091
}
9192

9293
/// <summary>
@@ -159,7 +160,8 @@ public void LayerNormalization()
159160
var inputs = tf.constant(np.arange(10).reshape((5, 2)) * 10, dtype: tf.float32);
160161
var layer = keras.layers.LayerNormalization(axis: 1);
161162
var output = layer.Apply(inputs);
162-
// Assert.AreEqual((10, 16, 16, 3), output.shape);
163+
Assert.AreEqual((5, 2), output.shape);
164+
Assert.IsTrue(output[0].numpy().Equals(new[] { -0.99998f, 0.99998f }));
163165
}
164166
}
165167
}

0 commit comments

Comments
 (0)