Skip to content

Commit 1dd95bd

Browse files
Oceania2018Oceania2018
committed
skip gradient when no grad_func found.
1 parent 1deaa75 commit 1dd95bd

11 files changed

Lines changed: 272 additions & 16 deletions

File tree

‎src/TensorFlowNET.Core/Eager/EagerRunner.RecordGradient.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,9 @@ BackwardFunction GetGradientFunction(string op_name,
9191
Tensor[] op_outputs)
9292
=> (output_grads, unneeded_gradients) =>
9393
{
94+
if (ops.gradientFunctions[op_name] == null)
95+
return new Tensor[op_inputs.Length];
96+
9497
var gradients = ops.gradientFunctions[op_name](new EagerOperation
9598
{
9699
Name = op_name,

‎src/TensorFlowNET.Core/Gradients/OpTapeEntry.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,5 +15,7 @@ public class OpTapeEntry<BackwardFunction, TapeTensor>
1515
public TapeTensor[] output_tensor_info { get; set; }
1616
public long[] input_tensor_id { get; set; }
1717
public BackwardFunction backward_function { get; set; }
18+
public override string ToString()
19+
=> $"{op_type}, inputs: {string.Join(",", input_tensor_id)}";
1820
}
1921
}

‎src/TensorFlowNET.Core/Gradients/Tape.ComputeGradient.cs‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,13 @@ public Tensor[] ComputeGradient(long[] target_tensor_ids,
2929
tensor_tape_,
3030
state.op_tape);
3131

32-
while (op_stack.Count > 0)
32+
while (!op_stack.empty())
3333
{
3434
var op = op_stack.Dequeue();
3535
if (!state.op_tape.find(op, out var trace))
3636
continue;
3737

38+
Console.WriteLine($"ComputeGradient: {state.op_tape[op].op_type}");
3839
state.op_tape.erase(op);
3940

4041
var out_gradients = new List<Tensor>(trace.output_tensor_info.Length);
@@ -103,7 +104,7 @@ public Tensor[] ComputeGradient(long[] target_tensor_ids,
103104
}
104105
else
105106
{
106-
throw new NotImplementedException("");
107+
in_gradients = new Tensor[trace.input_tensor_id.Length];
107108
}
108109

109110
for (int i = 0; i < in_gradients.Length; ++i)
@@ -113,17 +114,18 @@ public Tensor[] ComputeGradient(long[] target_tensor_ids,
113114
{
114115
var unaggregated_grads = gradients[id];
115116
unaggregated_grads.Add(in_gradients[i]);
116-
if(unaggregated_grads.Count > kMinAggregateCount)
117+
if (unaggregated_grads.Count > kMinAggregateCount)
117118
{
118-
if(!gradients_size.ContainsKey(id))
119+
if (!gradients_size.find(id, out var size))
119120
{
121+
size = (long)unaggregated_grads[0].size;
122+
gradients_size.emplace(id, size);
120123
}
121-
else
122-
{
123124

125+
if (unaggregated_grads.Count * size * 4 > kMinAggregateBytes)
126+
{
127+
throw new NotImplementedException("");
124128
}
125-
126-
throw new NotImplementedException("");
127129
}
128130
}
129131

‎src/TensorFlowNET.Core/Keras/ArgsDefinition/RMSpropArgs.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Text;
4+
5+
namespace Tensorflow.Keras.ArgsDefinition
6+
{
7+
public class RMSpropArgs
8+
{
9+
public float LearningRate { get; set; } = 0.001f;
10+
public float RHO { get; set; } = 0.9f;
11+
public float Momentum { get; set; } = 0.0f;
12+
public float Epsilon { get; set; } = 1e-7f;
13+
public bool Centered { get; set; } = false;
14+
public string Name { get; set; } = "RMSprop";
15+
}
16+
}

‎src/TensorFlowNET.Core/Keras/Engine/Functional.cs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,6 @@ public class Functional : Model
2323
List<KerasHistory> _input_coordinates;
2424
List<KerasHistory> _output_coordinates;
2525
public string[] NetworkNodes { get; set; }
26-
public Dictionary<int, List<Node>> NodesByDepth { get; set; }
27-
public List<Layer> Layers => _layers;
2826

2927
Dictionary<int, int> tensor_usage_count;
3028
public Dictionary<int, int> TensorUsageCount => tensor_usage_count;
@@ -43,9 +41,10 @@ public override List<IVariableV1> trainable_variables
4341
}
4442
}
4543

46-
public Functional(Tensors inputs, Tensors outputs)
44+
public Functional(Tensors inputs, Tensors outputs, string name = null)
4745
: base(new ModelArgs
4846
{
47+
Name = name,
4948
Inputs = inputs,
5049
Outputs = outputs
5150
})

‎src/TensorFlowNET.Core/Keras/Engine/Model.cs‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ namespace Tensorflow.Keras.Engine
1010
/// <summary>
1111
/// `Model` groups layers into an object with training and inference features.
1212
/// </summary>
13-
public class Model : Layer
13+
public partial class Model : Layer
1414
{
1515
#pragma warning disable CS0169 // The field 'Model._cloning' is never used
1616
bool _cloning;
@@ -33,12 +33,20 @@ public Model(ModelArgs args)
3333

3434
}
3535

36+
public void compile(ILossFunc loss, OptimizerV2 optimizer, string[] metrics)
37+
{
38+
39+
}
40+
3641
public void compile(string optimizerName, string lossName)
3742
{
3843
switch (optimizerName)
3944
{
4045
case "rmsprop":
41-
optimizer = new RMSprop();
46+
optimizer = new RMSprop(new RMSpropArgs
47+
{
48+
49+
});
4250
break;
4351
}
4452

‎src/TensorFlowNET.Core/Keras/Engine/Node.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ namespace Tensorflow.Keras.Engine
3030
/// Each time the output of a layer is used by another layer,
3131
/// a node is added to `layer._outbound_nodes`.
3232
/// </summary>
33-
public class Node
33+
public partial class Node
3434
{
3535
NodeArgs args;
3636

‎src/TensorFlowNET.Core/Keras/KerasApi.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ public Sequential Sequential(List<Layer> layers = null,
3939
/// <param name="input"></param>
4040
/// <param name="output"></param>
4141
/// <returns></returns>
42-
public Functional Model(Tensors inputs, Tensors outputs)
43-
=> new Functional(inputs, outputs);
42+
public Functional Model(Tensors inputs, Tensors outputs, string name = null)
43+
=> new Functional(inputs, outputs, name: name);
4444

4545
/// <summary>
4646
/// Instantiate a Keras tensor.

‎src/TensorFlowNET.Core/Keras/Optimizers/OptimizerApi.cs‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System;
22
using System.Collections.Generic;
33
using System.Text;
4+
using Tensorflow.Keras.ArgsDefinition;
45

56
namespace Tensorflow.Keras.Optimizers
67
{
@@ -29,5 +30,31 @@ public OptimizerV2 Adam(float learning_rate = 0.001f,
2930
epsilon: epsilon,
3031
amsgrad: amsgrad,
3132
name: name);
33+
34+
/// <summary>
35+
/// Construct a new RMSprop optimizer.
36+
/// </summary>
37+
/// <param name="learning_rate"></param>
38+
/// <param name="rho"></param>
39+
/// <param name="momentum"></param>
40+
/// <param name="epsilon"></param>
41+
/// <param name="centered"></param>
42+
/// <param name="name"></param>
43+
/// <returns></returns>
44+
public OptimizerV2 RMSprop(float learning_rate = 0.001f,
45+
float rho = 0.9f,
46+
float momentum = 0.0f,
47+
float epsilon = 1e-7f,
48+
bool centered = false,
49+
string name = "RMSprop")
50+
=> new RMSprop(new RMSpropArgs
51+
{
52+
LearningRate = learning_rate,
53+
RHO = rho,
54+
Momentum = momentum,
55+
Epsilon = epsilon,
56+
Centered = centered,
57+
Name = name
58+
});
3259
}
3360
}

‎src/TensorFlowNET.Core/Keras/Optimizers/RMSprop.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System;
22
using System.Collections.Generic;
33
using System.Text;
4+
using Tensorflow.Keras.ArgsDefinition;
45

56
namespace Tensorflow.Keras.Optimizers
67
{
@@ -9,6 +10,11 @@ namespace Tensorflow.Keras.Optimizers
910
/// </summary>
1011
public class RMSprop : OptimizerV2
1112
{
13+
RMSpropArgs args;
1214

15+
public RMSprop(RMSpropArgs args)
16+
{
17+
this.args = args;
18+
}
1319
}
1420
}

0 commit comments

Comments
 (0)