add overload for Layer call function, be able to input array and return array.

Oceania2018 · Oceania2018 · commit 5f2df567a78a · 2020-09-20T08:56:12.000-05:00
diff --git a/src/TensorFlowNET.Console/MemoryTestingCases.cs b/src/TensorFlowNET.Console/MemoryTestingCases.cs
@@ -1,6 +1,7 @@
 ﻿using System;
 using System.Collections.Generic;
 using System.Text;
+using NumSharp;
 using static Tensorflow.Binding;
 
 namespace Tensorflow
@@ -18,6 +19,22 @@ public Action<int> Constant
                     var tensor = tf.constant(3112.0f);
                 }
             };
+
+        public Action<int> Constant2x3
+            => (iterate) =>
+            {
+                var nd = np.array(new byte[,]
+                {
+                    {1, 2, 3},
+                    {4, 5, 6}
+                });
+                for (int i = 0; i < iterate; i++)
+                {
+                    var tensor = tf.constant(nd);
+                    var data = tensor.numpy();
+                }
+            };
+
         public Action<int> Variable
             => (iterate) =>
             {
diff --git a/src/TensorFlowNET.Console/Program.cs b/src/TensorFlowNET.Console/Program.cs
@@ -15,6 +15,9 @@ static void Main(string[] args)
 
             int batchSize = 1000;
 
+            // explaination of constant
+            mm.Execute(10, 100 * batchSize, cases.Constant2x3);
+
             // 1 million float tensor 68M.
             mm.Execute(10, 100 * batchSize, cases.Constant);
 
diff --git a/src/TensorFlowNET.Core/APIs/c_api.cs b/src/TensorFlowNET.Core/APIs/c_api.cs
@@ -43,7 +43,7 @@ namespace Tensorflow
     /// </summary>
     public partial class c_api
     {
-        public const string TensorFlowLibName = "tensorflow";
+        public const string TensorFlowLibName = @"C:\Users\haipi\Documents\Projects\tensorflow\bazel-bin\tensorflow\tensorflow";
 
         public static string StringPiece(IntPtr handle)
         {
diff --git a/src/TensorFlowNET.Core/Eager/EagerTensor.Creation.cs b/src/TensorFlowNET.Core/Eager/EagerTensor.Creation.cs
@@ -70,8 +70,8 @@ protected override void DisposeManagedResources()
 
         protected override void DisposeUnmanagedResources(IntPtr handle)
         {
+            base.DisposeUnmanagedResources(handle);
             //print($"deleting DeleteTensorHandle {Id} {_handle.ToString("x16")}");
-            c_api.TF_DeleteTensor(_handle);
         }
     }
 }
diff --git a/src/TensorFlowNET.Core/Gradients/gradients_util.cs b/src/TensorFlowNET.Core/Gradients/gradients_util.cs
@@ -311,6 +311,7 @@ private static (Operation[], Dictionary<string, int>, ControlFlowState) _Pending
             while (queue.Count > 0)
             {
                 var op = queue.Dequeue();
+
                 if (reached_ops.Contains(op))
                 {
                     between_ops.Add(op);
diff --git a/src/TensorFlowNET.Core/Graphs/Graph.cs b/src/TensorFlowNET.Core/Graphs/Graph.cs
@@ -278,7 +278,11 @@ public Operation create_op(string op_type, Tensor[] inputs, TF_DataType[] dtypes
             // after removing the trailing '/'.
             name = name.EndsWith("/") ? ops.name_from_scope_name(name) : unique_name(name);
             var node_def = ops._NodeDef(op_type, name, attrs: attrs);
+            if (name == "rnn/while/basic_rnn_cell/MatMul" 
+                || name == "rnn/while/basic_rnn_cell/MatMul/Enter")
+            {
 
+            }
             var input_ops = inputs.Select(x => x.op).ToArray();
             var control_inputs = _control_dependencies_for_inputs(input_ops);
 
diff --git a/src/TensorFlowNET.Core/Keras/Engine/Flatten.cs b/src/TensorFlowNET.Core/Keras/Engine/Flatten.cs
@@ -21,7 +21,7 @@ public Flatten(FlattenArgs args)
             _channels_first = args.DataFormat == "channels_first";
         }
 
-        protected override Tensor call(Tensor inputs, bool is_training = false, Tensor state = null)
+        protected override Tensor call(Tensor inputs, bool is_training = false)
         {
             if (_channels_first)
             {
diff --git a/src/TensorFlowNET.Core/Keras/Engine/Layer.cs b/src/TensorFlowNET.Core/Keras/Engine/Layer.cs
@@ -121,7 +121,7 @@ public Layer(LayerArgs args)
         /// <param name="input"></param>
         /// <param name="is_training"></param>
         /// <returns></returns>
-        public Tensor Apply(Tensor inputs, bool is_training = false, Tensor state = null)
+        public Tensor Apply(Tensor inputs, bool is_training = false)
         {
             Tensor outputs = null;
 
@@ -148,7 +148,7 @@ public Tensor Apply(Tensor inputs, bool is_training = false, Tensor state = null
                 if (!built)
                     MaybeBuild(inputs);
 
-                outputs = call(inputs, is_training: is_training, state: state);
+                outputs = call(inputs, is_training: is_training);
 
                 outputs = _set_connectivity_metadata_(inputs, outputs);
                 _handle_activity_regularization(inputs, outputs);
@@ -161,6 +161,35 @@ public Tensor Apply(Tensor inputs, bool is_training = false, Tensor state = null
             return outputs;
         }
 
+        public Tensor[] Apply(Tensor[] inputs, Tensor state, bool is_training = false)
+        {
+            Tensor[] outputs = null;
+
+            callContext = callContext ?? new ThreadLocal<CallContext>()
+            {
+                Value = new CallContext()
+            };
+
+            var eager = tf.executing_eagerly();
+            using var ctxManager = CallContext.enter();
+
+            string nameScope = "";
+            if (eager)
+                nameScope = name;
+            else
+                nameScope = _name_scope();
+
+            tf_with(ops.name_scope(nameScope), scope =>
+            {
+                if (!built)
+                    MaybeBuild(inputs[0]);
+
+                outputs = call(inputs, is_training: is_training, state: state);
+            });
+
+            return outputs;
+        }
+
         private Tensor _set_connectivity_metadata_(Tensor inputs, Tensor outputs)
         {
             /*var returnOutputs = new List<Tensor>();
@@ -200,7 +229,12 @@ private Tensor compute_mask(Tensor inputs, Tensor mask = null)
             return null;
         }
 
-        protected virtual Tensor call(Tensor inputs, bool is_training = false, Tensor state = null)
+        protected virtual Tensor call(Tensor inputs, bool is_training = false)
+        {
+            throw new NotImplementedException("");
+        }
+
+        protected virtual Tensor[] call(Tensor[] inputs, Tensor state, bool is_training = false)
         {
             throw new NotImplementedException("");
         }
diff --git a/src/TensorFlowNET.Core/Keras/Layers/BatchNormalization.cs b/src/TensorFlowNET.Core/Keras/Layers/BatchNormalization.cs
@@ -143,7 +143,7 @@ protected override void build(TensorShape input_shape)
             built = true;
         }
 
-        protected override Tensor call(Tensor inputs, bool is_training = false, Tensor state = null)
+        protected override Tensor call(Tensor inputs, bool is_training = false)
         {
             Tensor outputs = null;
 
diff --git a/src/TensorFlowNET.Core/Keras/Layers/Conv.cs b/src/TensorFlowNET.Core/Keras/Layers/Conv.cs
@@ -95,7 +95,7 @@ protected override void build(TensorShape input_shape)
             built = true;
         }
 
-        protected override Tensor call(Tensor inputs, bool training = false, Tensor state = null)
+        protected override Tensor call(Tensor inputs, bool training = false)
         {
             var outputs = _convolution_op.__call__(inputs, kernel);
             if (use_bias)
diff --git a/src/TensorFlowNET.Core/Keras/Layers/Dense.cs b/src/TensorFlowNET.Core/Keras/Layers/Dense.cs
@@ -65,7 +65,7 @@ protected override void build(TensorShape input_shape)
             built = true;
         }
 
-        protected override Tensor call(Tensor inputs, bool training = false, Tensor state = null)
+        protected override Tensor call(Tensor inputs, bool training = false)
         {
             Tensor outputs = null;
             var rank = inputs.rank;
diff --git a/src/TensorFlowNET.Core/Keras/Layers/Dropout.cs b/src/TensorFlowNET.Core/Keras/Layers/Dropout.cs
@@ -18,7 +18,7 @@ public Dropout(DropoutArgs args)
             this.args = args;
         }
 
-        protected override Tensor call(Tensor inputs, bool is_training = false, Tensor state = null)
+        protected override Tensor call(Tensor inputs, bool is_training = false)
         {
             var output = tf_utils.smart_cond(is_training,
                 () => tf.nn.dropout(inputs,
diff --git a/src/TensorFlowNET.Core/Keras/Layers/Embedding.cs b/src/TensorFlowNET.Core/Keras/Layers/Embedding.cs
@@ -62,7 +62,7 @@ protected override void build(TensorShape input_shape)
             built = true;
         }
 
-        protected override Tensor call(Tensor inputs, bool is_training = false, Tensor state = null)
+        protected override Tensor call(Tensor inputs, bool is_training = false)
         {
             var dtype = inputs.dtype;
             if (dtype != tf.int32 && dtype != tf.int64)
diff --git a/src/TensorFlowNET.Core/Keras/Layers/LSTM.cs b/src/TensorFlowNET.Core/Keras/Layers/LSTM.cs
@@ -29,9 +29,9 @@ public LSTM(LSTMArgs args) :
                 .ToArray();
         }
 
-        protected override Tensor call(Tensor inputs, bool is_training = false, Tensor state = null)
+        protected override Tensor call(Tensor inputs, bool is_training = false)
         {
-            return base.call(inputs, is_training, state);
+            return base.call(inputs, is_training);
         }
     }
 }
diff --git a/src/TensorFlowNET.Core/Keras/Layers/Pooling2D.cs b/src/TensorFlowNET.Core/Keras/Layers/Pooling2D.cs
@@ -36,7 +36,7 @@ public Pooling2D(Pooling2DArgs args)
             input_spec = new InputSpec(ndim: 4);
         }
 
-        protected override Tensor call(Tensor inputs, bool is_training = false, Tensor state = null)
+        protected override Tensor call(Tensor inputs, bool is_training = false)
         {
             int[] pool_shape;
             int[] strides;
diff --git a/src/TensorFlowNET.Core/Keras/Layers/Rescaling.cs b/src/TensorFlowNET.Core/Keras/Layers/Rescaling.cs
@@ -20,7 +20,7 @@ public Rescaling(RescalingArgs args) : base(args)
             this.args = args;
         }
 
-        protected override Tensor call(Tensor inputs, bool is_training = false, Tensor state = null)
+        protected override Tensor call(Tensor inputs, bool is_training = false)
         {
             scale = math_ops.cast(args.Scale, args.DType);
             offset = math_ops.cast(args.Offset, args.DType);
diff --git a/src/TensorFlowNET.Core/Layers/Layer.cs b/src/TensorFlowNET.Core/Layers/Layer.cs
@@ -61,9 +61,8 @@ public virtual (Tensor, Tensor) apply(Tensor inputs, Tensor training = null)
             return (results[0], results[1]);
         }
 
-        public Tensor[] __call__(Tensor inputs,
+        public Tensor __call__(Tensor inputs,
             Tensor training = null,
-            Tensor state = null,
             VariableScope scope = null)
         {
             _set_scope(scope);
@@ -88,16 +87,54 @@ public Tensor[] __call__(Tensor inputs,
             {
                 _current_scope = scope2;
                 // Actually call layer
-                outputs = base.Apply(inputs, 
-                    is_training: training == null ? false : false,
-                    state: state);
+                outputs = base.Apply(inputs[0], 
+                    is_training: training == null ? false : false);
+            });
+
+
+            // Update global default collections.
+            _add_elements_to_collection(updates.ToArray(), new string[] { tf.GraphKeys.UPDATE_OPS });
+
+            return outputs;
+        }
+
+        public Tensor[] __call__(Tensor[] inputs,
+            Tensor state = null,
+            Tensor training = null,
+            VariableScope scope = null)
+        {
+            _set_scope(scope);
+            _graph = ops._get_graph_from_inputs(inputs, graph: _graph);
+
+            variable_scope scope_context_manager = null;
+            if (built)
+            {
+                scope_context_manager = tf.variable_scope(_scope,
+                    reuse: true,
+                    auxiliary_name_scope: false);
+            }
+            else
+            {
+                scope_context_manager = tf.variable_scope(_scope,
+                    reuse: _reuse,
+                    auxiliary_name_scope: false);
+            }
+
+            Tensor[] outputs = null;
+            tf_with(scope_context_manager, scope2 =>
+            {
+                _current_scope = scope2;
+                // Actually call layer
+                outputs = base.Apply(inputs,
+                    state,
+                    is_training: training == null ? false : false);
             });
 
 
             // Update global default collections.
             _add_elements_to_collection(updates.ToArray(), new string[] { tf.GraphKeys.UPDATE_OPS });
 
-            return new Tensor[] { outputs };
+            return outputs;
         }
 
         protected virtual void _add_elements_to_collection(Operation[] elements, string[] collection_list)
diff --git a/src/TensorFlowNET.Core/Operations/ControlFlows/WhileContext.cs b/src/TensorFlowNET.Core/Operations/ControlFlows/WhileContext.cs
@@ -326,7 +326,7 @@ private void _InitializeValues(Tensor[] values)
 
         protected override void _AddOpInternal(Operation op)
         {
-            if (op.name == "gradients/rnn/while/basic_rnn_cell/Tanh_grad/TanhGrad")
+            if (op.name == "rnn/basic_rnn_cell/kernel/Initializer/random_uniform/shape")
             {
 
             }
diff --git a/src/TensorFlowNET.Core/Operations/NnOps/BasicLSTMCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/BasicLSTMCell.cs
@@ -61,7 +61,7 @@ protected override void build(TensorShape input_shape)
             built = true;
         }
 
-        public Tensor[] __call__(Tensor inputs, LSTMStateTuple state)
+        public Tensor __call__(Tensor inputs, LSTMStateTuple state)
         {
             _state = state;
             return base.__call__(inputs);
@@ -74,7 +74,7 @@ public Tensor[] __call__(Tensor inputs, LSTMStateTuple state)
         /// <param name="training"></param>
         /// <param name="state"></param>
         /// <returns></returns>
-        protected override Tensor call(Tensor inputs, bool is_training = false, Tensor state = null)
+        protected override Tensor call(Tensor inputs, bool is_training = false)
         {
             var one = constant_op.constant(1, dtype: dtypes.int32);
             // Parameters of gates are concatenated into one multiply for efficiency.
diff --git a/src/TensorFlowNET.Core/Operations/NnOps/BasicRNNCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/BasicRNNCell.cs
@@ -67,14 +67,14 @@ protected override void build(TensorShape inputs_shape)
             built = true;
         }
 
-        protected override Tensor call(Tensor inputs, bool is_training = false, Tensor state = null)
+        protected override Tensor[] call(Tensor[] inputs, Tensor state, bool is_training = false)
         {
             // Most basic RNN: output = new_state = act(W * input + U * state + B).
-            var concat = array_ops.concat(new[] { inputs, state }, 1);
+            var concat = array_ops.concat(new[] { inputs[0], state }, 1);
             var gate_inputs = math_ops.matmul(concat, _kernel.AsTensor());
             gate_inputs = nn_ops.bias_add(gate_inputs, _bias.AsTensor());
             var output = _activation(gate_inputs, null);
-            return output;
+            return new[] { output, output };
         }
     }
 }
diff --git a/src/TensorFlowNET.Core/Operations/NnOps/rnn.cs b/src/TensorFlowNET.Core/Operations/NnOps/rnn.cs
@@ -364,7 +364,7 @@ private static (Tensor, Tensor) _dynamic_rnn_loop(RnnCell cell, Tensor inputs, T
                 if (sequence_length != null)
                     throw new NotImplementedException("sequence_length != null");
                 else
-                    outputs = cell.__call__(input_t_t, state: state1);
+                    outputs = cell.__call__(new[] { input_t_t }, state: state1);
 
                 var (output, new_state) = (outputs[0], outputs[1]);
                 // Keras cells always wrap state as list, even if it's a single tensor.
diff --git a/src/TensorFlowNET.Core/Operations/Operation.cs b/src/TensorFlowNET.Core/Operations/Operation.cs
@@ -326,7 +326,7 @@ public void _update_input(int index, Tensor tensor)
             // the updated inputs are reloaded from the c_api
             lock (Locks.ProcessWide)
             {
-                // c_api.UpdateEdge(_graph, output, input, tf.Status.Handle);
+                c_api.UpdateEdge(_graph, output, input, tf.Status.Handle);
                 //var updated_inputs = inputs;
                 tf.Status.Check();
             }
diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj
@@ -5,7 +5,7 @@
     <AssemblyName>TensorFlow.NET</AssemblyName>
     <RootNamespace>Tensorflow</RootNamespace>
     <TargetTensorFlow>2.2.0</TargetTensorFlow>
-    <Version>0.20.0</Version>
+    <Version>0.20.1</Version>
     <LangVersion>8.0</LangVersion>
     <Authors>Haiping Chen, Meinrad Recheis, Eli Belash</Authors>
     <Company>SciSharp STACK</Company>
@@ -19,13 +19,13 @@
     <Description>Google's TensorFlow full binding in .NET Standard.
 Building, training and infering deep learning models.
 https://tensorflownet.readthedocs.io</Description>
-    <AssemblyVersion>0.20.0.0</AssemblyVersion>
+    <AssemblyVersion>0.20.1.0</AssemblyVersion>
     <PackageReleaseNotes>tf.net 0.20.x and above are based on tensorflow native 2.x.
 
 * Eager Mode is added finally.
 * tf.keras is partially working. 
 * tf.data is added.</PackageReleaseNotes>
-    <FileVersion>0.20.0.0</FileVersion>
+    <FileVersion>0.20.1.0</FileVersion>
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
     <PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>
     <SignAssembly>true</SignAssembly>
diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs b/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs
@@ -50,6 +50,8 @@ public partial class Tensor
         /// </summary>
         public AllocationType AllocationType { get; protected set; }
 
+        public IntPtr TensorDataPointer => TF_TensorData(_handle);
+
         /// <summary>
         ///     Create a Tensor object from an existing TF handle
         /// </summary>
diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.cs b/src/TensorFlowNET.Core/Tensors/Tensor.cs
diff --git a/src/TensorFlowNET.Core/Variables/ResourceVariable.cs b/src/TensorFlowNET.Core/Variables/ResourceVariable.cs
diff --git a/src/TensorFlowNet.Benchmarks/Tensorflow.Benchmark.csproj b/src/TensorFlowNet.Benchmarks/Tensorflow.Benchmark.csproj
diff --git a/test/TensorFlowNET.UnitTest/Tensorflow.UnitTest.csproj b/test/TensorFlowNET.UnitTest/Tensorflow.UnitTest.csproj

Original file line number	Diff line number	Diff line change
`@@ -43,7 +43,7 @@ namespace Tensorflow`
`43`	`43`	`/// </summary>`
`44`	`44`	`public partial class c_api`
`45`	`45`	`{`
`46`		`- public const string TensorFlowLibName = "tensorflow";`
	`46`	`+ public const string TensorFlowLibName = @"C:\Users\haipi\Documents\Projects\tensorflow\bazel-bin\tensorflow\tensorflow";`
`47`	`47`
`48`	`48`	`public static string StringPiece(IntPtr handle)`
`49`	`49`	`{`
Original file line number	Diff line number	Diff line change
`@@ -70,8 +70,8 @@ protected override void DisposeManagedResources()`
`70`	`70`
`71`	`71`	`protected override void DisposeUnmanagedResources(IntPtr handle)`
`72`	`72`	`{`
	`73`	`+ base.DisposeUnmanagedResources(handle);`
`73`	`74`	`//print($"deleting DeleteTensorHandle {Id} {_handle.ToString("x16")}");`
`74`		`- c_api.TF_DeleteTensor(_handle);`
`75`	`75`	`}`
`76`	`76`	`}`
`77`	`77`	`}`
Original file line number	Diff line number	Diff line change
`@@ -311,6 +311,7 @@ private static (Operation[], Dictionary<string, int>, ControlFlowState) _Pending`
`311`	`311`	`while (queue.Count > 0)`
`312`	`312`	`{`
`313`	`313`	`var op = queue.Dequeue();`
	`314`	`+`
`314`	`315`	`if (reached_ops.Contains(op))`
`315`	`316`	`{`
`316`	`317`	`between_ops.Add(op);`
Original file line number	Diff line number	Diff line change
`@@ -21,7 +21,7 @@ public Flatten(FlattenArgs args)`
`21`	`21`	`_channels_first = args.DataFormat == "channels_first";`
`22`	`22`	`}`
`23`	`23`
`24`		`- protected override Tensor call(Tensor inputs, bool is_training = false, Tensor state = null)`
	`24`	`+ protected override Tensor call(Tensor inputs, bool is_training = false)`
`25`	`25`	`{`
`26`	`26`	`if (_channels_first)`
`27`	`27`	`{`
Original file line number	Diff line number	Diff line change
`@@ -143,7 +143,7 @@ protected override void build(TensorShape input_shape)`
`143`	`143`	`built = true;`
`144`	`144`	`}`
`145`	`145`
`146`		`- protected override Tensor call(Tensor inputs, bool is_training = false, Tensor state = null)`
	`146`	`+ protected override Tensor call(Tensor inputs, bool is_training = false)`
`147`	`147`	`{`
`148`	`148`	`Tensor outputs = null;`
`149`	`149`
Original file line number	Diff line number	Diff line change
`@@ -95,7 +95,7 @@ protected override void build(TensorShape input_shape)`
`95`	`95`	`built = true;`
`96`	`96`	`}`
`97`	`97`
`98`		`- protected override Tensor call(Tensor inputs, bool training = false, Tensor state = null)`
	`98`	`+ protected override Tensor call(Tensor inputs, bool training = false)`
`99`	`99`	`{`
`100`	`100`	`var outputs = _convolution_op.__call__(inputs, kernel);`
`101`	`101`	`if (use_bias)`