From 9412729ca9968d05d12051b8e8542e2504ba2f6c Mon Sep 17 00:00:00 2001 From: Steven Date: Sun, 19 Jun 2022 01:16:50 -0700 Subject: [PATCH 01/98] Fix Conv2dTranspose bias Conv2dTranspose defaults to have use_bias = true but currently throws a not implemented exception when the parameter is true. --- src/TensorFlowNET.Keras/Layers/LayersApi.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index aa4f416f6..548e3ff95 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -235,7 +235,7 @@ public Conv2DTranspose Conv2DTranspose(int filters, string data_format = null, Shape dilation_rate = null, string activation = null, - bool use_bias = true, + bool use_bias = false, string kernel_initializer = null, string bias_initializer = null, string kernel_regularizer = null, From aac52940ade5c788bc7d8d6949da718b63293dc1 Mon Sep 17 00:00:00 2001 From: lingbai-kong Date: Fri, 23 Jun 2023 13:17:46 +0800 Subject: [PATCH 02/98] init pickle support to np.load object type of npy --- .../NumPy/DtypeConstructor.cs | 40 ++++++++++++ .../Implementation/NumPyImpl.Creation.cs | 18 +++++- .../NumPy/Implementation/NumPyImpl.load.cs | 22 +++++-- .../NumPy/MultiArrayConstructor.cs | 44 +++++++++++++ .../NumPy/NDArray.Pickle.cs | 19 ++++++ .../Tensorflow.Binding.csproj | 1 + src/TensorFlowNET.Keras/Datasets/Imdb.cs | 63 +++++++++++++++++-- .../Dataset/DatasetTest.cs | 17 +++++ 8 files changed, 215 insertions(+), 9 deletions(-) create mode 100644 src/TensorFlowNET.Core/NumPy/DtypeConstructor.cs create mode 100644 src/TensorFlowNET.Core/NumPy/MultiArrayConstructor.cs create mode 100644 src/TensorFlowNET.Core/NumPy/NDArray.Pickle.cs diff --git a/src/TensorFlowNET.Core/NumPy/DtypeConstructor.cs b/src/TensorFlowNET.Core/NumPy/DtypeConstructor.cs new file mode 100644 index 000000000..f84f408e1 --- /dev/null +++ b/src/TensorFlowNET.Core/NumPy/DtypeConstructor.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Text; +using Razorvine.Pickle; + +namespace Tensorflow.NumPy +{ + /// + /// + /// + [SuppressMessage("ReSharper", "InconsistentNaming")] + [SuppressMessage("ReSharper", "MemberCanBePrivate.Global")] + [SuppressMessage("ReSharper", "MemberCanBeMadeStatic.Global")] + class DtypeConstructor : IObjectConstructor + { + public object construct(object[] args) + { + Console.WriteLine("DtypeConstructor"); + Console.WriteLine(args.Length); + for (int i = 0; i < args.Length; i++) + { + Console.WriteLine(args[i]); + } + return new demo(); + } + } + class demo + { + public void __setstate__(object[] args) + { + Console.WriteLine("demo __setstate__"); + Console.WriteLine(args.Length); + for (int i = 0; i < args.Length; i++) + { + Console.WriteLine(args[i]); + } + } + } +} diff --git a/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs b/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs index f29879b0f..80b62198a 100644 --- a/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs +++ b/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Text; using Tensorflow.Util; +using Razorvine.Pickle; using static Tensorflow.Binding; namespace Tensorflow.NumPy @@ -93,10 +94,25 @@ Array ReadValueMatrix(BinaryReader reader, Array matrix, int bytes, Type type, i var buffer = reader.ReadBytes(bytes * total); System.Buffer.BlockCopy(buffer, 0, matrix, 0, buffer.Length); - return matrix; } + NDArray ReadObjectMatrix(BinaryReader reader, Array matrix, int[] shape) + { + //int data = reader.ReadByte(); + //Console.WriteLine(data); + //Console.WriteLine(reader.ReadByte()); + Stream stream = reader.BaseStream; + Unpickler.registerConstructor("numpy.core.multiarray", "_reconstruct", new MultiArrayConstructor()); + Unpickler.registerConstructor("numpy", "dtype", new DtypeConstructor()); + + var unpickler = new Unpickler(); + + NDArray result = (NDArray) unpickler.load(stream); + Console.WriteLine(result.dims); + return result; + } + public (NDArray, NDArray) meshgrid(T[] array, bool copy = true, bool sparse = false) { var tensors = array_ops.meshgrid(array, copy: copy, sparse: sparse); diff --git a/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.load.cs b/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.load.cs index 05f53d5e7..789f119a1 100644 --- a/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.load.cs +++ b/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.load.cs @@ -27,9 +27,20 @@ public Array LoadMatrix(Stream stream) Array matrix = Array.CreateInstance(type, shape); //if (type == typeof(String)) - //return ReadStringMatrix(reader, matrix, bytes, type, shape); + //return ReadStringMatrix(reader, matrix, bytes, type, shape); + NDArray res = ReadObjectMatrix(reader, matrix, shape); + Console.WriteLine("LoadMatrix"); + Console.WriteLine(res.dims[0]); + Console.WriteLine((int)res[0][0]); + Console.WriteLine(res.dims[1]); + //if (type == typeof(Object)) + //{ + + //} + //else return ReadValueMatrix(reader, matrix, bytes, type, shape); } + } public T Load(Stream stream) @@ -37,7 +48,7 @@ public T Load(Stream stream) ICloneable, IList, ICollection, IEnumerable, IStructuralComparable, IStructuralEquatable { // if (typeof(T).IsArray && (typeof(T).GetElementType().IsArray || typeof(T).GetElementType() == typeof(string))) - // return LoadJagged(stream) as T; + // return LoadJagged(stream) as T; return LoadMatrix(stream) as T; } @@ -48,7 +59,7 @@ bool ParseReader(BinaryReader reader, out int bytes, out Type t, out int[] shape shape = null; // The first 6 bytes are a magic string: exactly "x93NUMPY" - if (reader.ReadChar() != 63) return false; + if (reader.ReadByte() != 0x93) return false; if (reader.ReadChar() != 'N') return false; if (reader.ReadChar() != 'U') return false; if (reader.ReadChar() != 'M') return false; @@ -64,6 +75,7 @@ bool ParseReader(BinaryReader reader, out int bytes, out Type t, out int[] shape ushort len = reader.ReadUInt16(); string header = new String(reader.ReadChars(len)); + Console.WriteLine(header); string mark = "'descr': '"; int s = header.IndexOf(mark) + mark.Length; int e = header.IndexOf("'", s + 1); @@ -93,7 +105,7 @@ bool ParseReader(BinaryReader reader, out int bytes, out Type t, out int[] shape Type GetType(string dtype, out int bytes, out bool? isLittleEndian) { isLittleEndian = IsLittleEndian(dtype); - bytes = Int32.Parse(dtype.Substring(2)); + bytes = dtype.Length > 2 ? Int32.Parse(dtype.Substring(2)) : 0; string typeCode = dtype.Substring(1); @@ -121,6 +133,8 @@ Type GetType(string dtype, out int bytes, out bool? isLittleEndian) return typeof(Double); if (typeCode.StartsWith("S")) return typeof(String); + if (typeCode == "O") + return typeof(Object); throw new NotSupportedException(); } diff --git a/src/TensorFlowNET.Core/NumPy/MultiArrayConstructor.cs b/src/TensorFlowNET.Core/NumPy/MultiArrayConstructor.cs new file mode 100644 index 000000000..92927cd5a --- /dev/null +++ b/src/TensorFlowNET.Core/NumPy/MultiArrayConstructor.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Text; +using Razorvine.Pickle; + +namespace Tensorflow.NumPy +{ + /// + /// Creates multiarrays of objects. Returns a primitive type multiarray such as int[][] if + /// the objects are ints, etc. + /// + [SuppressMessage("ReSharper", "InconsistentNaming")] + [SuppressMessage("ReSharper", "MemberCanBePrivate.Global")] + [SuppressMessage("ReSharper", "MemberCanBeMadeStatic.Global")] + public class MultiArrayConstructor : IObjectConstructor + { + public object construct(object[] args) + { + //Console.WriteLine(args.Length); + //for (int i = 0; i < args.Length; i++) + //{ + // Console.WriteLine(args[i]); + //} + Console.WriteLine("MultiArrayConstructor"); + + var arg1 = (Object[])args[1]; + var dims = new int[arg1.Length]; + for (var i = 0; i < arg1.Length; i++) + { + dims[i] = (int)arg1[i]; + } + + var dtype = TF_DataType.DtInvalid; + switch (args[2]) + { + case "b": dtype = TF_DataType.DtUint8Ref; break; + default: throw new NotImplementedException("cannot parse" + args[2]); + } + return new NDArray(new Shape(dims), dtype); + + } + } +} diff --git a/src/TensorFlowNET.Core/NumPy/NDArray.Pickle.cs b/src/TensorFlowNET.Core/NumPy/NDArray.Pickle.cs new file mode 100644 index 000000000..b4d66243a --- /dev/null +++ b/src/TensorFlowNET.Core/NumPy/NDArray.Pickle.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.NumPy +{ + public partial class NDArray + { + public void __setstate__(object[] args) + { + Console.WriteLine("NDArray __setstate__"); + Console.WriteLine(args.Length); + for (int i = 0; i < args.Length; i++) + { + Console.WriteLine(args[i]); + } + } + } +} diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index 09f5b0770..38778c3fe 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -112,6 +112,7 @@ https://tensorflownet.readthedocs.io + diff --git a/src/TensorFlowNET.Keras/Datasets/Imdb.cs b/src/TensorFlowNET.Keras/Datasets/Imdb.cs index 56b0d2a77..016b352d9 100644 --- a/src/TensorFlowNET.Keras/Datasets/Imdb.cs +++ b/src/TensorFlowNET.Keras/Datasets/Imdb.cs @@ -5,6 +5,13 @@ using Tensorflow.Keras.Utils; using Tensorflow.NumPy; using System.Linq; +using Google.Protobuf.Collections; +using Microsoft.VisualBasic; +using OneOf.Types; +using static HDF.PInvoke.H5; +using System.Data; +using System.Reflection.Emit; +using System.Xml.Linq; namespace Tensorflow.Keras.Datasets { @@ -12,13 +19,59 @@ namespace Tensorflow.Keras.Datasets /// This is a dataset of 25,000 movies reviews from IMDB, labeled by sentiment /// (positive/negative). Reviews have been preprocessed, and each review is /// encoded as a list of word indexes(integers). + /// For convenience, words are indexed by overall frequency in the dataset, + /// so that for instance the integer "3" encodes the 3rd most frequent word in + /// the data.This allows for quick filtering operations such as: + /// "only consider the top 10,000 most + /// common words, but eliminate the top 20 most common words". + /// As a convention, "0" does not stand for a specific word, but instead is used + /// to encode the pad token. + /// Args: + /// path: where to cache the data (relative to %TEMP%/imdb/imdb.npz). + /// num_words: integer or None.Words are + /// ranked by how often they occur(in the training set) and only + /// the `num_words` most frequent words are kept.Any less frequent word + /// will appear as `oov_char` value in the sequence data.If None, + /// all words are kept.Defaults to `None`. + /// skip_top: skip the top N most frequently occurring words + /// (which may not be informative). These words will appear as + /// `oov_char` value in the dataset.When 0, no words are + /// skipped. Defaults to `0`. + /// maxlen: int or None.Maximum sequence length. + /// Any longer sequence will be truncated. None, means no truncation. + /// Defaults to `None`. + /// seed: int. Seed for reproducible data shuffling. + /// start_char: int. The start of a sequence will be marked with this + /// character. 0 is usually the padding character. Defaults to `1`. + /// oov_char: int. The out-of-vocabulary character. + /// Words that were cut out because of the `num_words` or + /// `skip_top` limits will be replaced with this character. + /// index_from: int. Index actual words with this index and higher. + /// Returns: + /// Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`. + /// + /// ** x_train, x_test**: lists of sequences, which are lists of indexes + /// (integers). If the num_words argument was specific, the maximum + /// possible index value is `num_words - 1`. If the `maxlen` argument was + /// specified, the largest possible sequence length is `maxlen`. + /// + /// ** y_train, y_test**: lists of integer labels(1 or 0). + /// + /// Raises: + /// ValueError: in case `maxlen` is so low + /// that no input sequence could be kept. + /// Note that the 'out of vocabulary' character is only used for + /// words that were present in the training set but are not included + /// because they're not making the `num_words` cut here. + /// Words that were not seen in the training set but are in the test set + /// have simply been skipped. /// + /// """Loads the [IMDB dataset](https://ai.stanford.edu/~amaas/data/sentiment/). public class Imdb { string origin_folder = "https://storage.googleapis.com/tensorflow/tf-keras-datasets/"; string file_name = "imdb.npz"; string dest_folder = "imdb"; - /// /// Loads the [IMDB dataset](https://ai.stanford.edu/~amaas/data/sentiment/). /// @@ -41,8 +94,10 @@ public DatasetPass load_data(string path = "imdb.npz", int index_from = 3) { var dst = Download(); - - var lines = File.ReadAllLines(Path.Combine(dst, "imdb_train.txt")); + var fileBytes = File.ReadAllBytes(Path.Combine(dst, file_name)); + var (x_train, x_test) = LoadX(fileBytes); + var (y_train, y_test) = LoadY(fileBytes); + /*var lines = File.ReadAllLines(Path.Combine(dst, "imdb_train.txt")); var x_train_string = new string[lines.Length]; var y_train = np.zeros(new int[] { lines.Length }, np.int64); for (int i = 0; i < lines.Length; i++) @@ -62,7 +117,7 @@ public DatasetPass load_data(string path = "imdb.npz", x_test_string[i] = lines[i].Substring(2); } - var x_test = np.array(x_test_string); + var x_test = np.array(x_test_string);*/ return new DatasetPass { diff --git a/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs b/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs index 8317346ea..778290bb8 100644 --- a/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs +++ b/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs @@ -1,7 +1,9 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using System; +using System.Collections.Generic; using System.Linq; using static Tensorflow.Binding; +using static Tensorflow.KerasApi; namespace TensorFlowNET.UnitTest.Dataset { @@ -195,5 +197,20 @@ public void Shuffle() Assert.IsFalse(allEqual); } + [TestMethod] + public void GetData() + { + var vocab_size = 20000; // Only consider the top 20k words + var maxlen = 200; // Only consider the first 200 words of each movie review + var dataset = keras.datasets.imdb.load_data(num_words: vocab_size); + var x_train = dataset.Train.Item1; + var y_train = dataset.Train.Item2; + var x_val = dataset.Test.Item1; + var y_val = dataset.Test.Item2; + print(len(x_train) + "Training sequences"); + print(len(x_val) + "Validation sequences"); + x_train = keras.preprocessing.sequence.pad_sequences((IEnumerable)x_train, maxlen: maxlen); + x_val = keras.preprocessing.sequence.pad_sequences((IEnumerable)x_val, maxlen: maxlen); + } } } From b27ccca84fe68394e4ffbf4babd16d0d2e05674e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Tue, 11 Jul 2023 22:40:30 +0800 Subject: [PATCH 03/98] fix:fix the bug of load LSTM model --- .../Keras/ArgsDefinition/Rnn/GRUCellArgs.cs | 2 +- .../Keras/ArgsDefinition/Rnn/LSTMArgs.cs | 2 +- .../Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs | 2 +- .../Keras/ArgsDefinition/Rnn/RNNArgs.cs | 12 ++++++-- .../ArgsDefinition/Rnn/RnnOptionalArgs.cs | 2 +- .../Keras/ArgsDefinition/Rnn/SimpleRNNArgs.cs | 2 +- .../ArgsDefinition/Rnn/SimpleRNNCellArgs.cs | 2 +- .../ArgsDefinition/Rnn/StackedRNNCellsArgs.cs | 4 +-- .../Keras/Layers/ILayersApi.cs | 2 +- .../Keras/Layers/Rnn/IRnnCell.cs | 2 +- .../Keras/Layers/Rnn/IStackedRnnCells.cs | 2 +- .../Operations/NnOps/RNNCell.cs | 3 +- src/TensorFlowNET.Core/ops.cs | 4 ++- src/TensorFlowNET.Keras/Layers/LayersApi.cs | 3 +- .../Layers/Rnn/DropoutRNNCellMixin.cs | 2 +- src/TensorFlowNET.Keras/Layers/Rnn/GRUCell.cs | 3 +- src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs | 4 +-- .../Layers/Rnn/LSTMCell.cs | 4 +-- src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs | 4 +-- src/TensorFlowNET.Keras/Layers/Rnn/RnnBase.cs | 2 +- .../Layers/Rnn/SimpleRNN.cs | 4 +-- .../Layers/Rnn/SimpleRNNCell.cs | 4 +-- .../Layers/Rnn/StackedRNNCells.cs | 4 +-- .../Saving/KerasObjectLoader.cs | 1 - .../SavedModel/serialized_attributes.cs | 2 +- src/TensorFlowNET.Keras/Utils/RnnUtils.cs | 2 +- .../Layers/Rnn.Test.cs | 2 +- .../Model/ModelLoadTest.cs | 29 ++++++++++++++++++- 28 files changed, 71 insertions(+), 40 deletions(-) diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUCellArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUCellArgs.cs index 75d5d0218..624756afe 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUCellArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUCellArgs.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Text; -namespace Tensorflow.Keras.ArgsDefinition.Rnn +namespace Tensorflow.Keras.ArgsDefinition { public class GRUCellArgs : AutoSerializeLayerArgs { diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs index db76fda06..d816b0ff7 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs @@ -1,4 +1,4 @@ -namespace Tensorflow.Keras.ArgsDefinition.Rnn +namespace Tensorflow.Keras.ArgsDefinition { public class LSTMArgs : RNNArgs { diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs index 786236e4d..f45032312 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs @@ -1,7 +1,7 @@ using Newtonsoft.Json; using static Tensorflow.Binding; -namespace Tensorflow.Keras.ArgsDefinition.Rnn +namespace Tensorflow.Keras.ArgsDefinition { // TODO: complete the implementation public class LSTMCellArgs : AutoSerializeLayerArgs diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs index 2d7fb001a..b84d30d3d 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs @@ -1,8 +1,8 @@ using Newtonsoft.Json; using System.Collections.Generic; -using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Keras.Layers; -namespace Tensorflow.Keras.ArgsDefinition.Rnn +namespace Tensorflow.Keras.ArgsDefinition { // TODO(Rinne): add regularizers. public class RNNArgs : AutoSerializeLayerArgs @@ -23,16 +23,22 @@ public class RNNArgs : AutoSerializeLayerArgs public int? InputDim { get; set; } public int? InputLength { get; set; } // TODO: Add `num_constants` and `zero_output_for_mask`. - + [JsonProperty("units")] public int Units { get; set; } + [JsonProperty("activation")] public Activation Activation { get; set; } + [JsonProperty("recurrent_activation")] public Activation RecurrentActivation { get; set; } + [JsonProperty("use_bias")] public bool UseBias { get; set; } = true; public IInitializer KernelInitializer { get; set; } public IInitializer RecurrentInitializer { get; set; } public IInitializer BiasInitializer { get; set; } + [JsonProperty("dropout")] public float Dropout { get; set; } = .0f; + [JsonProperty("zero_output_for_mask")] public bool ZeroOutputForMask { get; set; } = false; + [JsonProperty("recurrent_dropout")] public float RecurrentDropout { get; set; } = .0f; } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RnnOptionalArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RnnOptionalArgs.cs index 64b500bba..a6520589d 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RnnOptionalArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RnnOptionalArgs.cs @@ -3,7 +3,7 @@ using System.Text; using Tensorflow.Common.Types; -namespace Tensorflow.Keras.ArgsDefinition.Rnn +namespace Tensorflow.Keras.ArgsDefinition { public class RnnOptionalArgs: IOptionalArgs { diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNArgs.cs index fcfd694d1..e45ef79d0 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNArgs.cs @@ -1,4 +1,4 @@ -namespace Tensorflow.Keras.ArgsDefinition.Rnn +namespace Tensorflow.Keras.ArgsDefinition { public class SimpleRNNArgs : RNNArgs { diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNCellArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNCellArgs.cs index d21d61905..b84ea21b3 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNCellArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNCellArgs.cs @@ -1,6 +1,6 @@ using Newtonsoft.Json; -namespace Tensorflow.Keras.ArgsDefinition.Rnn +namespace Tensorflow.Keras.ArgsDefinition { public class SimpleRNNCellArgs: AutoSerializeLayerArgs { diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs index 50a6127df..2600f14ee 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs @@ -1,7 +1,7 @@ using System.Collections.Generic; -using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Keras.Layers; -namespace Tensorflow.Keras.ArgsDefinition.Rnn +namespace Tensorflow.Keras.ArgsDefinition { public class StackedRNNCellsArgs : LayerArgs { diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index b48cd5535..1670f9d1d 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -1,7 +1,7 @@ using System; using Tensorflow.Framework.Models; using Tensorflow.Keras.Engine; -using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Keras.Layers; using Tensorflow.NumPy; using static Google.Protobuf.Reflection.FieldDescriptorProto.Types; diff --git a/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs b/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs index 8d6fbc976..43df75b17 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs @@ -3,7 +3,7 @@ using System.Text; using Tensorflow.Common.Types; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { public interface IRnnCell: ILayer { diff --git a/src/TensorFlowNET.Core/Keras/Layers/Rnn/IStackedRnnCells.cs b/src/TensorFlowNET.Core/Keras/Layers/Rnn/IStackedRnnCells.cs index e73244a51..8cf6150d3 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/Rnn/IStackedRnnCells.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/Rnn/IStackedRnnCells.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Text; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { public interface IStackedRnnCells : IRnnCell { diff --git a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs index 4e99731f9..9905d39c8 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs @@ -19,9 +19,8 @@ limitations under the License. using Tensorflow.Common.Types; using Tensorflow.Keras; using Tensorflow.Keras.ArgsDefinition; -using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; -using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Keras.Layers; using Tensorflow.Keras.Saving; using Tensorflow.NumPy; using Tensorflow.Operations; diff --git a/src/TensorFlowNET.Core/ops.cs b/src/TensorFlowNET.Core/ops.cs index 2dc463296..c624c9901 100644 --- a/src/TensorFlowNET.Core/ops.cs +++ b/src/TensorFlowNET.Core/ops.cs @@ -571,7 +571,9 @@ public static bool executing_eagerly_outside_functions() if (tf.Context.executing_eagerly()) return true; else - throw new NotImplementedException(""); + // TODO(Wanglongzhi2001), implement the false case + return true; + //throw new NotImplementedException(""); } public static bool inside_function() diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index 5968461d0..cb85bbba1 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -2,9 +2,8 @@ using Tensorflow.Framework.Models; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.ArgsDefinition.Core; -using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; -using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Keras.Layers; using Tensorflow.NumPy; using static Tensorflow.Binding; using static Tensorflow.KerasApi; diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs b/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs index 75feb8ea2..27c13f349 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs @@ -6,7 +6,7 @@ using Tensorflow.Keras.Engine; using Tensorflow.Keras.Utils; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { public abstract class DropoutRNNCellMixin: Layer, IRnnCell { diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/GRUCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/GRUCell.cs index 02fe54f49..2b9c01e31 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/GRUCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/GRUCell.cs @@ -3,12 +3,11 @@ using System.Diagnostics; using System.Text; using Tensorflow.Keras.ArgsDefinition; -using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Common.Extensions; using Tensorflow.Common.Types; using Tensorflow.Keras.Saving; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { /// /// Cell class for the GRU layer. diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs b/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs index 025465fd6..b5d583248 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs @@ -1,10 +1,10 @@ using System.Linq; -using Tensorflow.Keras.ArgsDefinition.Rnn; +using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Common.Types; using Tensorflow.Common.Extensions; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { /// /// Long Short-Term Memory layer - Hochreiter 1997. diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs index 284a2b778..e4fc6dd22 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs @@ -3,12 +3,12 @@ using System.Diagnostics; using Tensorflow.Common.Extensions; using Tensorflow.Common.Types; -using Tensorflow.Keras.ArgsDefinition.Rnn; +using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; using Tensorflow.Keras.Utils; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { /// /// Cell class for the LSTM layer. diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index 6075547bb..0e81d20e3 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using System.Reflection; using Tensorflow.Keras.ArgsDefinition; -using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; using Tensorflow.Util; @@ -14,7 +13,7 @@ using System.Runtime.CompilerServices; // from tensorflow.python.distribute import distribution_strategy_context as ds_context; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { /// /// Base class for recurrent layers. @@ -185,6 +184,7 @@ private Tensors compute_mask(Tensors inputs, Tensors mask) public override void build(KerasShapesWrapper input_shape) { + _buildInputShape = input_shape; input_shape = new KerasShapesWrapper(input_shape.Shapes[0]); InputSpec get_input_spec(Shape shape) diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RnnBase.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RnnBase.cs index 018b17780..1419da4b2 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RnnBase.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RnnBase.cs @@ -4,7 +4,7 @@ using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { public abstract class RnnBase: Layer { diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs index a22f31c7d..9c199eb43 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs @@ -1,11 +1,11 @@ using System.Data; -using Tensorflow.Keras.ArgsDefinition.Rnn; +using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Saving; using Tensorflow.Operations.Activation; using static HDF.PInvoke.H5Z; using static Tensorflow.ApiDef.Types; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { public class SimpleRNN : RNN { diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs index c77f77790..e74b56925 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; using System.Text; -using Tensorflow.Keras.ArgsDefinition.Rnn; +using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; using Tensorflow.Common.Types; @@ -9,7 +9,7 @@ using Tensorflow.Keras.Utils; using Tensorflow.Graphs; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { /// /// Cell class for SimpleRNN. diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs index 8799bfb23..ece2bc5bf 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs @@ -3,12 +3,12 @@ using System.Linq; using Tensorflow.Common.Extensions; using Tensorflow.Common.Types; -using Tensorflow.Keras.ArgsDefinition.Rnn; +using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; using Tensorflow.Keras.Utils; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { public class StackedRNNCells : Layer, IRnnCell { diff --git a/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs b/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs index fd1453d3c..0bd816ccb 100644 --- a/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs +++ b/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs @@ -13,7 +13,6 @@ using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Layers; -using Tensorflow.Keras.Layers.Rnn; using Tensorflow.Keras.Losses; using Tensorflow.Keras.Metrics; using Tensorflow.Keras.Saving.SavedModel; diff --git a/src/TensorFlowNET.Keras/Saving/SavedModel/serialized_attributes.cs b/src/TensorFlowNET.Keras/Saving/SavedModel/serialized_attributes.cs index 0ec5d1a8c..325d3327a 100644 --- a/src/TensorFlowNET.Keras/Saving/SavedModel/serialized_attributes.cs +++ b/src/TensorFlowNET.Keras/Saving/SavedModel/serialized_attributes.cs @@ -3,7 +3,7 @@ using System.Linq; using System.Text; using Tensorflow.Keras.Engine; -using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Keras.Layers; using Tensorflow.Keras.Metrics; using Tensorflow.Train; diff --git a/src/TensorFlowNET.Keras/Utils/RnnUtils.cs b/src/TensorFlowNET.Keras/Utils/RnnUtils.cs index e8700c1f2..1e9f6d845 100644 --- a/src/TensorFlowNET.Keras/Utils/RnnUtils.cs +++ b/src/TensorFlowNET.Keras/Utils/RnnUtils.cs @@ -3,7 +3,7 @@ using System.Diagnostics; using System.Text; using Tensorflow.Common.Types; -using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Keras.Layers; using Tensorflow.Common.Extensions; namespace Tensorflow.Keras.Utils diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs index becdbcd60..5f7bd574e 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs @@ -6,7 +6,7 @@ using System.Threading.Tasks; using Tensorflow.Common.Types; using Tensorflow.Keras.Engine; -using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Keras.Layers; using Tensorflow.Keras.Saving; using Tensorflow.NumPy; using Tensorflow.Train; diff --git a/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs b/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs index 10db2bd11..382941d9a 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs @@ -1,5 +1,7 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; +using Microsoft.VisualStudio.TestPlatform.Utilities; +using Microsoft.VisualStudio.TestTools.UnitTesting; using System.Linq; +using Tensorflow.Keras.Engine; using Tensorflow.Keras.Optimizers; using Tensorflow.Keras.UnitTest.Helpers; using Tensorflow.NumPy; @@ -79,6 +81,31 @@ public void ModelWithSelfDefinedModule() model.fit(dataset.Train.Data, dataset.Train.Labels, batch_size, num_epochs); } + [TestMethod] + public void LSTMLoad() + { + var inputs = np.random.randn(10, 5, 3); + var outputs = np.random.randn(10, 1); + var model = keras.Sequential(); + model.add(keras.Input(shape: (5, 3))); + var lstm = keras.layers.LSTM(32); + + model.add(lstm); + + model.add(keras.layers.Dense(1, keras.activations.Sigmoid)); + + model.compile(optimizer: keras.optimizers.Adam(), + loss: keras.losses.MeanSquaredError(), + new[] { "accuracy" }); + + var result = model.fit(inputs.numpy(), outputs.numpy(), batch_size: 10, epochs: 3, workers: 16, use_multiprocessing: true); + + model.save("LSTM_Random"); + + var model_loaded = keras.models.load_model("LSTM_Random"); + model_loaded.summary(); + } + [Ignore] [TestMethod] public void VGG19() From 9c949e336f6c9fedd6a6ae5eace581084d16a8b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Tue, 11 Jul 2023 23:44:42 +0800 Subject: [PATCH 04/98] refactor: refactor LSTMLoad test --- .../lstm_from_sequential/fingerprint.pb | 1 + .../lstm_from_sequential/keras_metadata.pb | 7 +++++ .../lstm_from_sequential/saved_model.pb | Bin 0 -> 755111 bytes .../variables/variables.data-00000-of-00001 | Bin 0 -> 61038 bytes .../variables/variables.index | Bin 0 -> 1373 bytes .../Model/ModelLoadTest.cs | 26 ++++-------------- .../Tensorflow.Keras.UnitTest.csproj | 16 +++++++++++ 7 files changed, 30 insertions(+), 20 deletions(-) create mode 100644 test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/fingerprint.pb create mode 100644 test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/keras_metadata.pb create mode 100644 test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/saved_model.pb create mode 100644 test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/variables/variables.data-00000-of-00001 create mode 100644 test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/variables/variables.index diff --git a/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/fingerprint.pb b/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/fingerprint.pb new file mode 100644 index 000000000..f6ea8da23 --- /dev/null +++ b/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/fingerprint.pb @@ -0,0 +1 @@ +沦Ʉ%̟땐͉ Σ(Ћ܇}2 \ No newline at end of file diff --git a/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/keras_metadata.pb b/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/keras_metadata.pb new file mode 100644 index 000000000..5fe8f1a65 --- /dev/null +++ b/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/keras_metadata.pb @@ -0,0 +1,7 @@ + +&root"_tf_keras_sequential*&{"name": "sequential", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": false, "class_name": "Sequential", "config": {"name": "sequential", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 5, 3]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_1"}}, {"class_name": "LSTM", "config": {"name": "lstm", "trainable": true, "dtype": "float32", "return_sequences": false, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 32, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 1}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 2}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 3}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}}, {"class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "units": 1, "activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}]}, "shared_object_id": 9, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, 5, 3]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}}], "build_input_shape": {"class_name": "TensorShape", "items": [null, 5, 3]}, "is_graph_network": true, "full_save_spec": {"class_name": "__tuple__", "items": [[{"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 5, 3]}, "float32", "input_1"]}], {}]}, "save_spec": {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 5, 3]}, "float32", "input_1"]}, "keras_version": "2.12.0", "backend": "tensorflow", "model_config": {"class_name": "Sequential", "config": {"name": "sequential", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 5, 3]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_1"}, "shared_object_id": 0}, {"class_name": "LSTM", "config": {"name": "lstm", "trainable": true, "dtype": "float32", "return_sequences": false, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 32, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 1}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 2}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 3}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 5}, {"class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "units": 1, "activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 6}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 7}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "shared_object_id": 8}]}}, "training_config": {"loss": "binary_crossentropy", "metrics": [[{"class_name": "MeanMetricWrapper", "config": {"name": "accuracy", "dtype": "float32", "fn": "binary_accuracy"}, "shared_object_id": 11}]], "weighted_metrics": null, "loss_weights": null, "optimizer_config": {"class_name": "Custom>Adam", "config": {"name": "Adam", "weight_decay": null, "clipnorm": null, "global_clipnorm": null, "clipvalue": null, "use_ema": false, "ema_momentum": 0.99, "ema_overwrite_frequency": null, "jit_compile": false, "is_legacy_optimizer": false, "learning_rate": 0.0010000000474974513, "beta_1": 0.9, "beta_2": 0.999, "epsilon": 1e-07, "amsgrad": false}}}}2 + root.layer_with_weights-0"_tf_keras_rnn_layer* {"name": "lstm", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "LSTM", "config": {"name": "lstm", "trainable": true, "dtype": "float32", "return_sequences": false, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 32, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 1}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 2}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 3}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 5, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, null, 3]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}, "shared_object_id": 12}], "build_input_shape": {"class_name": "TensorShape", "items": [null, 5, 3]}}2 +root.layer_with_weights-1"_tf_keras_layer*{"name": "dense", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "units": 1, "activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 6}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 7}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "shared_object_id": 8, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 32}}, "shared_object_id": 13}, "build_input_shape": {"class_name": "TensorShape", "items": [null, 32]}}2 +root.layer_with_weights-0.cell"_tf_keras_layer*{"name": "lstm_cell", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "LSTMCell", "config": {"name": "lstm_cell", "trainable": true, "dtype": "float32", "units": 32, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 1}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 2}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 3}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 4, "build_input_shape": {"class_name": "__tuple__", "items": [null, 3]}}2 +Rroot.keras_api.metrics.0"_tf_keras_metric*{"class_name": "Mean", "name": "loss", "dtype": "float32", "config": {"name": "loss", "dtype": "float32"}, "shared_object_id": 14}2 +Sroot.keras_api.metrics.1"_tf_keras_metric*{"class_name": "MeanMetricWrapper", "name": "accuracy", "dtype": "float32", "config": {"name": "accuracy", "dtype": "float32", "fn": "binary_accuracy"}, "shared_object_id": 11}2 \ No newline at end of file diff --git a/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/saved_model.pb b/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/saved_model.pb new file mode 100644 index 0000000000000000000000000000000000000000..6fb7c3f0e8e4a35afa38cada60f78a6097b84501 GIT binary patch literal 755111 zcmeFaeQaDwb}z=;&5ve@qNpoUqDqve#aYcq&v3|Yez$kmGt%zN&hE@eD`{rdv%7QE zVmGB8kxg!QQ#0BHvTMI*@8bEz2%N-#9XrO3jkV+Vhn6jQ4;b zuwgvdj^WrbV8s6f&Z&=k>(sqf_oI19+W}{>6nSsmsye?qb?VfqQ|FW%_q#v-68R6* z{qHFG3>kT)*?gzwjuPjidFL+s_X+$<;oq;rzw?&~JRI&n*>BZdGF6+LJm(aK&z`?9 zI#L`jx!-b}|HN^~`()~sUa$S2^G>7NZrtB#-Q0I4NU7WE?HzQtTC>ycBpG?!*g0s; zk2G0b^QVdUMN~ac$t1jPHyizy*L&F5Z@I(w_x5&Z;klvjK>gox$k_FEqX%7eiC+`+ zYn%+o6)S%17;gQkO_^FB0lV_&Ry64Vcm|q~Lpt**( zz1Q7s^xe^3zuWFSprh{aTc3OPMtzb@h~MDnjnBPHHps}0y-u%B&jbxgQqIT@xugA7 zr?=M)(MCQ&&VHudW2oHN+kbMibGy}O>gX^M-zUSbws&@1mkc-CyS;hR?lfB;c?>K} zvQFoCKD`Y0K$6^i5JB~p65+3NJ${U@xxc1J*EtB)Lz(Kj3YHxG6g zM>GIb_V0a$^aL5(@9s4Zwm8%m$f4icHof7)KSy z8Bdj5@?#Eh-)waswBD)Rc-Y!{w7=Kx^n30Fa&9a5)7$U1w%Z@Idb4BhdEz!(y?(pX z=(qPe^3Ckn{3UXx+1hCVJMHZ>y=J@H3wq$pKSjWYwzYhd6Nv^YHU4%88FIc0OQt*^Og?$1aaPSi$Lmg z&XO~ahV={?ZhX}4@prT^J4F6lhn%_9=)yPxy{+aAAjCZdg}M)Vwc_1&heA7IBGiWO z!Y}jZ2z%_3op!JP9Al^FpL6D?34a-S3hi#Y!`la)EhsR$wb$8hKL{p9!5xBWIw76{ z(0lzo?cF7Ey7f_O>!82a6PP*p3<{qOqA=@M!6vC&Ce3~n-1oZ!y4QnqjwsQ2W-vF+S(67YryndefKmOhkaxko{%x@-tO-lSZKO#??r`agCB!p6L(P|fpZM7R3+HUakxyHta$> zXjP%-1A<|uj`QYONthGr57T5N|HdH~?(|^|+dkO&CpSldU4I!k_GcV2$!P$I*n9UO zU>z(MHOx8i$NX7_u`XC9$S`9(DMjF~`+LnN@~<*EEhLjz;(P8XE|d5)d!81^5G-Qn z_8VOgX{{Zv-EVcdP!EJ30SPe8`qtjM-Fnda=+%Rr9ay8b9x{gE3mVK&nAM_@SpxkV z+i&!to{lf6$p7Pzi94{;1M}TslR6-dO%D(t_gfFzo%xa1+2RXaS!{Kh{$EpMjOhrl z+{h=kX*7SH$O?qsG)6{hMNpT3Cke0L+imn71tJ|JbP1XXJ)8w`y0g>X@3ni;H=Wjp z9$OYdZ!R&J1zOA_!%OlOA&l_OgU`c~3Ibx3BL+f^Ek+8pv3Y$hx3Hy5JWo+G12*Te6CGN5`0Qv>u2=mH;E&uh8{u+-3g+vRSP-^B0MDXILhF$|0vo?k+md2M*<0+uB+iLQq9J6dlF`tL&&hc=?Kl4wMGh8i%Pn)gB zY{hpcgoNVrhKneWlJg^+5h|m#m#VdwmTH$Ci(eMHTh)bXZQ;RE^Zx4QdUL6{zPjPK z|NEc62+IA)*4{y$HuzqFVAUxyb8Z zWcm^Rfs~|>(nQ5Ksaj_p@9*^+pkkttNcCw_$9c7^IIkH36L5qaZ+5%m|RbrXxTWPDjAk$=O$$jorn^i?HqNwHDb% zeSoom-L>HV(fY7kC^?mReLT*&(Pl`GT#K}wio|GL6);VNW@(A|eZe!8_yq4B55c&Z zfMEPE1Ht&S^aTHiL(XY1KLSmwHHfS@=MOTemtc^~e4sy}5A=*{YK(k?JZtDfD%wr; zRjx#adxGFzX+s_$-~)Ip2b8bw7%zpHey+5fyiK)JFDZ(A&BLBcs`UFV?Z+>{+4A3LO;uCJ*8OxbMZ zsR);yCK^E+rv-pUM&`HyP3(XIG_hy~KojfA2<-BGrjH<*KTQM=l5Lq#{Or zn}(2a@=bE?sPvbp9&2ccd_X`iN0?;WG2+DoMqlKI#d^2b9?M;{CQ(fQo&-bGzJXQI)NO;VI<&{E7tXXUXvJ6m+7*?$$VcC!F4%Rwz0S@P_)YlKh+*o% z(bpIq;r=a^h0y8zWjL!CBgGrH-kt*g8B%-=ei^D2K0mbK{D?CILkg$;&)oh5nfIWw z?H263w!r%aym%V@gKo?Fu-n)Nv%R;vv9dzoB#(JBfHMs9Y52K{^oDVTQ_TEnuz4|y zc^w+7FQr*>$14BsdEas7cb~uhzdm0YbBC_~&(D*eDUjKc<4*51oE8sE$E;*ihaS^z8 zrrUaOumfekF8oP$_IkZm52_e-$3YG>dR}9{O(vkPCC}S}(*n;Ul~T!lhX2l71-dN& zuj8>-!3lh0$CDLZDvi4rf|=F>*8l}Lv_jnRz5RZBxBc~2mrRyUxg;2lUImPw?1cGr zX+kIGw2hoeJ2_``a;7wL&ZZ`3rsTS3L8I+7_jbJpEjWr{$AqL*I_C~EBM{Iq4K%PL zaj&=E+Tt{PkI^ur({SEK11x*CA(_=_xTw)UQ`107m)ubyny=PS6rtG|2*Y7HTdU}>ny`RbFQQHOl(L>SKU$0;yvKw zd3S8L1;BCc-BN`499m3 zfVk$40a_rSm=Jiz|AS5SXZ=5F@W*rhAIktW`|f%F0sF4v|G~a{!T*DO_oDv?`;HCM z4}%{>X~88(K%6PPC3+YkB=*mp1cf3WXf@&91oUHAX+ne9)QNW7to#Lw77;%Dt5@l{t_ zrC!rS;^$I}#F^6TCb2$Q`g|bFVTA%lx*SN(X%LZ9;cX+x)2EO;W&Gy~@yTQ=h|el# zS<^FC_CIBk@(XeN`3B<8H!**{mBgPnqx^a3C268G=o<7)iKgbyGr&N%dVuBwY-5nO!fmAiLuzkdDgyPv(Y zsMo?r>r3wKNbN{5rYpv}*`k_M^V{ytNX-^){+U2+TU<)2)^mR-Qf=aH05woeYT%Ch zW~2e*ZUR+WORChLERIk>3V5>I7SHbZ&sfawTxXMvmcHs1s!N<5OvEeW8(j+Y*+sN+CK3Oh zdpClpZl0vesA6pePO6&$#WXuWfJq@BooRMJGt=yVSf<$ll}fV%j95VchZO`=SV4e; z75u7SFbqE!i$Do>1V-VPaev3G-!ZRPja*#5A15w7Brfk^aoI=`m-nOM^3coGL}}3V z>6y~!Qj5ztti$g%U=`lNtMKVk6R)V_rIsuFp~!e?TRLCyS@1dd@`3&(c%`Zx<9Mm9 zzWl&<+nOvr0@ZN4<30M2`S0_Z9=MGN>x(lR%TWZaqM4w?VxrWtH}sevR-+SZu`pN~ zN|a<@b00-05+|V%ioyznrOD-_Ic{$N!Sh z=82i8u!O}D`u7{P6@? z*9v?0;pz@-0-D}~{R593=ZllCZ8%3_ue^=r+J?8Wx&-E6a%x{)rJZ%y^&a%Vt41Hy zqK}rNk5=H(GFg1kZ8Y0ptL}NNkM`kYhuuc^Y94zm?(KQv7M#2l3m-jCuBnfDUpr{E zzTU#+w+@<}PPfy6qA!qVqeZKh!i(euRG44AlZ{$FxJoXo58w!GujlQwAGJKxfi+T7 zMf~TYI{z<*H(YPtXYxthYln`nljYRkDcs=0O|m+`hkO)QNj3GSaB{NM==0A$MK;xE z0S$u2b_cYQKD&%{!0u6sBPe-!pFM?}uod!*`dpEMA#Zo!-t0pl?nUyPv8*zN{cc^B@QdG! zxqPM*MezrQRwU~NKs$K#lFAanuo|6+4k`x~UU z_BV)b?Qf9V+TW|QEc5_?$|AZ?nGsJ&(uoI)=Eval9~=<3(K|^mIr9zriq3XTMH|y1&e}C!4dqb8r#qE zW%@H}nO?lZV>pOV4m9h6u9~;sd@Ix^z^SIhVn7s2zvs|jamdqg9fG^lyh~&97dod( zv)>qNR_o#}Y<-DV)LRR2L0Q$?)C2xaf?B_W)D<1KbUKY{-;P!>gw`)Qp>cz zY60<7^>I~~hC|IFA;Ap~`((&}`oBk95VWrvibV98#jfonOy zfvSE6aA9(Sf~udH0Q*gF8Btw@SXFPi!|bPt+n0jI=!@!`D7Q-rgVL;PZ@Gopg6bp0 zXYsan8oi!7N2ogsekJvnRh8^FF8R|-_8)i1ysALc_W3fc4}KOMU3EG=1Hg?wrFt)# zdLwoQJe%{U+K*q-tCM)yt^^4*lsOS%_zf&@i}DcS297Vih9^Y>5aMoqjS? zYhuvV*Y#{)!%(SjKb`GciM0BDT7A!^f4z6G3qG-MA}pfHKvXt059|VFBqEW)tu`9p z1p;HyErOa<`(AhaJ+=pa1Gd04eq*fqGaKS_5 zy4*UuQ{g9cfNeKb^FG+Zi+w_5tFHE`bg=C<&A!z}?5vwaL?5}v6Z9J`)n1H=1T9fP zF}dvs)wiaYEQL%|%xPEySxs=!fH$BDs=a!6v5XW=%Ut&&%=H#|11=G;djp}11V#Qh zs+HIt^Sk2LD8!6G%A+zGa1vkV#*bUd_~D+7Jfef!PGXjtBK>`BO$~J{_|sJ}bWleU zCR<%x*znC(pg+H)cq6Djg3X^z8ifXj-|}Gk;Zv$1q`GjhxeDsaJxEH6 zlInJ_T zG&+xb?Z~px#uu526*#LFs@eGUF;LBh>R@DnXnB_uN9hfK&Brahy0f>}eRHo#l{ocn zPI9=AX{gFgpJ=IA)N8yAU5=npwB5F$50E)LRFj;YH&{g57`es6Pl;%3OlvVKr}taWDSWF$6Wdxn#vV!UPqF9NU0Vw|KM6L$4;Fu|1NupRuOAEy@krzue znPkvTj+x}uB$AQTH=hk0l4RIh9W%*mNhB*p<7h@1ws^-(@_G`q0Flr&N?>;=q=u|*&#H4mrmypFW zNf|#HqI1!`vJqWPBwB4Nj)v%5)URwr*Aj_7gaTHkDcCy~4J;ec%ZWrELI+DlbS_F* zHlkM&iB43*;zW0fkr6*Sah1Qxr2jlqKbP?4`hsM1`a5#H;*R~@Xzv1f95KqFhs3XNI^ke`b0 zIU?@}2{Q8H8gb3yIH8d(W`#zr7A9yg8F@h?u8-sn4U;s~JfUQZS)oyDiU}G_MrLTr zW?3Vy#T+Lzvc;^>sFlYA4JIQmXvB4&~5f9DvLH5e4E*%9R5GWS8#D&|6AYwmd6M7vwJVYvBxLTWL7sn%JOtNXb zDVfBAiEJw~Cd!pFMDbl^*0rv1Wm=}a+a`-C7lJbTMD)MZE>mL1f zTTPMQueQM|rQG1KiV7eQwmsm4h_Mh0>#tj2mE8-_z_2174Hhz?p2$A~&Z9G=RFh4$dUiV9+Z6;VnC0#pk7FZGI zXCPo@_tF9@u}TN5jr455{%H~W*NV_Vat#$e8E#2Ee-MY+Oz{xy|x9u(HBSs^|?e z<`pAH$18%?utLk8)L5hxq~9+@=Z56AbeM%T0Xu!+rOjS4EUWD$!@`G`jBuGCNJkCD z5c9b#6>cIgCH)GaiqQm8a=Vkq4cDJ^$VGGY`u$erMBo$0*#q5fWr5Eq3KJFXY9LE| zp0VF--NthZ*OSucB4k9f+?90bL(Gwpb_TfCYHjJawfF$IsT=kiPb%($@1e&qU-a)g zMK4kD*Sdw7G*@6LeWCQbR@5A&ypL85L1y8d{MRR79EhwIZvih_ZOkUnV6#h$mQ|<&+A#T6vDl zu%!{A^&CJ(SO~offAm`)^<7Bc{^+wTH|*^LhzGjcs{Cz%oQ6Hp{y|?vT<*DlQXqd+ zh$e?EM>Fv9u$iWkk<7B?`jDv{3&)c_?v62dPniXb$}_GHUtNl(-WsW5!43-in{+>i z015bCwOV`+QUvPPDZLhh#yuG@#++0}RTG?D!*=$x9Hd^b|g{}MR ztBuY3Yptc`I)vMwW}okOTaZk*3z6+bPFxpa6G7_T%FASCd#}rbd~Y9g1Pk!o5YM{T zJkNh+i~kL+Lj;nE2YW%Na7ZKk+)!nT5XLi*UigE(E_+g+gS33J#1hg3-p<9oy%hTv z;tN`9E<^e|h(H+84oYNHjzzgNn^@vPVu|yKC1w&!OedB&msrA0C@}{~C+%o0Kr+kc zArV>lmaj1hbh*AzI+LY%xilHuyJG!n>9l*9#E9Q(^(tf2# zL)%qY1neUj3J4|^orTg+h8D=5kCR_7k~1GZY(pAuSPPrp>T+!j*mhPt53IPJQkMl86K}h38qE^Tnd*t*maYx`d2!^}h!*mj5pEZTP>lY#Q*p z$dL!XPm@UQWq4X5JeNJp;0d)efxix!Y=9*f*dmjG8DNnm+emguD(d%~uKM?!X0M-@ z_neCJ6As)yVE+~1|MW2#4;sM^9ugaRA$ZHXpr&vT&vGTAB0!d&`V~FcXlN}f(pzLw znE@6d|BX+aMW@yYi-rjtf?Eni)##$%DbT{kmR?ZrS&If*vgl#-HknctdevNr=#dFr zr-6(udJ&h%gOD$hGu&jcC=3^e9}I_n#~~xUHn>#Ha(B?#PzwS0RL(GeeQ_$Ej4)J% zJXbHlev`XX#s5}0L&iIxtcb=7v%~ZbIm`cm5fcCAK_A4x9h0^C6w*j5ipGfKNJA}m zY~a*de~VK4LxiB<)G{Ju%nn4e(mQY>EaPvUy?zFSJ&*Y7REC+jpl^~>pk_g2utj#t zaOnBkC8MG`bR2t5^<9C-1uUILZ3pidjG;*VF|-yGFVu45g(3QL0WYwN<(%6kFQ8U} zF4HrDkl=x0h0jD7zebk$w!WPHXP6~Q^h-prfj?q)>w(4$DWdf^$W*XA!1PMDMy0x2 z>90|Ahnzx9;1z0;&$j?ZdJ&bgtyOkUO*X}f0F|FT#r1!oXHdQTym9gdGcX1XtMRh~ zdyi^R)s7yjnjNYthDvnBKui);?~(Dqx4{TjEhn&+j~=XLJ6JUgmgvgiz^WO+TFwcq zm7@o1#SYdo21|71a9}MP!CJ`)tkoj{tAi!hDh6vxY6t-)7%tJ7;FXdp@e{F3zpN(W ziV?2Wc(`H~%#tp)){Y*pwK%*q!6iC#IJ{Pkc&+7y*ZR@pwH}9;Cc8vu4u{v85wG>U z@Y*R&4CP_B!l~)I={40b6`ofaE z_S)fVui9&0K78$H{SL#va`@WOf)}oR_3*W$^&wpQ+Tm+Q(llKA`e3!+B&Wq@2{sqG z>^XnFNJ@Tz(@{tNq^LOY|N}-8SIIhwE%^@W} z*fETd%z#Bt2b(Z1JkKc;SkMxkQ4f(D3_a_2EGVhEvK`rR-zB#bY?sU*#CR@idCc;9 zZIg3a0x3w%23frF-;jG`#?a0HK9uIL@k7+oKwh!^9r_;}a#|z7ow$Qi+|jc>O9Z1y z=d|(yTJ`pd;Votl<9jtNEMAOheG1^P%$!bcpNz?DqOSi7Jr|B?D%|?D+8*Rk}*k;bV z5xi2uz-Igpon45W$zLX8UExV4XYu^$Md6B6zj5nrz%a!*h66(-&8-vH37A`lxTW=F zNGlz9YoY^L3vG&kv`h&nOJ>hdEJ4u}2#$I(A@m+<1UrIGN!BxrKxzU&ne-2pJ?!l1 z1{{8v+FwVtV;9dUIpT)8Eu*b@G**#Eqgsr$B1h_7nVak6<^4damZTWs{Wu?_@ZEPpG z4yJSbdj-6UJ{9NE1+gP7r`Dq`KgpD25 zL&}yaORhYGtKWq4|1YHWn2W1W*vojfaiSpN!m419?Ubm$!4UGN0%kstjHhy3vb zS;tNUUf>PtS^NoBmsYFZYOS{Jo*}1@(-HUwoF%8x18{|kJYpV0p;FBKDfDOs9xW4e z)v3iZ(z69#-cA#4%ppf92V6tYt)U*u#4gKkF%`Sp=|KH25EU&WJ~Jv5UL^Rgno!Za z9*+XWt`bCRzl!epc#P9!u^FKJCoe z0h)Rd1Xq5EYbdm?n?6>_*hFZL?Lh1S^X%Oa?W zy$W#&Gqws5zQbOHxSbhWg$Pt(uR>gvjIBb1WUyBu?oh^7A%YLss}PqfW2+F??Cn*E zTbHp_hzsRVg(joKd50WO$y5F_@h`|P_(+!}>;zrTLBWQiDE`3EO2Va2XhoBRvNv2* zA^v7orrBb?zFe9w*y^mqe7oK!=K?Xng}V2o=Pe;+A1@;VX!uQ2{rD1?zAC8}GO76Y zOs!yDUu&hNT1l}!(SWsnt%ViUf<@)0KcPibL^dgTQ(s=6<^RyE z3%Y9F?radw-ux@JT*G?k0A)3wi?XKLuUM`RV&V+@qtpZbO&sdo-_5c9_<>OcZs~NI zG-ws;EOl$C^p|69vGjWm{Y5SKyVe6~UyTJyZJTk5VXYcYfdM1i)H4|ynHA)U8YaJf zdFN1Lv9vnfAYlEn7G5^gi!^W{Q?u&lVyL2A0D~OnlM1j`n$5S|VfK?6xEhmOvjy?zj z{FsAOZ!o=f;rasT{=qh2@eVTHzQ(5U{nmqaCx6TLH;_7>svoURGjlI&R`GA5ZA7ZZ z1yF$xrCpYXJo3yPaOh7L2}<~wVi?@1@W9{*0HzAq-XdUh`bovZrozTSS6`#VmTA6* zp;F&|I$O)4{+{}NT7A!^f4z6G3)Wimm43TKj$D{4TP^)t1U0Glz3%vXt?pj$jrOA! zRWT+^8)G#ut+(;$$0KO7x9FcE2!Y`LbsdhJ_Pcxg?o{{*9bnr{mjfV^w!g|$L01R( z$g9$7>d*Y0D`IEeB!aifzU>1o)q&lEwHPvDpeWx#F>Ny))VCJ&H;p(@%xPEySxtWW z4^>d@)x(RWTy<5^w9Iucau)g)c_YsY<`kUfQS>O?__3HO-r3-`lbBd%Mf&^Nni}d@ z@F#X1QXNT{Y;_T0z&AfZfGLz&DX3aZ)CdIZtXs`~n=}fYPmOR-X3!6xQcQ?;A)KEy z1u>;XNp-}&T!ptDpmd?=^5I@Y6vQ7;J@!SES%gVcAL5M1D8yH|A)5l3TIu?>jSiE- ztvZrmbsGfH-?DAyWKRx_=@!NmO;>~|COVKaikQ@>zK{2MzTg%Eemjq_T0dh7kt910 zHXpb2D%>CJzPZ;#*~^klpJ=J@110&$Lzg3H*stC`K<4mJO>%bLNJvtZN;kYmwZsWJ z%Dkc6yhBnnm6xALil!jnVNL*kfF-F3K*%zfu_;T4vJy>f3zGXCTU(Ia_}JQltipM< zmB{dr&NpLQ=6zQ=d2T@--JF{GfRxqhxP{MW#Eu{G!yIE?&=*mlIlwo!yVVIZH zNYhq!v0-DwWV*784O?4(H;R)m=!^&~0mp_nx3_z(zB_^cqDUFlzk!9Ag_ke{-!rNi zI6kXhkzCAzB{-=hveYV#&9oAG$Viz`5;@5xltiAh2_=zta6(DsGn`Noc@ig-ME=ES z&uK&+Sxu9QqT__5qG*_silT8!DvAawsi+z$rs6X5?o_28*(={C_xK%M=jeXYzc59| z^k)eU`ibmo3Hzx1Mz;YTBxrkz?1=g(-$0XFg#uNZN7K{^CZol`X5UHPOOd2Lm*nM{ z=&dFR6~)*`LNU8w&IXzu+X&Z^C|2|HNGOH{`IsqQPNG72y zoFO;7hfu&$5uJ+$mW}A;#F0LP4wj1OT$HeEM6Vs8xfe3PqJk<8z{e|t)QV6Be2-bQQW6YNqFRp;k!rp37v9<20OMM z4>Y=Sq4G9OfJUtYCTK7jd2x-*b#)|D$;>r+0<+Z6sMW#*4JIQmXk>D)yrE%|hMJ*~ zEoPliYE3aggUQGYP1!7KWCpQggho$KmzrzT%432ClaUuRGG*E^LZj!sObv}P6_}vG zWaI^n%n^5t(CC>fQ$wRn3MObU83RD`(6o5Vqpk7FZFFV<2E<_tF9@u}TN5jr4qgl|p-PU_}M7z=|j(0|6_$mljxwRXSj8 zq^AXzXH~JvSZTx<99U67EU+SW%Rs=&?xh7*VwDb98|nD~D@`DS11lUJZg@}T>6vKsK@kT>F5J^c#4sjN+vGL^My2d1*N1(nytAUL2dB%RTbsNtu)SMrg=0>)At(Loz4tJAJ}jp>E+e+=}9r@KPv>xzCw}N;z^fWs*$bJ1O)J zOt+#%t;`hfJ2N8x6kpu(LIUW39Z;MNjZ6%Tx-F85Xp5I-j6iqjADXVlD54lM2p`QO zAgh{slrnpgI}o+q6bc+s5iBi2O$h#E63(BiP9;<9h(3ycjbjF)6&yta;3$Wc9@TQ{ zXaJ8KZM_;|=v&XQv!=5aJq6;FlEzKFakW=w6ukN&<<+iGraZ*4Bz-&$T?SXo=&Sh&C1*j!j!+FDxMXs*;Y))J$? zzf5Mf_qrc8x=nBUptHqJBt-V|XWeU_=fASW|E|^6D(T|K&y_C6Bv_v-&BeaG6#Ew9 z3))ht*UP1gECOLfJ1CJ+ITq#8Y+{KEi6zb_mY7K_F`ZcATw)0~p~PJ2Yz$fpr6~v( zoObD|FO<$?DPAs3#`dmQzgjx&UXIH$aIIb~O+d_U+q!Ow8491A%Kiz7{lp+bu9U`I zO01#kpckcVRhHqCvDiYqM=A?s0+hq;_5%-MMfUgH%K-csDMC`CDadn9im$;hL$$)^hc=uaafT{y z5@*}}-0fG%XOdZ^(P=`<{ey0YnOK?}d&W$Ayb>XPT4YKfpSbo-hXTF z=DyNgQ;Kef-Xp{8OPBwov+rA4s?3u)ud#)c1E8Sa-P`eCtN=o{@SIbbj;yXQMENP6 zD`CC!X>t`5w$N)K*!kX*H-mKk^b8A5(%pxi1e)wOkR zWqD=!B)1A!XZR*R6qXd^?MZGGw(6YZR;hY2PmCo!i#YR%J>fgatrD2fyp!B2*dk>; z7dgqT(gOo?ptw*iV@OpgCAQ64Xh9VTCXJa-uw7JZaZz{Fh<{EtPC%!ba$zJi4xyP4nWf1vPWUAv+ zsPE=v32u=?pj~hD&EqDC-7@T+x)gia-N!pdVlOjP$UMUn(G?iBd zg%nNYl|bQMaw?SU`4cP-z9(r{`Y|_pLUA6f7;3Rcq<^7wWfbnpCuvut(6pu2NKWIC zm&{0$o+K}!k*2M@q(+*yvWtzu$Ll2R3fzwHuS*FjU=HA7k{9N%CZ>|+U>~NE${`r( zi~hw0OElvETA zR8mnjQcPbzNxQ;wDyfS!YV&Af-oP))a+SuulRQbg5}ITDZbU9|zcb~hI7zz_SvZuK z8Gx&kbCBc`@jKHPpQK%3cPDl207?DKm*vuAnmI|k0v_DGBrED6vx1$ZU5PTbQvEVZ zuamSZrZxE_?Mlick?3U3mZIS#?MjIKn2(uPRU4!MCUlBq94Bd4(w+Dnmc38Xt{B-n zQ4zE6A%9~88B$wh2vAFVh;ezt~FY?Mn;M{Mric#U#X!{J9ZN^n2fxjk)g5jhUO&gihs;M zecgp)es;CCFkfP@#6Y75cT3GRYUMF;jjWW9aAvNtWTcS+$BvO}^r(@kp+RR7hVayw zjJ%+c5!Q|o8a?h~YG}|Qi~$-<#sJV5(~QWFaw(w+oub>06i?EwTt6iRj8k`PX=wED znW=dM9jY04a7VAOhPW0WK(Vr zH+S?PzUk)Pe|t zAaBNib)V#Di3sGgNL)wPXvl%hqXf@KL@{I{Fv ztxDh`-)@EyxXcj(Iof(HsCIKDlEGyby*1t z9Jl1GuUMCy^_93KXMN>pmz+Nxc*&6n`D+WHqkaum9H9GT#bMgYajrPPQNfA>{ByG6 zte&hm|Io?a{$W>~JCHW+X126(WlP$)k}Cxt`B_T-hr(eMQEjPNBy&6yBKz^U(QP;G z@3i*zgPe$HOCvWtRdR)YP~{mE<3EwRpauC!E#?!I+mcSA{ME~|4$o70?l$yqk^Oi_ zcJk)_$Ic9U;8iWpYoX`#buuC^DDi)&L{*v1?bj8;=_tNR&WV8k5TBY|lvwQb8(WWZ za=IR(9+@#z4|$bF;IOZkAwa45YL#PaAg?f&4*jLV-bUxN!U9gE zd|xrV^^|R&8s96!zA-38v2s2Ve_jN2K2N5454-KoBd_sMy9a4Tdym}0tV3^+NgmIB zaUaII)m#jQ!&Xs3bm%hekqKO*JA4ajrsp+k6$%${*+oUZ9}}?Pb8<1(6a5xBB`V^) z!i#d z>fekv+b8L5GR13xiJ#j*d?<%Ds54qoLZ z`n@lZpC2LDJa2oi`ypex2m1$t@jh(#A9~x6*#15!Sl()_wx;cZCP+ynnDLgt{O13) z#s31s+@n^v)7pVI(`2gK+B$#(`cB^qo|ee){dS`V8Y25rif{M}Z)6}Pv-Wo_^fzpD zwD&9U_t}sN4UeFKt%GK#)9rM`2mW8ItE__HZ`ok*H*4@T84rB3MDf-~`!GMjdztlq zicF}#Jyrp0j(ZpKzg#jId@ge@(Z>T^Fiv;Nf2 zx2!)Ex7ETmh^toNr{bPa_^F?}Sg%*y&)C<9T-ibii209Yk?x>tii$J{J(G*(+Q!<<(P(FyN(M7-0VEWMu zssw8@+OLW&bFeWRCu|l}g&^beAme~c;5rS2faIVTahW_w`J!aJ#lVUFkOSh-?>Jj!qEaEx@gQ$Sh=^2<{|zkkAGLfk;}l&i z0UX%g%S=;eub+W^xYWAEX8cWG<}5Oq#iMho?+QG=2GVI! zn719gV=!uaVkd6t?fUN>+Rzr>lxt3^D1!c)0~R2( z-c<2=Gs)*$03*E^tUJse0n1XM$I0WmXximBB<@ zF~YSP4_AzGP}0TL+R@{+7KfK6xI||Thu5kRueH4JT0eTc*5mNfWS8j7;qY2B;Y`*G3#(n)ni(IUHW=M!Yr-fft(W8>(_CSd_(h5FJI6B%AiitAkbk6+!}iVaZ;5 z?eMi%?X@o-zIL>JhhbkieC=q#3)jAS_}bC>5Uzdg@U zc&lJ=?}@w_ORCfAp*q;7>LDUUJw&vqhlm*U5K(j9N7lSYp9)kO`nwJ(-C)+{+i$%E zgCKvSX!EedW#lD4?9SH? zTCK0QT>lq(F4Sf3AvO!7^*;e8n2DaK+5>AcSlCFBIzzNe#$}6ea&K>QE6+A{>vk;% z>$XGTz5+IH*EI9?Y=|k;;A3+)??~`UO>dj!K6G7-A7sOZX{;-bI;BNy{`8`_6kES> z>uo?Wg-O%mvGg*%XY?JhQm05r%O!BkWA^00a#0C&)j%eU-b0OG*NQ30iH0#qO#oj!N475JE~Q7XKDUCCUe~lV8Wp#`WQBUQ8dcYC$F1soN zXUdH_Qd@^El%$K>;HM=LmSaKkEb1~;DFup7@; z$a^+7o^-xlBG*@aHYuia-D(;Z;%)y5H-fSOqga=cnvDk1j)!d zE32Dp-pcxV%>@UX{T*0KAMWimTV3$r0>32mfVnk=AAzG(_z^f~g&%=`mb*+)j7YV< z`jO4W9$%C3@X_<+S`e+w^Y;<#iST;&eBAT zTvi{b1AW|qHBwVW{I)z^o&Oh`#`|5kF%$?x==eHWPW_$24L;l?s{?$#E}*qiQ>~Ez-}j(>BT$D6b~L@>>rwSL08S&y^0F;3D5#drN10= zi>2Rl=&v~B>9`Kb(*bonHm6Fn-xzDET{(_vZ6Pk$SE}Bo9`J7x)Vgo^VZG3COQ+MQ z_U&keZ`%GM=x5K^>~JI^7^({jx*K zQLtn$(!hmIlvFEy0;s@Q z+Af_!uD9k6IP~KpSx{V+Vi?@1JE4K0idBpxu;nUXbo$9mt%*TbU)QsJ4MU~A{dBf( zCDQ8qY4tsu{`KC$E}ZR{!*34~=&@`)BHSXVZM6Y*$KPvp_j+%DV?K@FA*%^G8#%Xz zXlE>A&rnoKII0Q-L@-h&Pic3`k!D7g2f(RqL0aZ|K+Jl1n z@~eubWv+IFx!xsj*N-ApGKKpxC`F30c5qr#Wl$X4ni3PUqDcR}w#tS&7W|35231EA zW?h|!UE}s>X2^n@11~8i391$|HUa_je^Fv!5cyPOs^e3tMWoem)VX%*Z%%O_yL`kod4U~O}twvRTevcN9MZLz1$z&cNOL?d!IXiFU z5tZp4L`Fb@K<_edD7OGmDVoa5mn=n7%Rc+q=T<7c`yTxu3BZ-iOk*2M@q(+*yvWpEH8zy6zU2NFe`nyqr@&@Sxc7q?~+}z&owfgP^{)<}I zAXtd&UkCkIX`!aoFi%J&}WQ|3WJCk;upqG3uZipD9aC>p4wqH3g=zRoPaQKYJMIG#jqeBGsVkE6e}?^ z0L6cu$km`095cl$Nfaw-X`wjAL$XAZNe1oYm`PquA{j}2^RdApNrt`EF_XNOM6yye zj%Ji$i+9WAb5X*w5xtT~bfOv-C%RLNbS`RGHlkM(iB43+;)veG zCrt9VQ5-O(JYz`}u&xZjQHv}<>UTANJ*~aSS*LyD%rcwG?ZFbTQv3DyylcL9gP#HQpXVS=<=v{eZB1e1^pCRjN5f66=z{JBK+)`l>=;1;Q#5npF==UA$vlQuG@Bq8|8}`uHX*~dlE-4q| zpd!ykxbE6-x7F}$C)5V3lyVUaQ^P7M@Y*r4C`9;$SXh7E0;}v^fCh#Y@n|5lg?b|Y zOaRtKdOpBPQ8_rUqJmgpMQp5rfR)`#3#`N{9k4di^8r@M+rfbq6~qE7qKFLytn6M| zU?o=RfVGjH53o{b4-TxTAQo5=Z*3r8W%tqoE3rxktc~=v!1ByWRv9ae7=r^VDu@ME zM5Y@ESlPX_z)Gyr0c#^YA7G^kWN=_b1+l=27=8l*E4!B#Scz3SU~Q!51FW>83=XWQ zAQo5=?QkGqW%tqoE3rxktc~<+!2W3w``6;QM$(HJ1&uf8F|rep9vh14b|a&wnQVg=kT}XLd=QgU0yh z73i-P@XbI`4cG34?+qGfv{4E;*VpcY2aL_Ii6(gAhXV>8*h_|WJA28nvcgNMzz#AT z7QenIU5NyN5wwOCTK4#qBA6uoejz$HB)6r*EUXFG=?gDy_L5;)Z7&%XKD=av%LG9> zm!aINK9{A!O{PZw3ZZI$D2Ib{S}r$Sf6^fr&DHDoTa^=mPaJ0tDCGamVxmzLK`PwU zK$iGCQ?!LjhlB{xTeqC#ls*?B!=mM`q(dKKj>P@t?F?|O)!Nc;Yk>lAQ#b54o>U|T z{|*cX!N2bmy+oxj3cxnpic+PJGlb8X1<1fY>51%wBAi9fog@?ZP73`4)2(PxD>KFW z&Wy-EMF?+ajrmw)pPF2y}=3q3LRjB1&BqIRdh(sYfZZC%Hqv zVazFjs0fx8VMPi4WD?Gwt4?LGAQF8PPanq>L`yje3&7zHt0}4rse>&%My=vFgUG^& zTD7P=1GKx1+Qrca5z|IAgXh&OLGqofapC)Q)6~%=_En#8C zy_iE%<*r*;icPHR{G$r;4><{fYyzt&;eaxI&&THh$Pi@x+r@B{^}j8UzbVuW0o7;1 z8HIQ@5>rXT$3v!a+42;{$Mq?*pi)N?`-d+)MN|Kcbh0=I1^!LCpF?&2zluHn{POn7 z>ix~l>Oys8tF^GQ+^j8ZR+rZns%y*lTPrKAW@{rc{`|{iW_zz|2tWU9RXZEYVKtbEUc1x0hnyLR3O)&E?WX7Pl~>IF!h!9E);kHnGHo#1iKd zOUxvem`*HlF0q80P-3ohHU_PQ(i8+4PP>@Z7fNTc6fc)1V|!PuUoD+>FUO@MxK^*0 zCLq#xmPn)U$*HWL)Jw!&BLgWG^(&=umlA7$yL!D;8nEs%kTRB)lxk@->n97486pE4 z%$0^?d5Q(D&cIWJEXAvkl)=sprZic=wL=+NApg2TerJT7`S4+Tr{&$>Yd-OoH#gRR zk!Qst@e_hA?Zb8x(r-c1grYl7uHd)3umjya*!9}*mdC~KL75qHR+K^M72xp&axU^1 z;`Ps%U$EQ@#xGa~hKpoIRHaAmTY^^JC3nQDeyh{l>o&UG#*@djZVSRfi6=YlUjIR> z4?W(MPk5OFa1n>#i>T0ccW+mG;w?chgEn^?{oR8dPh{hOSB`s)TorA~S5>du+B)cV zA*l*3HcO^OvHR^tuhDFJ>|a&^b;+c71oaQba{5hPy?i@Fg%68 z=QMy?Llmlhn|x7pBS$=;Z$Wz+n$VG_$y1^|E0$2`JaI)Kj3hk&jziY@i^aQq^j~44 z|4!{TjJ|yG28{k|t^VsU`im`*_eN~Flifd3-4ghF_>bQz`CfrMpSmmVS0Y(yz~}%BjujSu#Jj7s+=L?9avgLw+pkwkf~Clf2aLmcdrejB7dP`7z8zj zzNVa@WqMz#1Bf^mQ(B zLIMFp5zE#1sM(~nY2mM*g&3NrOMgBF`B46h96z%W zvNfamxQyqc9fz;V?aHEG8_aE$Gbv%ym<;nch%imN*ahT^M{;^>S5QpW0} zx)*dp6`7w&+~vB-TZ;J?eT#~=<8jT~AXQr1;`^nwhmbKD@(1O=@pq46Q=8%N=YII? z=ZoZ~IE{dxQ5D?uh@D-;rPT#fFNrtxI`wN{NWB`93RdRny$1HwxMDK1?;My;?dJq@ zU?sJ`&1eu^0E1`-ljj_)@bzF|B4AT>+kT{tljQ!Z+ltbZQNqpw

x90e?Ck;gYy{oT3Pe2y-Udxj*$KKck zd{p9Dr&*e`WpYh?=bP;8AC{mv53a5!d;E^$eC*8eN*3k59Fndyn(s)6Ztj2VToSK^ z(I)bCmZ8L-Mqbh2l~;{>Sqr_SZxO;jm2Oh}A1Vjb*9Dvy*V}NCyZc1+Qj^D!_6cm$ zzBX(>^b0v*nWI4uK?+JiuAvRKy#z0y*l;PST5)1q$tyazL;p{QxO@OKV%&*4D1i?> zr-mq4gDIbBdO|IMvns-;m8l5wx1jbyN>MR-M_IK~P#vdw54-KoBc8nirdsb2L@;*1 z7%wQ{DXJFtfhStc#bCs2yRQTrS(*NbL(X!9aJ}yEEvTPfih`@~nTve2sIEUV?$=W8 z6=SAB0ce{-wQ(NwSuq7+PG!c=&M<#gR{%V6sq6ZmGxTwV_hah0fx$DqbB!Z_tu>mK5(ozEMI-wNZ!vS2;_D z?}8t+J5I)GBjV9zasf2wot@TJzx|kLot?%5P_Em}o?9S8mF>te%w2NZv~N3{Gaq

jAvg=YY~nP<6;I=j5~pv~RhYYh%UH+3^eqTd3#5i-P@6uHr&E1#IB}R9$?!#*vc!dtH_->4*qJJ44K>TI+UHuN&g>G6MFGHSo9AHwm7>i1- z-1(Cq{SAlw@dR1N!rkK;TewTI#qW4+b*&0Uq2+b=3^|34>A*5{mYhZpz!xm?hVu; zA?i8_A_Wg1aS1oJ3K8tXUWK@w8(W14 zYhkZKT-%MULIiNIS0V26##SLh6+ji5jN(2}zIc+8z<(zG1^ERZ>9T}5ddbNv=8Y(d zKQOc+c~xr#T_=WCG)XASvAKxz&a#`i;5|D*O8&VgnC~XhJtq+0+TS3(wZB1hYkxD> zUHm;b?Ses^zE92tVt@;E?+LgRBU=GpMl!2rWia9z>c=Ph`l_T_uw7&bt*87-6}V{_ zTEY8ct(8H1wTvy`BY&-hK^&WInrOf$`&tVt3Mq?qlm3Jjs}o(Gly`cy6Njt|x@z8j znh?$l{wuaD)_PD1mBo54%9_eNOwIREF>%38QuQ|FuB8liE!6rQq^|fOcg0CfI-N$f zZ$~S9kn@XLS^=#G^15v-P|8q(lUHlia0(0<*`^*lh6_!SS+nh zHwakZZGaNWhI)|(F6HEbgY$+e$_23B1lMo{7kWp`Ir_t-Nn`Z9p&fZr2Wz~(K!Sr52KdyyG(6oKNXzKGMi;g}B1HggC zvLeUS8+4xc8o;}Fzx4pDhglCRpYl(M&R?Gna(EMVYlHvCuI^W}^`q4R-~1bS9GMP* z&|1ybxB#k`QwJRS(?x<(iKZ9^cj``PV5njh{p1vJRlw-Jm+;Pc8NR^k;vA=-z|cgRQq0c{JmCpulELQfvJjK zV%iw1d1<}FNk1Mzo4v*OJhbkV|Nc6h6ob`!pAN9?rfS|NZGV-if~t9v{v&X9R&zV) zGpW~C5j!VaC(u$I*gY7VgiIu8i3*Bon`z(JK-UzLrC@NyoNBFJ^w&UE6PQwaY`3lo zs=azpP_F%vqG_4Sd^3iHK8P@T5iFP{t;F`2`ygGTKLwuf3K-q^v6w27-r%;Am}7z> z{e5js4RtK|6Gzrn9Z8sMb$--@b3UV-T|w1C#zr8}Z#mecQAjuBrcS0GKBcfmyAaM# znu3_pqNK!-U6~GVJwWNYYVSovLHq$#))!F%8UHU|U`bRTVq8)1I3u~il^@t366qOt&zmXu2XyG0}%X^?kh8^98pU@LU46{fsI2 zmh8M9vQ3H8B-1BaD*O;!KJtyuqX-&SUorXsnZrXh$=R8Ua(a(yDLHhMc|*Cm4yI@- zFaN<5O~DnMoB;d)OHvbnkiRlxQVhbwjgcxv9$&Hs`F|qk>MdxbjG&K z9=YY@xdj=Tb86}XY-!n&2TDR%%4rO#k&msdzB#M=^v``%5+Fqi`B!hNx%p|WSk&L8%=0MVENQS-DF_XNOM6yye zj%Ji$i+9W$Gp2*-lHIdT$ybunYl(!jh7l4wOW{9G z;vw_B9tR$JYT490qAU_7JaGRr;-R@d1gh2&(ioFfn3N3iJe4HSVaY~H~K}4g;-dB-2$uZUVsLM74fJK ztG+xY0Ba*XA7G`Z92{6tK`gK$^29*E%I>8FR$`ToV{N481FV#{g99rnhy_-};}{57 z*}b&DO03cWYa=}$V5QI=99U67EU+R<$w0u$?xh7*VwDb98|i6*WfAl2GFBQf1_xGD z5DTn`-7*lcvU_QPl~|<%)<$|hz)BOy;J}IsVu2MAZ3Y5Xb}uck603B;+DOj_SZPNY z99U67EU+TZ&p^P+?xh7*VwDb98|m4A{nH}8ljR30;#!~Z$#6@;8w2wB8T1(0iAawP zMRmK8QhCsQJ6VlamDZa`wWhK*RmxP>rX85d+7>)hS(^+um9>c_6cz^$gMT^Ce-?y< z@;$Rl>aZ`yC9yz%wLmT@wOv$W9%r;s3OLu-?t=%6&9I3kcp+n7Ed~$lCBwR%y<}Kf z;U!h{29bG?oBVXVB4`aOw31?6)9)9eb3<}lWNsKS3u^*)`oc?_y<}Kc+e?Op4=)+v zGC`1z8j2z2b6G0fWbEp%5ULnWASJA^x!iF5NrzlCSFhi1RZavxaqO#b;4}KPxh~e# zKt}mIW53zDjpr67c`%E-<*uYdA7YM-v@^i9R%=VYt;GkxP2I5Hcv5i}{Od9d-9`Vt z(lSVr^%CKg@KPvTx6hdc2zWgyquYw$6+L&7OyD~y^bbt8qD8Iv4hd@67q`5S zpgv#+6lX&t69c1ei=-mj;;i}RrmHcED8>xJUGWIWs-_;L%%0>9?HZ#vBPxQW zMTnK)PbT5~xoVGOV~icqCq#vWM==S}3XY-yaFoN^iE24@jD^RjRUAVQ85U8i7ENb> zcDGTxII)CSw=yeyzwVDxxgj`9<{p+d?>` z_n#EV9~JUXUSO^oGW5x)7TPQan^%CNtZ6T^{5+h!)>z?6%xz-D{rbzp};u zUSD3TxMbqN9=}P+YJubu&ka?k2+Chno%CRVyW)n+XNGx$avBXSbiRr`==Mqb}2_@!AXJgP>C{1Nan7B|n zlcjjMG#T5wA|#_c9iNQyTD@ADfSBF3m{3M3Q7xUy{t1cw#2`YhK$c8OtfA_l zmf@4J*iWMIQQ?!(te<2|sySC0j^!X0a5}B>Q(u6jni+~$AtbDQ@Iy)+kJSt<3}tA6 z{P&dnj|K8+oCkr++SEBR*Vk4yToy;0r`qwBAjJG4nd?GIE?c6lXoActxxzolkQ;MB zeqv4i1g)sFm|AFV2;I$dRh}~>A^F&uVNWCZNBrmE>=g71WJGRv`9CzkL1~P3VH}Q0 z{8e&J3nM7<9p>a_Fho5vW2kHRKK%;Wf(Cf-|_Es{7NZ)!?a`d}$>!y-lWgO+nhi z91?eaD2FzvGhrw$hI^n9q(z*LPu6*xoc62AZ8*AQOx7s}L@-W3ThLc}UTZseKER-S zjhKXVB!gJ&aubv9P-N)Vn2~IO_7B2F-8+K21`8^ z0z6=J>Pr&!<`!V^StYNXfMdLHZO&U<2Q5D6^&-f&nw)m089` zoDpyOeS>(Q=lRo!D(OEDv*;nR*kKc$*|=7Dj?AzlAR0qQ=%Jm5OleoEn8me8hSy%Qi9)l449bULb4B^^H2Bd1CBEnCbwAnmIGVS;B8gg<3K*s;7twQOu%;~ z{v9(CTu?8U;bjRP4Z}-EPl3euMHlY$8~xVy!A@x2&>t=m+dJ3xg8D>LpI{pFa!LJK zRlnBMugj2e7A2vbX8%=A%OtcTq*!wtu=m9#piQa&IgpqmE`uy&yrm;fp*TEJ$(Ts? zO_oUZbGKh5pGjsFNc0Dr=KevKXRvJY^uUan_IkZm54fnh30Eyk{!)TYY|54)JG7{C zAomlNbhGbUS~`UEx~!6Lfnzh|7KOaXPlBFjkX(c1;H+h5>iS&}F1M^kxsY@eIVrDN zWXibf&k*vjE97@Z$e9lxws%_I{k`TBZ*6032~3Ey;*t2dv$wYoq8y}ks|(rA=E)WO zb{8_6?H=rUZFt*Yxlkd^nL9(yiZUogB|N@B&P5(W^1>PO3zomq_yx;id6CSBsxC4|cpqt!}5a!!1zP$W_s%e8mh)TL;}P@D?sMOQuD!`|U=r(QJC`UseHi z$)tGGmi!3G;-<-z{tU1(J*^p^F3-*wYG7&OqV+=ZI2u5$AqrK$O};3)kt3eax1c=@ zP3Xwe>6N24*mMYQ!%NxXlc6_fuo=@Es_iUth^eEVvMkgkHIdv~}z?8^Lx#$%u=g4Uv z^S~TK!P#g+bm7fL|ILG)Af5P9>WopwtOqaVO~znhYrTfO z1`N!b_K(|%KhB9bw>(**?4+RT%#(2<;{W3I$A8V!@Bx_-&B%n|I;FXM4jpQJ0}!`v zXn9(ENABp0+*m>U^&yR?T9zp!du7m-%3$E9Z=Hf*;1exKiTNibaz@a?YX*ZImI#!J zQ!LzCFsBvFlK$YiCe5DL#-f_e`>1w~oZf?@RdjoDZVGrKiGC!#yBG#Rjis+C7ig8< zm+ZizYyqbIDfv+jwb@2p z7xo~>&_ss65kLHS2IL*J+cP|X7%?QW^PIk%oqwo6VhA*>GpoVJL$FngqWzpB(kHgl z>770tM3VJjjK*EbXl(CwdtV=NC!3((hdftuE@S4#(I{pqWIt?2SDkL8uXB+T5^OLO zO}PdiHPCJkT!g;L|<1s9GeJ@#SezWj9n+?=FclsE62hG z$zxTv8<40LkyFoRc6qsgE(f~`>kC9le1RYP&O4bpsd(u{bdiOq^32@M-Sbx!i=!^V z=!TEh<1${4?%KXAuPckHHdxz%>_8s3KA&n}wiLcJCc}IuQazT!qwb1{m`MBwk@~O< zx0QZP$f@FQfiz{Or?=fppn(nBOiz)?7SoH%9VkEw+igbHlq7|TjKcL@A{Q+7ah3+K zxhBG@j)yjV1M@Z8^~-pwDHhi znui?B#b5lrQCw;>i0tl%l?|UKuM0a+_?wPVXI7(3J}#8G{G$++w|F|r5*XQ$cq6+~ z>n7N;Jwe>+TUp^vFlmb`CcAJA%-Gx(n%BT$&HZaeM;Q5f96FhAIltwDU#IvNV1maN z$T{&9KMcwu4?-w=j+~Jw2n;~AMmq?_MTVkcr`6i%vpBreXM%o>5M85(J~N9+ zvHa*2U4pe4?N>~q?x<-nv*^+lf*QUEGLFbJuG2yY>>TteE>i?4pO+I)2zv?GBr&~|(yOU6hwk9>rjE~>vwXb; zFw(2Rxx|KfOLS#8ut3HxaieHvEf+MiRz?rj ziW{tD43_B1a9}N4!CEN@tma6-+QU0m6N5D?IfMWc443Fk@Kf47s|qIN!9rZI!qtq2 zD~36!nA_Is=t7*k+wJ5ySMvvE89A3uWB|0-4UaMBT){4Swee`&( z$KhpcU!pU^;k9PPYkdg3&|+WLmCKDq3(9GZW7pV8Hr$oh^Hu&OLIQqa$z6M6_}amU zL7lK+V3vok9i89d+E<3J9UXY#+MC1Ij?Ra0?W@Drj&{>sQFj_V6WD{x~Y}b3KMQ zD0H(J*XqOz_lSM0ole}v`B?u^351P@eNbj}{Q*zfu@v&MKCkA39`fUNv@SUvst?Gi zyWoh^`$i9prNjq%A=G7mOB)8H*RzAC^7KH|$Yih}VIyT)VuG^MvLVo=yW8v(*`_Jo zZWJKhF43O{%U`s18-{p$E;K3h;A3$&??~{IUf$0}miJY11r}_`SAxf|fD6;9zVJqq z60!N2MG?lQb^DFC0L2s{O@~|g^I(A#SivBB7$p5ug@ ze;kyjlB@7+=g|Del;~X4W0hsI@M9fz6!!P_fH#TkDX_kwAzh^IMR&^iCsBchHy7+n z{obJ0f7AngV;jrgUx$Pa}Vmm4)erh|AJwRdbTB>fhK=`F;FRK1+EvP|IsfX zjjwvarX-kuHlhpmoEo(GXF}?HC~BUt!JKCXR7jl7Rpp*raBdbCI&@q-Bs?~Y3*|a4 z9ul9L#f8=#7Y_*#&Ei5GkBf%{Xl8Mtr^m%Z@*}gjP~hX@A@PY>Txj-j@sRWYhA3p= z;JMV~^WdNx43URK#GED0VEnWjHh%gNc{h3_a`(ZMwoq|)P%(k^dYkpe#^y?WVY7XI zX<=o(*Ij6D^qLF3wfoEMmDP=Queq8OSnn>md>`EPERpp#yfy6l>Ibp&d`FnQ<+bG{ z9}IQU#@c(Ka>9F##7IdYT{F4pAHfC-yFstBy}1p%Esy$l(&M_z*5|>F>a|1zFj8|= z<9QMNaqA^e$lZ>qQW%Sos-wz^C)Mt)*TF37mDC}|;wltdCRA7|vcj#_T`=T&J)1jh z4B5goi~QR21mZD2g{O{lzBD!rn=WOI1eQtSIXfxD=O2wd*RdJ@?BcEI^u2z2Z*Z_b z=+V@sKQSgY(HMXuuoK#M+6jq+f9uD=N(cgQp=(s#xJF6-Wo#o9XDt*qk(knUA!8e% zqeB()9imc5Bj>S=kg%XhBN?!{rB2YXjSxF1q+w37A=21J2;AY)neYw))!0Tz8Ms}% zJyv8;{+3*_M2t~AM zD!;?nMktgPO9O-Lo5nUm1MpoGDM6|fBlBZmBlPQKq|MW*I>#155dTTr{fyQ^C@xa- zO$vhBz-S1W56uK>lg!3L=au?4(3EMt`Pg&_i+@-|5XpMX+&U9@1%a80!#1fn)1m)6 zAw5G>9Zr7~hC^xXAjTF(NQzyIh0)K(Ss1x8RQ!0^7k$NQU*w=oZ(?aDB{-35DLL2S|5dP`OqA#n=b@DkcAMsLW-}M#ya7UsK+) zmqU5Wq=A4wkKC@(@u_);^2^ z%NAL%{kPlw?e_hh-s=a^Ojz!)cfv%GjCaOv>CgO|RdNHHuR=bPy)NVo_j!ihm5t`=IAPOpZ`gr0c9=pl3nK0VaiD(vZ zeE&)|Li|^J|Jo#v85xz#R6q(khu@@m8GB5ri@aO2pe~v)0?2&_R60gOIE?`WGhjZ8 z^y5YevnzqVj=DtPu1?x=6x3!35djor_UN)LYr(%MktXWWTOyMf$^-EtWD*kv%+4y@ z+>0-jRTLAq@tjkTA>yf&9q}%Tf-Fsi?jAL_2l|XJ6miGXt}hIxkpY#vU;6@hFLBM z%H0$5)H;HY!GDlGWbo`qDo+<6ER$yG^hu%Mq#EMXvbVP(>c{qj-TiHr0!lSN&3g0X z$)s7QdEaw(@{0mdnJ6n5 zcoC$Ph<0+Kx<{@fzTxc#oF*#mBWI?P57N|ZPnE@?{tCGkZjU9-lWh(gn7Ty&GaRO& z25fBUlc6#u(fXjg87_xA8x$J>@AM*>Vky@Z{4Pcqj%k&h)9$<0kr{#)qe(V0)g5~i zxF>TA8F@D{RUg&*OwjL~ASZZ&5brF{EWOhk4BlZ0km+M0ZwSuv5sZaQ?`fe|7nxzu z>aTO5L2jx1P|P1dl2HSfN{K5O%hn+2dx7UBr^Cq8WanvvQY^Lt_w;ycpL7wpa zqr-!v!vXMMXKIjYB4h|%d}f^#a#PkvxDN1i^8P`)-`?5jLB8i*AYcLrm||J3y$f%i zH?v+ZuP?D5!Q``xe`MeN)#bH~8l(*p#41nN5fFEkTu>cG*|Y)ND*c%fSwfTH?FLNU z&O>y+6yHa>$Np7`%qRE3llvr3e#kmWMe<9vE|%#=awU#^GbU6(G&Y|#)iC6ToyL1D zzddY`OXk%E!tb!2tF0#WZ7(raU*y^+TPo~kX}P?rCW^aRerqxuudflhSw5RQ#H~Y~ z!4fVUp?jz7ubMAQ;d`f|2VLnxH&a!jsC#EM=H7`zC#Cex&7NQfGK-)LJ4VKXOh~(< zI%&o^Q%Jbch0Xi$v+2U-p)t-XPN>o^I}BSxbSsZ|31ov>){PZ;I%UwU0~$t!`s^d2MZ_xv%T-}*8fx^e}2MXf}0yFE8aDEv6Fdv@m*-H?9G!K{Ep0r!97cKL8vwY15FD9@TxRTt)3*9ZFrB^^ z?~~I41Edm?IqF5Xv=WHQl((w8`ji!~ZS1H?4J$=xa>R<#ZE}r{Wtrtsmq5sJG5SaoWkV~EKI(y*S$xe4TEqscXyKH| zjbsQ7)uw95&{N5!2#s@X)j2^Y?@LZ~P-OU#=E??Io&kAB?e>NiOLl%3UDiVd5<@_0 zv_dmW=(Pk>v^bP@t50mF(>r}Qh!CT3S27yg+~8r@Xn|nZQ^;LlWe9wHX|xX?IU`_CPHKJ10{cT)tjKDDuDz^ZvMPN zWmaL~g5-WL+YLxm`x`Sn2`e|In2!-Py1ZOKmxEn}^<`OgSsd?{I~heihvqZ%{8h!` zsJsg(H3eFa%Xl+!cdjB zR{Av|0S`29ij+&v8RJKq#u+IhYbv`?Gul8C4F$rgj)yjVN}Bna?fT_ec|jHupM&YE zt1Zlns{&p}6Ixk{HzUP$sw)=G-z3$$ViKy! z{9NKK*Gm>D%zr{jROG!w1RD43*C1WmxZ&cj37a)znGpXbJ zw-z8}L5AYjz_dHAXOl6%z5xjXUUgAL8jcJG)RXt%s8veqCn00Ni_RK9;i&PV=ZQs( z1)*fnp@vK4g@d`1i(^2e52RjA1p%Ns)xPqVqlf;*aS`A zlc-79I1oTC!`X2lfN>yz$)yscP7E%3=$`Qy*SfGj<3Iohy@5AF&a_yrEWzB-uaQd( z8<29C5P4_K@AO8W7Xy_-yZcCnRM)gey0aUi($cpBSi;{laAtsZ_n3Ri;<2* zgEqvqM8|;uxQzgF?-wrq=finMtG0w{KcYYYYwJrZ<3Ir8KmZ4$$)on?e_SGeJxwy^ z;9FZ-srze<`WhIfpF#RnFr7G0X3zt$gor$1_9CHD%u*!u2n-;+WwQ97-|lYr_6`TW z&Qt0)eD+vK2Ec$EWuu0do*_5&ODZ2TDz9?1LeXc*Q_-SzTsbPdNS;N7aifkGo*>ut z3!Dq~ceWq)d{pNuY3L&Axu{NkV`&o)`f!W{(UGC!Yh*d~dm1-*af38-yvRqfN$ROz z1sPALeaK&Xj%?`H0vZI3+k20C{X>=~=Loz2m@S_!z@CDUP>npPU)Q8y$c2Lj5cdi4 zw6&};q9V&7lwKgu=+dggqTb#8E)!F661ev&bmRr{oGz}rDeI1-q|Vc{x~q`0Ij#zk zy4799ax$JGovXWwmE0#YftS46?QpQ3R z#va8D7u5D7XOMa({)NQw@RGTg9PEIH5u03U(utz@1xqV(<{7Qjbv|Qg#n=gDIo=GK z-|eq&55RV8npBk3AB?lk5~9AbHEeu`-P`yMTetC@8Lr{?!DNl)xBLpZ5G)_Qsrw%c zAWL>|9ijLb(c-1l_7GiL{aCMMo=UofOjhFW*;>I2y3tA=O9g8SSm9-~kVmcmx{U^` z*D_i_7Vi#`4gIAOc|wmZ4hqgE_2S*F_5pY?@_%U716>c4Thch$ng;Q8rSkWuKpOuS zCHlK1^0BxML0}TzrBm_)y{1dEPi$%;kCmGK)j?dKEz-SBJ#oKDQ0sTUcGT;Aqi4Y_ z9Z#d%ccT@S%l;O`fzlKjJxBwTR`F5Y$3>$WKJpffTuYDL%Oa7x!OqXpO|c1@Q!Fh{ zmj^7+wN;~F$zG&^ORG4k?q>iOCTCc{zzzDW?&=HO?j3K6{k6^ZrJynT3H?rVwo3|w z(hR~o;FmY4v3!!8^gBE4!2rzDLC3SR-)TLst7M<}&RGZ1wf(L2V8(a_3-9z?9Ji5EH-RZpq)Gprdf&Z;Ck8w8?3PzLYp7_&<9Z%Jd9=OXE z6B1{FeiL0IQZ+7s3Vi!h!{7><{OvMXh@TXpVen?}hMrJ8RjeeiBO;jS^wXJI6N9eb zZe@ELhDyKtnQZS$r1kq3_4{o3w+2UGfX(zQmf$`s5eelyskUyIyK#7H2J;zvq@NB! zZR-oLcluqX5_%Qx!ZiMdtQY89q+=(^REvcZ`n?nA*0*^3sJplK=AAoYEvj#~|Z z4L_kd?%z~z0Kr|Jw-jY4$ren%AamMD&rfK0!~>neE5OO3+yq6qEOl zP<^X5imF^iN&)JlT_>G3{VX(Yi&LVxosZ6@nt^0S{!4VrsPag76lOJ9v`W zM2Y@Ni9F_N0K`FFc6(%rFw%^;wU*>NNQB)Qc-kxEJnK=LBnn0`v6JGLp90O zd7Bvx#%y3A?%B>Zz6W|Vv=NwH<_#4V+apC&MaB3?(G=LG7X;u}lK@=F+|(5nVD{`S zCdQd!$7*Zx!wOAot9guVtsY}rYsIydIN{ebw`E^j)#SCcQCL%7#X=bCGG02ywgQ_r z_sNxNd#b{WN5Um5T}5pgY1%4k*GSV=_HDy84U;j|@i*!`gW!Z^ejZeFp|;a7I9M^8XPqsnOClkz=Gbg)`$&G7P*QtmB=u7}W+atd zO-<2S>XPfJOK#|rnn|*vR7V2Uq@qaEkW>^6Q&LehPDw@4KqVE`BgOc2Ci$JM9Y$`? z_sM%Lag2}bFa5hx4A019uW$ol|4<})Q#U8aJ&MKi+Z|w*ABh;v^@W#nYoD+2^vl*E zdvl*2@6mrSZm?!wIGK!bJ!X3ANsCoa?2%9mN8vG3+(@EWug{TC3l7TxpW|EspB%@v5t~MBwWVl-$Gs&w-Bx^=v zG@}exykjPLEs12!G36mS`GkU-<}s7Jo?n6PX_uOt$k$cDvj-6^JbA#7MSqMM0CCo*DjMDJn^ zlT?j~n{Y}UOR6j%EW-zns`wj*y`IrtL@(!-cd;wHmbxGVL!&}YKu}Dp5^qFA6VrsW zD%`bivQnn|*rZU^h&6i2Qsiu){El&ghFXlk;uf$mSg?sIvYrT4S|~KaQ$bl=rhFz4JIQqG*$avBRpn~6B^lKR%rCeV}k~hQ4}=76X-ahku7G0MvDq;&|or( zf<|~a9Vaxh#jMb1k%A2xOhyi9wrq#D@Z3rZO-i-wZ(2`n*;H0cw9LVV3GTlIlQ{m) zipNGK4sR$tWK&u3&=Lt79=QJ*@i0suWUu`C(kKA=ZNnrk+*Sk;x0X%tbx3)LSio?# zF3v7)j@U8D#_^_{Bo0huTbVJ@;tVX}3NXPW6f#e;vAV-y;$m`><_Y3*ZSw?^PzWZN zIZVD84Tp(~yG_Cbv9~r%FbRcVg1N#`W8z|KlQ2Pitql`QLLr!7c5u{~xcJ#5Ob|nB z!vvF%5tCmn5#&Rs?c3!R6|r!)8Tk(vY9Pi=c9MR#gl(20-3uOIQ-6ziL0Be-oimoS zHB?ZM?;_kZ?RU#+%1mBEU9iesE40^So(^2r^Ma;lq%*I5fY*~d?ibC3(v?%J2~81oh|*D5hwsRb=&Rnq$4pX z3q6MUqVji&I#KbTO$oQfD@vC_#t>>|PS&cCg?yGwW6u=&J)2jwsFjK0l`$i-Pf^<~ zFC?fBrUQz+VUUS|(cU7dh;|q(#%$;%`d2pB7)_LR6&V3p_0*%3*^<0Of7%+o98nP* zEn-^s8#B}M?HGQib&e!zMljNT!ZE@5unbD*&24T0?m?aWVLNqHDRB9x#fB3;uw)Ni< ze}00Op3LxfqQ(DLdybrKA8h;Ey9YbHT^2aod)9kqXK=XdFWui*UR&#Q7dqYLjfIu^ z()vPssk5@sS?{g2o6GffW35xWPA;<75Bj~$UccY#`rY0meiC}+m)>}*_B^??x!-@k z-S7IFM|&cN*Is+K=RM`!^nLZygM%ZVH~Ied;g-MI-q~Tne^|iq)tX19AMA4{OIFLk zn|ykrHcQCi_HGZ7GJkcy&z?Xotm`our><45#s2zO?5_}&(3M-NRjpiQaSJ1igG%Ky z2_@zdOI%4TaXGQXrNj~!6H8o3Ea4@TxK=qIgVsW2HcOJRh03`s#jBOGvArup0=Jp? z1a3E5^~y9v`pyz*6ka)#^_3Q6l3OLY7>m{oNHIr=Gr(P|1-a$&)m;TrPGu#f4q50j z0+ul~-L=Y8?3`kOtJ5+_kmqhPOYu5{mUUAXifkydgDgNYy7V0)Kc6e1!t&bklE1vM zv^q|xz)ag2`z+HtbXY{k{@xx$>>npoVA(Xr2^Gc(6@r_sKTfDHPN*>8!rO5|g$tI; z737t-B%Z(`4I|ybIH3Xy*%J<3I!>sNCDH9Tp+byKJe+-EoKV4tGtWe)aY6;5GzzDQP)G+l2a1v!BqmgF3!YAd#o*N+Z61YhkMk&q zJd08bR7cv}aUKPggi~kn2vSIBZQpwlvH2bHDo=*PAjLTruKu^(BQSr~;OU`7FJ^Z=DK5v{gAw!lB zCx(Wbyk%wiQS$ot3E`Z#OC$nqs{L&DNHYr>)OSCaO<%EJHHvP6utCd6!`4sQpyIcA#77=^^_ z&6EjboHZdb#`;)ujB$)d%Q$O-@+MMZ6RsQ^XH8(%?McGZA(Q%yvnH@CX2GG!q@ccL z{TyrAy*6qsID0@I7PJSi}poUe?tCI~6A9Ta;%|<#CKxJlJ052AyVP?8Db*}0@lLu;ESn1qa&xWfC(T{eFp zUr|A9AYTzrAjWBNoHap6ZT-NLK?ZY;g>{@Y;f-E@_vo;FxV^uZI?IRHYh!^OXHCch ztk|$)fgNW}$OEj{^<#k@XHCchtdN4l0z1x{kOx>HLx}};oHZd2utJa&3+y;+0<)+Y zXH7UfI68z(0VsTO_-ZA;2NwxzS@R~N%SPhai+jBa`L4ovi+{V@yDIR;IBUY;V4O7} z5FhFb&p2y>iUA-kOPC{!KhBz9Sa&!)3QKgHHDUWuC4dq0M{m_SU5m1F<~~-pQ96ln z)&y}}I*bxyjOR1Z7$-~k`k{Bun|b&X?FYU7n@5meVYgR< z)Ce|84Lc8N%H-VWU_=yVAd81-c#i{60ASvi|fO^6MvMc5qg{T#>VDKePOeG ze`#T5z1Ll6Z}gfAy|w$x?UmJycCWcQ($okL`nx$!jR1j1tCewT1hg?+BDSEmfYOgm zYJ@tZMsP>mx9l$E#Eb}a$cT`s%5fruP=qfA$fy$`{98i)u|z)W!$}9JWO{oL)*liK z4SN1g`-5KJ??6@F1DtDunL@7%pjYQ`1t+p_MqKm?YLg4 zKtb+#Pp5T0q`A^BluF+&UA_gcLzu_KH{tcSWs9#Le7kh%wf5m_M?0bC^pMzp@XneQ z&;&|q@ivZzNkoZmlar!5p8S_e(CBq*H(T7?9w$Xh8FGtVCmfJOC`>{=M&TYJ?-4mA zhrv@H=&T^DbVBV0u{S;bmtOgb9Ta-`E7plonlVqZ}Tw=8_cYKvxu|#%f`K}sE&e`agrHxOz7N~vQwqA$yFR!G|M?uhgsLk2CLAFJZY6-oY6jr{oua_;@D?VX-~f4}>Izp}oz1euQJ#3S){ zXCJn;&OY22di}YwH&1ThUw0uz>F&|4zYV`~^MV1CxkS#3G6y^DPH$^}r^{cxLM}ue z`}Mg?p7n+kh;;HQ#zlRP2Xlt0yB zkMA5o*0Q}r9~GM;7e%rA@+4sYvI?k2&Wc~QW$L@RGQCJ<&1Zm>?P9li{szt|Ir>BQYP+0reQL(~u9zw*Sy1pVzv@=WTkc;`ib zQSay|rZJ5!O!|E4UYdaEb0_6-Q?6YgGYWI%h*+YKE_qsX;Wf_41f0WC>W;P8VupiP zZVJYf@=?Oorz7XZWpc6QnE|lNl@CkwXG?NLE{aY;u3E*=ST?Z+OZr#xGwD^wP(HFTQ zRQ&Zp9jcb)<;c)Yk;&GC>C@^dm#WjZPJvJBM5|Pye^Mf61TDN~2yk$)vwcX#DHd)m zn9~YoNq_KMlV*=GZONdS_fhQ}IlTu*tLXOR+!SPxk?2QwK`{)18cSbOF3>8yFWG@b z*#b=afZw#WQP+5+ZevGHYPddmWw^Oev7&UFobR+OvphL*-X%ng{=N3zmN&)z%1LXY z)yP4~12wbQMX)Yt5gV+cg;OGq_C?mTng2O8WSgoXSavLHw%7^eWPrx`QWT8bBJT@( zkb|Je@HgUzKhJ=?qjr0S2M{BMM0TFjm$UN^6-W$$hIM8&_;?7mijN26RW>qmryh|$ zv7Juu^x+^vjK*EbXl(CwdtV=NC!3((FOiy@%b2-wG>TaY*$=tUe8iUfb-Izh&P7g0 zu)$C?zW|x->=yI^Du)aXdxEJ`b@4S<#lZuyKL^-n| zfe#A}@9z1lip5d+w!Mn_q(+ukir1sNwlB-;%A%?R*0!!M;nJ84^PNcbI14@6KEy;! zB>sa)eK-|cEB%^~Q^nr`5#mlyZ@ZVAGscfJjT6pC)|4crR7<@%MH|3*<`GtPJhbT> zn6KHcU!Ij0oL`XC`s!*6^Wv(2SI?ASy;+JkBE_}ioTd13q`3AevlL&66xZ%nmg3Dw zaqWpT|4jH6?_Dto)ntAy@s{f)Un%B4p(HBCjmNWJgLG-*imyr=5B;Ti$cI+^#orsn zrMC7oxx}RQKmo{KL2%E$9QJGq@B3ki-sj2dB3(WFO~KQR78Vh?*9$ zAe8*zH2K4_lASV*k-xT9U;D9;od)vVo!$T}Y(-K#k>?J?rw#H+zGwO3Z$O&wgG{q~ zwAXEeOcODXOfs7F9hN*g>pM&wob?@$6k8;#1eij3CwvtU@TS;bn_x_`Tv3>kxs_L9 zSSxuQWUb=km$>+a497}_o$zjWC%i?T&%6^(&Xu?Yj{5foY<-l>MfSCmWQv!&0sCJ3 z@0En#PsRQ|-s_|irqI3azkG`PQHk8Z`g2f~z)sNb`a6TeU4M0BWz|rRb5%Lm*>1t2 zRi8WJvrvb?XHY|gl_c=ALZ+BV9~Q2m0jFmP(flL)ioe6wV0>rG9{WCdWI|DcO_++pgy@|sh&x;ZZkG6tv=q>K*`lWQR!Do^e;1GB6w%_$F$ZCx3ys5)EGdtMoZ|^ zL5Z9TkrCR3%!44bjVPjnk)+j7Z31HOzm zzA@#g(3};bo&AFnIbrB!=Rz&$VU4kO`w{430pW-i_AKvv&|?j8YRT=}USqJnGR`$Vo#ndYbn=XD7cX5S7Z%VBm!sl!$iB zD?;j-x<{@fzTxc#oF>Y&hcip&H zZm!gEPoZn5hPW$*Y%@?YQpw{)nG$)rUu4SPb7ZD*rd%#xDwWv(OSQ+zRd8u|AB=*J z2HSfN{K5O%hn+2dx7U9FSyaKG`Z3U?u$j8w-+r*o93TYg1Gpcaseue7qyaBpds3?s z${i&9{k@$J;FEBt5KfBF4Kl&|2;zvt-X_?2u=E@|J3X-J-30>Jk5kN&*}L%Oc_p3a zmd&f{t4r)hCeOw{a{2b^dUK-&sv$wFQtyv|xU1xX>M*jH25_tNXG&xVO@_A{Fm*c* z(fv|J4J;=Tdj*_x{+LoW8bW~oV%F|O-SxlwcqAFGL83I zF4@=$$(z8m1ZGeUf!5-J( zJjAU-Zlx3vUn%g_d|3+LI|IlPOB8kQtj63sX9Hm>usEZV9RKfNq>9?X!sARxUqzL; zWX!H57Fu2<=DJ3Iik#{TXK{Hf&(AD&dylp|z1HnF-U4fv6bx6Xc1bOp>01OA@e#A)HJGI1@TwK-FV&By&St$#J*BHDkjh zv*u=wnqNkJ#&$=ua-4%jr|Wi%@%$55DdSzxNC$NoMH;Ax9#x~rGAe@1c5p*0s0cFE zxvm{1A;PKjq6zwuK{5I$tQhT)PLc*DrRhkV*+}gvR`7-Q|simHN{9LVKyRvd~%Yt+kuW^>$;e6BiQz7HUe5{udy`!gD0fbcNh- zm`R-UxLB$6d9V<9Ezz`v)Ew1#UPR2+dI?V2+c8xNGnhDsHz5!N54y?1Jm`SE??8p6 zA}ic#eF;odzCdzm0psWwe#_*~|F}f{dYY_ZO|I|X2S(_9rptYBa3u7(a4XpIH-WEa zKly7bYfJvxDyVqRku%8h1GKg0$qae`{%Mg%%xoi6iuvz_9)X6|TP7${UhnG%pkM~~ zkAaHsEHpHR%okpIhTLQ!nzmU2R)6rdqh9YDJ)g$|6XiQc-Mzj3-X0WvmgxKxK~Wi# zJ5qR&=vYvuiHxK6C&+c(k#F?+`vZSx`(e*V9atp|T|_+>)v0d`Z*U_%(BZA2<7;F& z^?MpOcyWU?bG*n$u}SKwUxh0a&&&!fK1Vk6Yx;;~&tgMyGPVkAk80#e{kkRvLoV*s z(9|c0j{EMTvc`z=(#-H+f#`tp%sNn~lEs1J>RcfT9eIH~r#qtQNFBl5U4;z(xV`tN z*FR)r%c$IX#9_iqI2^r59hckPi_{UQ-BqNHwd}4UbyQ(@6{+Lrx~oVXDHf_Q_9!mw zSKE`EQ|g)c7xpiB$=pkr{zNW%F%3mg{DP$wIrEHG>M}Z}XvNqGWjVg?LV*#%qqkVg z8f<9;j>HhF0%Wxpg5|?Eb^il!b4KhpK1MW~nOgSTwbhTk4a`$Xw~&dsf6vwm=JJhJ z@+ds4EnsfnXd#a_;dL7gnCmxMKo=DU%TIr)M4r%N`xE?_{+vFh7ws_K8hRWNCw<06U1&cZ%%S(Qgve`d!4Xn7F0m zY0{uo4x#m1V8^HFH+ql;D1B?Ax{r%SHGJeP7`c|7O|ZwjuPxmao1h6*TAnTsSl|Is zje;e6kp?b(gQdEk0bH1zVF3f})mh!u7rNa$-W2;w(R@?0eJN;+enP(!o$ZpspfrQ< z4v14HHI`43lYVEX4Mx=02w2|l>~~tv>nei=(|a!cB1oWh1qQdz4^4CMv+(G;)9$?` zyjsBKOS9e}G46v2stPs2dbxNPnf$%P9^CJN+C(X^+zo}yBon2|KaJS&RQ>3ImZKP3 z$_e^Sbd5;WxBx0p$)<+Ep1-j~zg;E^@slDn4Bl*bV5pudRuYUX%kPvh0)Ql8OJGr(aO6+&YTZ!^r7GgH4fNVJ)b4S3N6qGuMoz!9kPHU<>a`~kxF(&k7 z`rnOHHq^1;D^^13jwGzQmWU>-76mB(KB*^3P_+Re0)-BnaemN75tur>=XRiyzO5X$ND9)-A#m6Fm&7V9TZl+>k~tzS-Uj z+-w|oaMRNwY5-NDzfvNPxf%d*kXLyTd|CUi?RTl(V3VYx=&1?-Tj} zNyB=u8=%SkG^nwp}u)Fszbm)y`LHIrmT zsg4AyNkx&SA*m=Drlg{1oRW&7fl4Z>M~d<5O!7NhJB-|(@00g<(v;HZ{?flI#qf+f zV2cPh5cUriy(QeEm>D)0m!Nwp5;2@~QLNYJNGOH_`IsqQPNG?m*P|Rvz3yzuMl_ZMw zZt0*n##FLGlSu~dPtvsnZ%5bhUh|guWUru6Nw&z`>N6uBV7plm5t~|BGE%IV5x{M zgagY)^l~E6L-1g!h%ST)%SQA{BGHL#Slrf~VtNdib=%+Gw@tDzhT(x8SO=Mo$iY>yTWU!3o{XY%wb|`n0e?gUKjrK?%>2!lB_IVC)M@wwM(feWuu;!DM8HrfT17 zgvZQrLL*zu3XMK_Y|vmbih@RX0v#tbvc;^>Xi0^+dRP}6oLt64wG+2!(rm$ zZj&%U?5zzGOhO@;V6Je~n7G*5Buo%rYr_PSPzWZN9UL_#E`Bx%6U5NkFu^2b#N<~? z1o_Zu`*yiSMJ(JQG!6Ise?xoz)8Rr5#2ETd((jhA%~GU$!2@jSZxJtGqxAqFx};5v zgNl3?;ihT7TUJx%QKww6%3UskVQN@K1+lRxMEHeRSpU!gtL$EY28I>Q(cmBx?1}uE z0IZAjVt|!fWqx2q1+l;`^SBndfR)`#3#`N{9k4Faivd>dxA}n;6~qF&k{?*vy|lne ztkMDNBE1-3<))n8FR$`S7SQqJOfvv{wu~LYUA6QXAEU>HjftB4$ z3#`N{9k4Faivd=OK=K1CDu@MkEkCfbduf4{SfvBjMS3y7N;yh?U_}M7z^>;9R(3Bf zuoA0uz`97!2JGLIv3;$IVZ^yUVUyuigf+(7hE`s7(U+Zw^w`4km~tb{@*saZRjZQT zM5?uwH6;!4^}4tNTfMFW&sNs8hug}!wj~rX2M>dC&Qs5VXi>^CyQ1}=G4^?r^!F#R zXP~IYKF;W(6wF-3-3JfYG{YjA;0L943m&*jhIu=8$uP6ROX|Q5G8`5o$HXgw)|5ue zor6_Gm89RDjIIqi+ahbjidmQwaMKq)+T10>d$qe{c=O>UBeP5pWTJ+`i>Xh?W1kfIJ)2jwsFjK0l`$h~S#7txkf1)84k+@5K_&)9 zdyAwZ+F`I5v!RzLgzR_V%}q14tH=n*s;3^M%$DRO`qS3f;E0OgXc5~=@RjXwYOUrA za#N;EJFIX%Ha^{|2#?lY0?%1{Zcq-2hS>CD|tR#vI^O#Ee{4p#l?(&31Y@SPcCil_jzdW&7-}};r9NXzt@J`Qcrm| zeP8|LiWW<~r<)%jnq=7p)I)tqD;xj}5-$I{RfO;q+l2pLwizJc*1D;kOv)OhV>L@sV zH)M4Sc;}JjH`*nqgGR7cR02b<1ixm@S~LmsF*KBA`UVjZsNqh>15ulKkTuKd-uuI$ zV82W*7$FWJiYUaH%zUS}TOCfg^3@j=ZKmXEpr{ZtCHj|;)ZIWRlrZdU=z@REC@kO% z81PW0YVj+&_S=8gCQ{a$v{P{7O#TG|9V5+|nY2y~zJqaLedwx;@{oG)#H04kQP1;k z%|bex2_tFTcK|!1AzS9rj~gY-u4v#Ki6)(Cx-O6zl~Pc3wp5!XWWd&que#ql4}G4g z{NR*Vsr>CJuUz?Fi9&GXChAgL_DZv}3OD!SOXW=RL->z7AAP?>o=Vvf@1iKkQWw71 z-`^GAsJT7R&!nu)Br6z$5n%0y^B2|&-hU!F1!V&{go*Yot`tpMsG2Bl5BdS*@HMs- z=>GIw6dXD{(l*$XY5d*x;2}?G z*&`=Ks(4TSOGA0{P$wm8q0haiBX^$52Da8>M z2}Sf#?HoDH(yWTEPR>nHKv*W0_tXb!rb9^%dDMR*mk!e22 zi(cp2F`2VDN2Pbj6w9gZ@xLMB^W=wlOQdhFi*BBg@bXSpSVN|$J2mb1$O)MFY-<;b z?i3)w*THG{09@8LumC-Et#6alvV(rZHBxy}*t|kcibVAi*3-Q2IXn49foLN%@C}YB z5$)teb&p&}e8bxf$QG+OE6z-*ouL4>hkHXp5aDopM&4r?1vd9a1R<aHgx!ZQLW99OTiZK5`dGBJU@Yhp;>Rexy2uQJR)3uf4RUJ> zgdsaUIK~yvAU9X)xTnxHR72dALbmDa288&8Mwk*RPWOwXbPp0(&NR-H%jHX@68nFt z_Bgrf@9e+d>-$H8?Y#&708(Few*1{*|3MG($(P{XcFAWmb-%w2nO)jDzQ_>|nds-v z)IfA1zmG6ye3ekY-vgVL{k@$J;L~9Ls1E@Fgha`kBrNCsB}o1N)e1Ey~0;i7mH~ApW;ajDm2VLnxH&az2K?GJxorx>v-Z>k{YJtUJsY;UvvTO>&o>0QD zxm%Y(urvks=;it85=4_>|4qXGS?-iGs2wab&V=+;R7qCRa!9PqH3amh$SIj9U6MUN zv)JuDVxemw&o*$*DHx1W?UF)Z`W68##YAmtV&cQk()X-JFQJP)8si`+V`?t_%g8bU zTeh4-IGGHvYXh>PgVWYy6BctrM0qAtcO2J@ja1BlW6FC$2>S<0*&=U`dtx*cO| zn0x%FEkX)2_t*_vdwhw!3q}#4@jt;9aY_lpX;v1b$RCzk?4vN+X>rSmQjASQZ?oRm z*j%YEY_{(&Ev&5fx?s-GYcBNG?k~4jRyW$c=4zZx1FX-K76~taQt3GoXZ%BMI1<>O z^fU=oVPsn`0cf{lsuYmM<$)JGGNi4aEHZ6<2P!NTS>aae zF6d!jcWUx&Q!kEe!7G#hlBBI&X)e{r+S&>FXlQM1zT917wXHXE>x|@xN-fBw9aj7Z z1qBG$iplXaR@)weU52{fPLavhgvqJK3Twt{Tc#u)fMtrbmou@uu6W#ww>&wgC+90; zwQYfBBazFs(mxhwIO1%gxIJjBwncfR(H(!Rwq;7ivD!AOeNB1KioQ5j+eR^=W3}yr z-Tm$ESZyn`oTel`nbS^ftyX1%&9T}xoq6F{Z97)mwgziZ!ZJZ+W$3}nqCD7k~A)k+Q9Q{&}`~SQ`e*FYV5rd#nUp4f()1DLmJs!gb=CM@4}EXI_Zg(W6+idB zegKxZ;Ge?$hiA#O{_e9100|ing?;moM=%e2I9I0JhgtY72cN-jnL^*yo7_POih}zx zGA2?L7*9ov<#1UMAUOO~M7Iq;6`@(fPeokO@KX^KGyGIUbcCmu$a(I?0x|2vzW@_F zz94;jjE6y46deu9o+D=@3M`!1T=^G3b$cv+)wi&f+!D5u14C=(a-Qs24bIYMby%Ur z;eD7U=+_9*HG1eX4VE9hqD!zgEk;%@!!yRTge*X+^i49WD+C6&2r`byG_KP^2<#m6 zDlSt5DI;(y`-SvQWf7&*rz1x0HS%<;!cDlqb}Rln3xYtd7p zHkO{1yUkJgUAGcUv8a!Lw(N6L%cCZ9YMq~=)P9kWZ}j^61Ak}xVc>!iXIF+*=p~4i zF1^Oj-#Q1gURub3ttbeRUsHR|BHMG6nA%~Ph^sAw5uB{{^sQ5%nV)EtCTO3W6k8aI z^?E_~U4zHX+vr76n71AL#$dby_T`fr&lp-KX)qVSe7T^xFhPGUmr$LCGR z)sv<%ymqI7U`wnFP@R%t^>pr_u%R+Ndz+=Ew2Yq{*!RF?072ClJyZ=hRCNrM=!%7y zB&gmcrzKS2rl(M&2rS{^I6PR(qX%o*4ORn#CAu;kSm3HEal>FObAQm`!CDzTSSxO@ zmN8hOE5m`cYz1qjAh4Pv0c#KMSWOJptmF^^OfX!cGr>>#RES@QRqzi97UGH(u4X)3 zF$ZSF+_qLnkJoA(UdF~HIx`$zO)FljMd7tJdc4-+@G|x;(V5}!TD9V}Ruo?AqsMDK z4liT-5}g?iuQe-P>qFp$7W=xcTy8AdPb&|PV(cUv?#k==D*qB80l%>1uDvmQ?T8%- z1G7AQ?dbdt*S<1*?dZS@*WMhyc62_3YhN9{cC?#@YhTM(`|DcPSInLB=gJ`1OhYyy}O@(pNu3r09o;7X1(rqaPw_E-Pg1cl6nSrJ?^k*oDR@@9f^s+utn^CU>8H;akdnCRs0aMLNu z3f~zYvN#)j2xEHA42HH8a&SMdCWapS(XPccO+cd@8jRM5nCHnIvL(bhWp1)=xa2&a+-qr%6cdpx!}l6jp$v&jSAo`2{SGu5+Ea=a!(G#f35* z7Y|91&Ei7Kj*ExnsAh4Ye#gZ_l0>t((97fEAxWB9Tqx{u@sJ?NEG{(pxOhm8Vip%F zeq1~xMt~vG!OsQiu1Mtb*N{<#hXm`KH?BPmF}Q%`g#m`Jr7Tefd^zlW{m?t-%{=^x z_Jdyk%_A@>+wIjpdUjb$_cxZ;);isVPIq}@VWqybzR+IktSofadu#0`i%@ zSgw$}XY`(sB@P(J2At){3K;A4^5;0j`aR5;CS@IP7VH*rm2fivQvo(qk?Wph~ z!O1XjqdY}gwDu=Z?l@oT`h8UADrx8rsOO?O^^L7(h}V4J2UcIu@inrX`aO*sytqM{ zIbP(W*d+DTuPX5bwD=s^(60qF2pYHd9`*W%EXdgr1XyJIx-P(;&Xv6y(OJcOO$vrw z*c?JrpCCF#C@yP^s9<%V^a9ZtbXB@aSrR3%tV_E)?*;OljSl%n93_I`09qVXh>W?h z`H_`HpQ{Ry2SqS&lb@=C`n@EI8>6Sk%InEUf&Sb0uc4`4w^@SU!AH_dgggp|rXaQ+$l*>wD_! zb=Ouu78IDLl5QcBfZ#n_D_Bxsw2~(%xwQo>FECoj6Y=|X8x1%zpV0yeCg?DJqrX%l zPw27z34SsD97yt1mTI1wM&%01dZ6opavK~?wx+?(u2T7XmOJGyO7wS21f?8~c1S7? zv|jF-?mk~1Fg5jUpW4%s$~@QFgcb< zl?gD2Fn?Be6><^W@ut{cisl>poo~cJW7M7wSI%}xVNjYu7&1wHlAQEAJMF=MB^>nT zcJ@21=XE!N22+R1rN3xo$u_us&IOu-pM^)yGXWqklV-gkUL3l6$li!YJn8LAY#HD0 zL1^>h%~5|EvE!-w(Su0j#l*Fnpx;E-h*XUWpaMH;_g)_Iq_zj8F42EiCJTC2r5FZp z_HJlkaEftT1{q(Tk`7c&4!=Q9M^8D*L`q>7uV3pRbAgxX};?=&3@HJ)7cb>h&~Q@#yQnfbW{g& z4>nCgLJX>psGyj%q4@rD+Dc;_RJXnxrVc(;VpXQ)V$W%}QYGd0w);43yC z)E!A!Y%T5?$HmZ0Yz4_lKB?zMP_2yaDq;9$+=@I=?*JaLJ{`xYP71IvJ6fIYTDJFUtR3SQ<9&jZi6$AcedoRd$ z>$qqqd6|{yuawARt_DEn<5gbV+28NKw%?6GB~JMD%x&54UDf2ZwNY48U&WFZ>oQ(C#Zao3>=$4IAr`192Qh3lthx+gpx=|nNSj$$tRRVw(|)kk$*r! zN#r$p4wqI#qlzs|(F zv$ey>W%53GkNfGDM)#NgT`7iVWB~`^2EzWKNWP|SPL6vN%ZCW6DP>gw*Ha`%G)EaP z=hi-7<4`>1vULd9-1naM=sy@YShFvjOvbn#Grjfn+s{ZSWzJ{TX4)2uOv~dcS{Gwv3}&qBm;MH%p^CHNJhK9T{kc!$#Azi zW|CKvNY;$TXhs>Xc*jigS`x{cW6DEv@(Beu&0{8cJ&9z^`Z-8;pUKq}XEI!PkD20) zB#Jc)n}gziGAtadTnOB|hbI|n13( ziHukr(Ysi~B-M@LCY(~ok}6;wOSMyGQ4)W{Fv>I9i|9Jt7iV_C)ABB4MQBvW2?&a5 zRbqU|s50Hh=8dXGtkFxBB4-2TcZ?G>)M5k{w>XNyg2j8tRw7hsq0nH%*5iT3G%nPd z+yrR!Ie_d`l;wzFd|cG55pEXB;1C#V8{u|b2$$P7)@zSju1nB#;-wwM(fee&3#!DJK#jd1rlPH1F{S)tLQ0vj}# zjG~|sZcN7sjchS1G+Lx!g9ek41DY+{;Vs;=(n6EUYNSek(|T&lrm|w9Wezq>aQ`is z#PN4lJT@|McthbKo63rZmPpv}!2QpNhhh34d*#=cMghog8zyn#wjzkQwQPc~)A9-| z&IwoR;_Tw)h#iw`9B;}=;=n|YN)s(>^v`SsD%3Ur}dumuk1+gCeBE~{2tbgc$Rdz2x z1H+2uXs~GutmM}OU|pma1FYOC^8+g?hy_+ep2!8P>|R=6C06Nxb&*~SuyVi653Hyl z7FZFFBNwo;duf4{SfvBjMS3y7%1t{zu%d!kU`3RYT)@iir3F@El@3@J>1l!GnU$P- ztQ2D82Ub)N3#^FUk_%Yby|lnetkMDNBE1-3r3fTHu%d!kU`0fmT)@iir3F@El@3@J z>BRsm>TxG{!z}lK%cA_6!u&aE3iH&gh~P%v{CY2M^dZ!y=mC2c>ok9=J<}c{_K> zFtfr-;y@os0kjlaQyQ(uBL?j=Uio!&NyBh2zYR0MA!G<_}cp1)2istAZlWjIcaZu*ew|2#=%;En|O z&P8~jf%iWq$@eE)=OcN-A%{>XHVe-^Vk>ERdBRqXh2x1PB#%p06bar_WdWo38H{T_srUu~`~wC^{%3rp+w@3$M>_05%~m86jJ&y!1=`+ZA9`KP>_zOR0A1d)f( z-^(fE$6u>l_vVNrq)F>qI8*xr>}PgG{S>m+9Tz1gZ) zrXgmx>s+_)67|ZN?5~j6S1en|jml|{5@)D7=y)Ytl~s7jTu;J5i%oJ<4G%t6oJX@Ah``29--_3a&hXMZs04gB?HW7Q)pnGPJ41BfucL~iW14|k7t;H7r= zQMCIh)$i*eE!gQbfoLeA|-`{Qz+FjO;>y-+;!UO1-QM)0^ z_P0xyZ^7#j<`J~E?e5#M#n%tMUApvI`|!1+ozQc7NbEm&XUz&|0wp!KW|vdOt~lMp z4-R^a&4MU0Cq;KW`7f2A(d*W3wjQ;2jv#@PW!A57@zma6noR-f9aL4*g>I}zha#jwRv*QZ+E)>=Fy%`4AkG> z@%N7oVHJ9lPnT*JBRs$)nJA7uu7$bdtK^D}Hw31xvE-bMj#=9Hq-%lN*Dd6;nebjo zeUOtXx#%B3k~GNZ)$43;Zuh$WUVFC(TasLn}% zR*MR(RichGu|-AGD$&0zg;UN=Bo%{ zW>&Frkg&As?iIfpU$xS!?sOXR7P9q}&Ox)<6W z=x0*aW|B2Zti!9rQm`5Arj8&dmKQD8#E(%#a*dC}A>Dq76 zRj&*}Lb)1G-#Wz)1;_>U2PJZX*9NLLrn;xjB(HJT7kl`2iv%|BquM!gdhcjgB)f$P zIYj~C$OY>xrzBWU8se1Gze?XGr)8}$0aXLkthb({-n*m>vInmW*M2$^w7{=xGU0EI zO7D=Vd+og~kN*vIGEcs%x5x=5AgPOP8h6SHYsl2;lr;hrK>e%$W`A*qgvT;)zBe7X1NO=Ro4- zUVrGWf9C`_!K?Mo-i2JEFhvG~cNi`7v1n_-SkNcLk5PJckr@W9{yG;Lz+c_Pz`Zc3fZQw8xTBBlqsR&bic?Xd*{eZ<4n0+zEmo)|CefyldB-1 zejj8!M}zIX2mavw?ZeKNzuW6S0BJ19#~y>^+H9uo_qQQa709!N>}&u>-kBPRPULr` z{QW$6T_`BwZ@QzEsWY`@9sej4_pKA$W(;h2D3^Ep`+GYd1k!P#bB1n^3EoGz4jlG2 z;jmyaTz7VQJKKX@Ab|Zib$`G6fp_7}^W+>KP@jGGn;R=D>_^c3ckz$xJNt3DcAv}$ zVn;~aRdPXf7_X=~fG;&@jA6G2wyQhRn0g~rmjGYQm!#6Ls=HW)~PkcBnI`9qcri3F)h-65iPWWX2K<|yW87Q4MiEK1kyH{JryIR(R2s$EjcX8INZjpambYGUHU&(im-%B0Z69*uEU zbybyq8Cip2Yrb;`CzAorgw7XG_1J9B+z?UT+|-vATr)PzGiz?@QB7U*%c#%T6mM3J zvt`CO#`q&;1y`7f+iqBg_a*Xf)R>JY8cWIZ#PU0C%dXXKce&H-bQT(G&CP|C_G)io zy|cc#u-1-oMjiR&y;pqFM#pKb0o&%29$md<={z=^P8>DgF(n^#BB#E zm^@N*RO5LO>00Y0IACwbR4JS@;?$OcM+QCUWMO(x;OaY2VX4Rpw_0~WQ~SEpo+-33 z^Qjj{wlF#&fAI?W^%Er7fNykvE{2M570~PJ@DsIn*6mB3C zhOVhBmO|63zOk+{&2R8sYKnvJOjA6Al(eFFkLN1~u_jYH&ys2V-DefB=D5oP|K=e@ zvd{Q}xiVEMZcujzpF!PCq3`NVF4u;l03JGnR0YOU5rrUJR=CiIp9=r)@KfQu9DXW1 zZ^KW8+i3Wy@Ogx%m&kb`^=$|ts1Go~;|o$GY&;CgBL5*MdybruC@_bcxiVNO9gjNx z7G|!NFmoj>8)%|J9(3$ywFUm-@S^Vt`ZYpyjUM_;gXKrB=n||=%h~MWS8Nw1`X-sx z6#_e31Q|zU8rNwd1a=O36_+W3lo9A6ej&XsBBEmYbVN2%BTu)OH6M>=68|ZxUAU`O z8)!bcN8|7#RbnYXpvlItM4S6ePhF1H{I!sXb?r?K$e|=3r=XwKjFt1eg$>6k8bbhQ6Tt zuEFD65xpo1^R|QE7>q2F(xk>ShSo_M%mujh7Bm+o=&uEH0i*`TTtKY^U8d&*A;AOP z_0L5neuFG=4~J^{zu_rSp2cI`}eBPLv zJlGZy58%45$D7lZ0jg8d15JlZ4&QI-+1pI`WEnp9X(Sx<@2CIR=5?vV%EYNUD+%QUC|db1cGpTrAy*qUEx z{|hvTa5PYyIRrSph0GzALgo@UZBEA+vnU44GYRc8YA%lx{Z)kZzag&zB7GcEb>Fhb$HJ z8aBk;yd%N9(97E;xeuxPc-WK?+F?vQpUw_LQ>RHtZFJBI+vuP%XB>+=x{asC%O= z!R|GG(^T1dcy?(%g7vuU=0_7#!;DW_|&Sq3c{#?zsi$W^tiI$Hha! zW3#wWuH)h%@u^u{Xx(w~kO0vvF4Xb3cu0U|78iPYTs$N{GK&iZJ}w>-pP0pkW*-+1 zNe^I%&T91pw|sP|$>*;jTM7@!(>Y6AOCMkCBV&A3-@Jc+b8WT0P;al@Us$<+zp>C> z+GxP1I+$OrcbnaAGtT(xB{06aOD^9B8*WQntqpGt2T}0-pqwMf*4p~Uh7X21&9Oxn z1daIDYSHz7Q1Txq$!GA?;bAvYhb3BBUTGSpUW?=!3oaBBVQ2<4>e_(OtCJi2g?frt zgv}qsY7(zBVqQ5N=C2%g8PfJg zGn;Nz=r}xv3Y8bqw!6IF|0RaU01Gf~>9Gjx=)+!jG4SZfX-!h412TJ2C8EXvX2k=2<0>h}eQdKV~^8qG8mMTB1kf zlJ#(jP?T^Q#?T~AN0TNWNg1%1~(L0FQ^xO5JcS)+jlU{soF>MlW1&Y zs^cl^9Tz=BLaXznd~bUX*sDt8BskkLf7|mAApx99J;0`MLOi-ou7IO6xU52qx<@?- zaI)8a0RFh!UE$oi8Nu})dDG@+GF*7&6mGT)WE!*+o7)e3h*NmD?_KvEBd16iyh>)l zc94|ehBG_SnEc6!_0s=Zny5WRN}JyNo7K4Ra4Sn|4G8$C=L43c&4tEMk4nyff>rvz zm5hjj7r$cJ93Y&ab#bZZ;R!lKm_ZzMFg~SG_P6}du1Ab8+nMr&I2TP`2ojZJTgocfYZ_p9-@%cJ~wU z*O>@3RREu{yPrRH_hXL5VIikdgN1w}$L@X^z#6;zWdLjJ?w62@aqRA=2U9 z2(PibU$*%*cK1ufYwYgF(oLx-vWjDq|rV^GHHMfe}ch&O24HN7|6#ZwgL|UdZIL0o;p#>Xn6k`Gra#_-h7dK zE_tE=uLSnNhe!QA@Ilz?a<2(CmG%dN-oS6J)i>zA)vc!L7_mV2Kyw=Dm%z+->dyY_ z2Oj@R=f2`-X$ZUf!cv2*gspKuMxJn`24RWL`2QM>>~nfF2_gQ!Y4U$g7BEd3&8ByT zoZ@LFSSkdtRX@%BkeiKVyq|~!9r#zaYpA_Q?>6e)6?3<_Mo1SjNcH??V`U?<-OTA< zn{1nk+i$M(dKMis>)WOOpS^dFjqFPA#CTa`vv|7M&AueN*{2_3Q`34-!(l!7D9=Oc zo{_AP)Gakz(u_Rnt76q}V4wL4W}H$Jl)5b85;@5Z(DXLq6aa=*?WKQ>BV>@!cr&SJGY;EV8jIu4?%76*k%SIm6qU;G>b zb@~;xV|IJc6YHI($ELqZv8-)e6%HL14o2v}p+mlZO3XvRdm7;2Bs3sQ&j7fmBIr!s z>C~CYVz9flLGK7JCnttFb6j@|1Wpn_0T|yTI!YjaAb;>wP(uP@DHFAuz-5>i;sQJ2 zwbD-`CdM|ps1bqcw5}<@C34_V&RJaC>(fA*Gg4D&Re#{1DJ>%tH}~hX7cIchK6J7*s~ZOlI&L$yQ$o!qmN7szFPJseAcz z72`NW-78{9n+|d*|8l$Oe+VNhCs&??iWTDBJKf#swA%M6P9-2JbnjErXCIw-t=@a> zaM$I3aR_2i0d$D)c|8*`{XU0gYp%YK=JFpo*h6`i4fF(&5kq9$iKEa;o`H{Y^=8u@ zf1%lYtAwXnW;Fn1XdX7ep$fYqdl@FBm^_<1XySUS-i2IT@dZyn#>D%XlgbD7rO8u0 z2EVO8Oit4_&1)3&S1FkHA`N^Ky=t&{7&s`*X*GsH!kevod;2YrF#m!>gix}43A_IS zehZM6S*p+hE$J2qNOu$E_F(MfU;yE%AO{L@&H+aUNc1M?R_pB@mwtvSUm762H__JlQcIK5P&Fl!(Jk0+ZnZ`fRKTg zH6(=c077^Logj6E;xe&L(y|X&$Zs-5Bh_MjMXF#d8(Pd@d@KyeN$0@?Q5MVG?)wGa0^p65}?03H0Nu$YE2Fl|ep3dPYY9ZXI5S~#9s*dW?5>{DMv zV+z5$HG@75rz61SYDGE_O`FJjI`$-R2N}Q*QYG(-gT|!vkNm+n8mA}??rEweCTJpW z@3eM(e35}Ss2DLpTi~l!WCuWfVpgnEM6~ja`A{R*c0d`j3+UG5pbwgk9sajrg#lbs zQiwZJ@H{=40aKquHzj;^pGD3c_Y9gUoy_HObB;s)cL2cb0^ARM01#D&-B$aa*ZrW? zYwUP?e&-$>g`RQX+&1Ttp1R*@-D|bsHi=ao;H>6PE`jQV{}%S}Z=hS^W+dU#N^Vgk z0xA8)^$x&<`Ma&|9#nul9NXG&K6GbppA}F+`UJIBULAVR}R~%GOI>TEf=(>&j zD=df{;MnhDon!ynL5uy%;PQDgC=_N`IbvLt@kakdEc0fJ-YZIcvSd8XyXcfU*V5C& z4w}=?djNSp(1EfqKT70e$#Wvxj~j%wGsX{f(8H_9Vt6OZ-y98dq?AG@%jfzxajgdz ze)}Dm4X+r5FZxR%eC`zWQI2$>bEk+gtaIl=%(+uxO-<@?o?c-LY|bYGr7iRoMQ5B! z(sm?$xfQiRb%X%sj+?6Fqpu{RDpA@vII_&%iCN2#&Tt~y1HE7Dm9l2>swp(QgeN~QmDkS767dB655cj1$AI`Y_%WwogM+`IfSDcoN{C(sZ~F~o-U!MN ztwkAg62x9c&&t@Mz%1Rp{U))Nq*b6-3{a34(3fO%Sr49eTum5%kv$3Co8yvToJRH} z%KhcZc#iBzRQe~uxR2~fRQo5v_>k;LtoBcWaU|K3SnHnz<4v+BvEDxk=3D_uXmb<; zPUH>BdyoiIKownmE;%5mi==ajPRF9EFBo#+Jx|M}s6sdDGa-*hvr+nY{liulP#y^e zD41XYs`^8496pVZ_(jIB_8Vqz?Ki>lRpZ0Ft^KZ2LKo1?&IH4UPU_ynE}^y+P`OxL zMh4ihHcjcPn|Xazk~yT>%D-pIMcvbDxeRi>Va!3@+iN)ta(8gcR0q{3QOiL&mRK~o z`0qRDs%+b@QJ%f868}6l4pua)qqGaUY^WI12=GhDffdHz5-W@!I{2SE=*hSOar$A& z?tMwdCf`Wcl&2HCB%fL;7bq!YX%nB}eiY_@zryE3Azwe-hsIYh`l88&eu#n{p z#HCi}r~-q#s?)NlaI1OK9V4Gu>2KsYorr_X@aN?_(caE$7{n$b)VOb)rKivdud!PP z{Ps%-fUtM>8@1y4jgk3L8T01Gd<^#=WL2Np^J55P*p zf;I_Yz)PffTRvQpq}J>9)gk|qvK>!U4j=9`hNWHpW#t%=D03#L-~!e@J0((YJNO^x z&~kjI2sMK{y%`F@va1-YK#XLAj*h>WDmT{X^6gr>x1p)zyI)TCE|;`?e^$OvhJUSl zxCiFGW(zF)j2tm2wrj&1NXb&3d)>)*{LX&&RX76U_=B+=m-CU!L~xj(*ft-JD~G+6 z8-$Sf_RX7g7ajb%&X3OSw0JH;@BqhdEXRFtgy*|5tk|_b%MASC+XCpy)aNSAF z6=Dwm589dGSyu+^B8pco5qvG)bpH-eWhtdWUbs)%9D!W#U z)eT03tVrTh$#2@~q3%$HJY_*tC9O75Fs+*`NOFn(_qM~FKK$ijF0D)(s!WyPs>oEW z^iiY==4rCQ0|g`N8ufNyxo$aV>riGZN;%Iy{sTADuUbVfyeM(9J0mfnH>q(a@|c5@aKW*5$VSgE>pEhiH z{eLIW=WL5AtXObKcx!vR>-XGK>Mv|vgJ6ZFjsPfTfB)32P>#N}K~9CIyk%=N@EH)Kq?Nis^QSV$gKxlZVps?tpTqAKmwFRIc| z{i4d2BKvis-ko0RMHDex=)D?S#*gh2|L;!Bo{=b6;RHhdU`4;y=Nv^WtKn)&Ii4zd zM16dz!@0Ik$2gq25)NC7g3W&Jc@O_z+6h)K3#}&ExIW_g7W)lr*|CqM#;_DV;u@Fw z)mVt@N)l>`oqWm8<=# ztjzl6yMZHC8O~OZxXP>jsw_7e$J5Gi#Cyb5Uh7w7xnmln%KcX;IB7oODzEpevfTPv zs%&4A7ZR+=aNvE!HQwk~W4VPLpvM1?X5nDmg}}M{=v7v31@ufz+pc0k(OCA|j2};> zhqCudSLtHkN*`hOwSW_}^iZ~6=_+07Tj?Wgz!Ft@C_Aupl`i+K^bz)8i7GvmO<1~0 zSNc}E?=~!M>Q2zThq4VzSLtfsO84D}#Z`Jwy%+w5nifN6e~h4L0W+zn!HN95>ZW@LZ~KCs&fEute7{76xAEO{#xX8LiuOf z3L45WA{M)Xjfh3HLse`gLWvGlG`iI;tj4kK$iol~&oQlNj&y@t72 z2!q2%AsU`zTG7bW!Xz5C8p9F|a~~PHXeg?Sc|h?T(~3r}DJIdV)krOx1@m0P++rT3 zXn2lkMI%=plW5dx3`;c3-RDt?hUb`8G*VSy5{+7oVTp#hF+EDr@Ep^MMyeD{qEV|c zfM|A1i#K!6N-CN}7R3wrpBh(fo>f|zNS%X8CTjT&GKt&Y8RfB&YKM0udGM^#%0p@- zO!836pHd#0=>sp7S1cTdKz>g%i3{fyk%;}4jqU4{%R^-g7*5u=fwtNL&R%jes{1IAC6$T3S@A>vIJrU%Mfh_5-cxXQo(XrB@?Wz>cbH% z*P{$BSe8JnU>VNOK!WAPODb3{t7L+;RXv?x|9wuiuN6Q=bl1mhGTc0~#sC%y!#;)= zB2r>QR9$Z*w>*l!9Zx3E-b9i$#nm+p@#)%j2c~pw3!W*iZ4NiZwM|P3ERNa?!Z}Zb z1&~m}GCMEtL1XOmM)7|$s(J>pWX$cX9F&5dD|Yum0O@9`iY9mu+h%f3sx6dLOJmWLqwXEDj4Z4lBm)agT}O&6KVp$55?e z;#Nv+N$%i|nUI#qnBu^$1WOB>R)U{oh7)7ePK_DDFK5^hJq4j##SBzda0Ly3Hg|Y; zB6CiRvG732RmBig6pKo(mYvQ3`ED!us>p&$UyW=Kg6=Q#>$`&PDEX-Pf{`AnEvu<= z;%iP*$*HPfN-8<2GZG&mhMeMtoQ5y{X#<(7pz2ix>s)HQhlh3k|Bj+5I3oeMjUYYw zUq{i8M@?KVb2?idfy@DwgDK|OaLfhZd@%&3!`(50_bd>=D2miGBwQc5x&Z%Yv?z5CZxwyTxwY|1lTrSqvww5bfTczcC zVWR}!ip^4Wy;*HGt8t+3F9GQL^Js2+zw-fMW!^q)Hz@jYyT0eUPrKJVPdvJJaL7oT zfdgpA+XfaL;yLN97uG5XVdF35FT3-|0%=mal)n`F^kVE&z!$WpypX>@5C|i?13}?C zT3g7^_lDbcc zYFG19?qw7+{a&jT^QQo_+m28drT&3ePNsha#lB*gLayW|U5u$m-1t=Jt&tZvE}?|n&?GXN$iau+SIW{XBpV-lh+P?GKu~uC)mJX zx6t&~i)uw%2t3f zqb`s{Dm2s~^c1r61SMn%!ZtFMqU4gkTZf?8+6J6Z% zOZgMAb55!WY#Ao77O>!}LUxEqDzWaC`=Brn2dlOVbv%swg1dH7RztP%m0fI2y zdcl5%^@Hd{Qu{hgXD?b){JbT_S=YdwiT-#A{l&PL>jLIOIghaU;<23TE}%J9Ivr-T z02AHYXp=o+lydb>r~dFkspErMn1ulUWAC0190&d$520}mVRVsAt&FtY+23QoJTe;w zdG6JFgbkRnYQiJOy@nVeC~%O|M@5)Z8;6|^Y}eIj^Jtb8Z43Ce30E)qOA;{5W;r~I zrdez;!@;^>3N0B!%ky)F6bLOtG+iKPz`?SjNfeTP6TLBXc|zHO{4{AoL7qfUu>7pD zgh=N&87!5ckR*hE*FkIaZBB;y0%`rXO1Gi)c?e;me97;<46U#8O7^#2OXTGAAK@Q$ ziu=7$^lajysPw_Sboe;Bv1vJcK5;2^fyvb=;ejD%B;mxM%s<77@LIk18ZeKaV1IET zalvY2FvDFGpA@t%O=E7M@UWyOUB$EMj zaY0FbPL)*+Kwz7&C4jo~HA56vJ{ctA12o05xehZF;Dx&r{0rvcP#_9lW}MIl$?w_N(1LSAyovu0qmF_bmo z0A0Xad^@nhbAWCi@YS30C`x%G&SO_}YFI=W**qIjGEaI-X&7dM@Z7v3&IE+P-W?;K z*i@52SbFY>o<+6;MOGJf-Vk-6HQ^Hp$y&diNG!5#kr7$y0D=NWB-mo`&H$Oyp?HS% za#|Mk0OBMk*@!fMDZcp!X^^+YY)|9?m0?Z73s37eVBzlzArT3bj6JJF+krM(~dTN1nCu9mH#e3{)hz`t2R4kUBHl=z z<_fJyFu`D@%av%W*=A`I`*!UtoUEqupG*K}(@zLz(~lf{OU!VjBa~Ji9#LivOqqQ0 zrwNLYV^9k4`JU$+R8h=tc9nouk9Ik{%a1F8Y6Ql%6xsf8K~EximZxi(hXbeHmHB_S3?>Y`{pwoc~)e50v7CSHd)p<@odtVi2@A>QJIJZM? z@stA?QRLq!{Ezq0WRQvK9LHsY&j(Lw5h7a|yFHhQtO>u4PD{(Fm80ttMgZ6a$rN4X zJBBKdCdKFFbCP4KDnzey@oh9M6ZDEhsYDl@Qqwe4qk%7|F@{l*H_(}3D|SHFJFcm8 z=0iCI!pA4rz!JMmXX%R(gODZk3`Gm09;or(qKcVKbe84~EyVsFy|X!lHf(~YDwI|@ zl8dZkWWq>NTj~oIprkFBzC~;MO@tIaKTakYW3-_eHap-%!FRZd%SS3|htSCnl7HDm`+_2KK%0uc|mE#vy#V)L}DlAr%qY0~Q6jo(O z!m1vNu-fX3RaJ#G&36caOjL2P!UT`xtP(#E7w|90fmkt$s~Rt^m<2PhPg|?UFR#@& zd1(_DE6ma4RW-_Mby)IRJAQes#mP&XyI5h4Ca+bayw-*#ul3`X*Ls}1wCRf#=4kR- zGs& zAUDcDr7AK|sf!F$DkB4x+ME+|r7aamO`OoD<%T9^!*LCR;x;e%;h=>Mo9*`Pn>X=P za0sK5=ZsW=1v%k48K_)Z;j^M(K}=bccOxh6SJ2C5KePoKk-WwI4NSE0Hk#3HZdRxj z>+sLv9-1@cLyS>w^udM?o%qcJn5YRq6*ET|6t8wX8kRd7#j_Bvo(+hjC0XGKp4 z8WCU8;xp08r&kP5)m~wIE=PmK-#e*M5L_=$cXv9i_I*mw4W@_P`#{p@-~%+V`Tn8r zf7=J+9rgv!gzDz)$cyUa_r6PW=v0@bNKErUKZkPIA!@cso)Pp1d!CJ9=BaDOOGD6% zI~a^URn>S&Q;na~T3hx4RaH)l5j>J}fX!XaOfO*E$gZ42(+vtRKfZ@zH9WWx71BmAZe$Bdn6=p^%eUMb>dWpKw#UDNsZ`+PPp{mqV@)OG`-f>oB%CZl5yS0-BIl&5ZaBHB$-Ijf+fkk z%dgyOx4=Q_mBcQ^aS75J)&?Z#3^T!<+9tSE-LiTVo4Q_nDFGp&Yw#moU9WC@CVZ>X z`;n5WrS&6?d&lNS8uyOPk2LO`%#SqlBds6!;3|wp!3r#;BYfsZ${gU)e~r<<8bx1L zZJ{VW$!`52IJY(cEyM%cSa6B#9d@Yaa$(w?-jNM02~4KN-5@*$-q7VpGHV* zaXgqez-H=W2b2`xS4ckI;)T3*@Ub&T0zjK&4ljkm@ylqO1H{ok#FB`tAad_*zw*u1 zPQfyr&;Kp4p8uhP-yM!FAms^tSSkSL_*DRZ)kg(znl?eOas3p||5k%_>v12f+^9v% z$^m#U`^}Z0IV=YU-D)rDF}HxhKuk^19eW*8#+EZ8KCi}Iks3hKfi=6CTr7O<>*yp) zh^mu>Rw+RpEJxiDFzXnnDY|1Xwsv>%oLWxV#Is7SL8u&jiO-FokDw_L8Qr_7vbV9V1p9P!4Es0;d;6{c(}x7@GNO)Kyj0iS^3~A0qSO5~)$2)i9<` zY<1M@SA8L!v4bJR`I$s=`?YDk`?KD0ipT z2CMUY>4Zc7EuBSUcc4eRlW3we&VnwZ^Kcf}-Sq)p<^fSpvtKLc>9-s-MqYL4C!W1)m>lpq8XDO~itum;U0Ot!ym~`H@dGLKI{UkP z>H<}hvosrNMQB@1kyuthB;^%d3J05wumbNH^i=JNuD}baRC3G#r?h0AtW7g`=?k`r z&^$i|@xx+sS^RKt?OUJ|`Ua{*Krgvs#bz;a&w4sN_&2k4@Wc zueu&O8(}nY*uLN1|DYW`23k2|BevGJvpQ`c)`>iYTHFQTvZ z?*`vy0KO}xjmd(g#g0795!Y*VmYct8$>|6)*U+vVD&bk{Np#hA?Z9zaYa^vn z`aXR;%I4(GXw_mA$R*yqwl%dQSP4#xfkqbq+O}4w< zWzJ}zV*n(t%oz>9;>i{>(P!q2<`W$nxNHY|HaXbh9Xm2-GzA;5pQtitw84a>7_(>2 zXkE6cIE7H)YMsMM&r)pj6=7x0Xi{973yI=vpolATMpL?%CNIT(LXlVIjAoNp=8Ohs z#Q23#>tC5On&6dL00zX&8BM)3HA_ZW@WKvVnKRna0Z=k$GCQz9(T9iubv2jM*Xs-Kfj0*b{sJn(06M}W7csz^rGWef-vQ*8lwG_@-S!!Gmwde_5X z8Co)imW`nmV`$YFx@rtvGls50`#6IexH=E$IhsY5!s+$JWLA-Li2 zW%ihalg*%oCG$KORdTd_)iNi87S_B}s8g#dACu)$1})s~9_)9835D=Gv-@0T(89pW zbnGb18MLs_8_r&Lx!!PeF>i*f&Y*=;2rGjYju%!2Eu3zK*W{H!3+F%&#cw>!GnXc> z3|csyyfSFvh)13@D^PHYGic#1kVTzfM%0=mieil!wztTjg*mt_4ol{7ffoMt94I!q z-q9;$;KFcf`xNXs@L!togB6R)LTUuz8QHFe8_x_wSl45ittuV?hVY*_pyg(u!R&e( zhT&t0BAkH((;J+x!VM1SE8vDgXm%1H3Yqj|4*pIYIPiR^Qv_pJQ+Gf9@WAgGoy4uR zla5B2O~zvf5d3uLs?oK;=Py2?5Ehu$p~WY)>pIs+lSg1g|XIJH}os&}Fq$9wH`!CJI?NxHhWZMS!3Qn#c`riBkXxJvPQy&F%6w*u^xL%Z&;cUu`st z%V3RFTyAWwRqGpDtA3%`7YTe5o!bIXKmr5qZ6CG)V{5=q_FW7jC1I zUN4u{z4a;>d}Z+B{_fAJMc2RhIRrZ)eSr5Gf!aI#Q*nylkLrIh13n6J($ zB)7}<`{fMNS12wMdQ$u}V)|~Qi+B8Xcfa#Or&E9UR%ylG_4fcC&Z7q1sgAN2#dVB( zeHt#3M{4LlihmCN0pKBP8M$ZK&sSUB9%w_!rW=1wi=h`K*qYhkfvsdUGiXXws@Z#Z z;8QpvV8@^}^ms_Fb{5J!mH%V{P_BPMP_BRE;NNgiRVmV&K$Hw?>xwM+i=F*F_De6U zkehw*CA#_};>!`T@*g|s>4XJwXHCTmQ;vN(VQPA{f;Kn?S%*ov0BM9uyT2$CjouL; ze?717_IUI(tZslkL)gO3;|# zD<@Z;go+yCxn`yp^U;ad>b=(vcU}G$ht6yAWQ@6mxXk^T$o<_pG+T3Ze9&C}BZ38= zXW2kc5E-%K?M@uJRPqdbl&d$J?)VGM=36B^%`&S2sBH7FK~R`nk-ZEXq#k$B#PwFa z3%R)B3!Z?CYu^?~PlKfT20vi0FiY%V=r{k+LE|(vcYO0;x7CAYC4XcNvhPjv8U-P; z3hzZ4_$GSQ;CwW2P}D*eB)r+Wx3}K{3G*-bG%dTKd>J)H!D=FQR@!!P2-}_ovhBHxE`{@JAMSZP&!b`% zIQUOtF%4y4+Lk^QilbLL82b2HIG$SAAlfnPonAy^gz8Ofl?J?c8i8~2{0BR&U4LZ{ z_=)#G(ygA~sS_1aI$6h_1nwZ0WwN^uL^}<}>~rLz|6m-A(`4P#6#aMC?{?oNb-@>- zxdm;3ud;{H`pPI%M6~kfe5jFYJK(;x3s(gML>e?5JN$3Mc22=-N(ymD3ZADYGhkkD z(M<`%aRktHz*;?nrb;Jsx!j!NkpCSZV7LJ6M;`#h{$aP(zUOs6X!RO9-k#sN2S*{m zYQKo)JknG5JFR;y!j;4*C;$Y1{^U|4vdX@JZZQ~j_&lp`r--YGA&B4@O9mzSmEnX7 z&x-z7{G^O<9H5Bb-a)+sGzb1}tGfpk7=a3mZS6N7x-+-WqBFEXJ@VVDtgjWwLoj$~ zst?I;c(}T;w1uX0zKvt3+yyiv3apN&(fI}ZD-J3so#Cw#bY1Oy$$lT}9Q)S}TI^p2 zm(P>rv`!0!Pg6mQaZ$z_{S&dwn@i6{`q%|@-a=5lWIWA-f?AzxDLL^@h~*hB=b;BR z?u)6$zDR`*lurn~#<}aW+(6#ZbQ_VACC`a$KW+?%vf7%(@J^P$IT|)NDTPj!&-HKO zT8}D|U^Wb$JGn?ie<_5|oq@>u#u(PQb0Oy3sowo1bV14Kc0*T1rkGAM=0G15YR^cBGBH<1H`4)#hL2fWRR&pq!d zU=u2bka9&>)HlGXr1K3Wd6mvbpAaij_K6FwrmPaf=qXnRNIApbp!cA#@b~-=dyg^x z#(JsI+}>_%EN?Z+Wx(H9TVLLSyWi#2LZh&{-mH|?SGVKvH-MZdyRB{i?B3RXCy(H! zp^QSEtz2X0<_0qH}Ol&f#=S_S>1&hI09tS#5yb?9wq=ZLAk7YgVfbc(|Hb zZH(Z|YJ+>INc5r1YJ;*^z#d132}xy3kXda!sP7*7Zf3P{>N?6h?#R;E$Vg_jLHXG& z+$NdTh9_Cgj*czyv*@^=S#A7mv!;CSdGzPt;i+0}+&eh*sO1J&Ywvj5VDC&|9X!Bf zSbMyk?H?Jz#-0lQF?pv`XYlI8{Qh`uZcl$L0QwpN%-0B*V<4HLMqwd53C_#sB*#E! zk>v8(312=h;M-_gCI~k*NB89$!QB`p4pxnZ224hCJ!(acLA(9k?AMpj zGbYXrQ&`l?a5Df+k{*%bK5anP9tVT5@dQU9}|!xVKfkzLMS+XY}5VO%5D zZ_(O*6QOVWo&B!2+qxfkFkOsDG~?l5tv8G@`@NtqfR%;T7r+XibPd9@6pT@P2FzxG zfGP-(UE!;dW-6fqg91+e8SWB!{0+q1CPb$nrl-aLmi>t?{e@*e-nAO_hdhOG61)9) zM)VwrYJx)vyW$HD+qvV@)CtAVes;mW2UrG*s4^Z7!uWHVd&V(r6sdU%35%h(_3yk# z=vNsJhq$XSL;q`oFyrBf7na78P*sf}IXbxlv!TpcAs9iqD#XuO3*cTf$>%JuVw|c51q0 zs&e&tTzNRYpHmRC^RSPHgBZb(;tOc7r^l{XQkBIz2$Sx#GM?1g%PRFRi2HzpFS}?` zclV_^bgIizB>E;m7(zMhjNlHyq4D5iOrC)9KfFA3&G>LwJbo>X#p9gT7lKyQROOF9 zhsRwbmV>2k>01l%sL;#Yff&@*0d9mgamocjW?*y4GooF*qhzAuIZX3j7&MMdKWP^y z`>>z_Dx$+<$KV0<6v9;-Bs~)pT24UP&HkszJ@K<>AMjF!>fm>NhEW}o+Xt*xR|}ba z0EJh|>;no;DrmLLK7czyxI$k(d3}Nc**gx->;tqr;pvXwIP7!?Q2;aDlAEKACTDx z04-T?xRCoD`&y~rf&(Uo*zAOF!7GE9j_RqG?WtFfp1NW`7B2hh(NoXt1Hh36z@`E_ z0_93Q2vcsVVidbdZ7?F!DU`cLjI$%I$q-9{{wjjASA#0OMX; zt5laa{APW*vgX&9*H??%%a!JKp}1XI-Kf;p;#eOt`+%Qq{D9xV=wFQ@LNUQBjQq`A=C71u zUYUek#(|6*XnId8CQ4<_Cjo}N_2vrsc#9YE*1^Zl90`Ey8*_Lm6pmjGVX|`ofg~sC zT*zGSUHuIJ7U!k z`m%AQOH(hF8-AbHE4(;b>SbNB0k|{VRu<$AXeVhKt=e6)I)w`oty5f1_z!aE%!n+5>DQ; zz@iTN2p42i8fQV5(Rm<423p%jul2z9_Ui5WJ>d9hHGzj0jVx_PirA zNz)Y5#MGMr)O8u10tcn-);;f_v)|iyFS{4f1j+%1*)-sapd1`VrbkMnUmsa_e$^RS zdYZ6TE#6*0S5$Z_U}9}I>zyXUxFTdkxTW+5)mDB!)Q++V#_!(TzjZLctn^zB8Y8c| z^b^nCHB1ip91V?ZBSm<)gDx$iOJ2Rv^tKP%(1iPKXw7!7voGkBDb@09q!poUHOu<4 zVtg;%+EN(37toYP5wL{1<_UBexO79V5945JlVdc_r6uzmZJM}CU$D)8=4mmAAC{KO z;)jE4o4|{83nld5pp`%i7CNlF%Qy1qlS%YPFdr~xXlNBi{_I}jZ08<(pwf}g=iEhf zg@v=vC_^~JnL}sU(}Uf5!{6E8ZTcMuK96Q1!CrBG&itS>@0cIJAxD;x@R6<;JCOkXL+^k#TWUq{cU=3pP4cWUbYSMtckXg(s86KIUayON*m z^E}_@`9$pVOZgM-JhC)OZ5i0po<&JF{}6pFP4r7}YPaWoDL)$X^mR~4ylN{qDp_VD z?6MxZ5}inDUx!KWMQe(mx1>1hx~ylZ6Bqgqub}UbqideGz2EtOEU*?J$+gu=Rnz!T zI#7WA+M#ry;#Y(I0&YO2V1scziz2q)5SmA0TdjH*#;`{EsVQw>CeePwETR1-Gk~h( zsle~EAun49f(#nO%evEUvlqlKQdE*a@SA5C{3aPrDX5Vwx=&FZi3ckDo<^tSZ2*Cy*w+1HZF3pTi1sV(`5IZ$jB&Mh1s{ujti3Y;_?j2v5FV6bWddC1YsW z7+NugR*j*n#?Uom=(-4n1&#s2m4IxL1H(ZzW`HT4{!{~lxJ51Q8FZ3Mfyv;J;e%dR zf!GSF^)TNT^yhX2oyf33Ki{{4e&$hpi%mwh6^jjGB za7;RL0LPseZ7K9{Ibi0JMJjB6=uMnId?41TP1Vs zVLTdUj{*&zF>~w@vvvYuWx$^)gq1n=#0x8P>`CV}rpYUF>`CXSk~#LIlUL^06VXfx z7WNGIGZDOH=GbF{MEwjn_WXJd6q!PMSWph08T2Q3B*EQem$3MHMR`Z61S)AfF+6wmjJ26NWhi3@cH zOnbr?>SnL149Al$R$Z_cnK2k*fdRU-gsUjV2h*KHu!7i=T=i1)mTU9Hr$cv+u4SoP zd_ut$H!VnuPipset`V(@SS|&(s|7cL3k(>&kdcg1bfd7yi%%&LO78p(j-XuwUjx4b zmT+CL=Jjbp!M$%ZCqN)xyr$pXx@LH7Qf7I@>zWpB%^>UXMnhe65og(RgKM`SY=?)z zmHy(2xXD#+XWbYEIQfEd^($Ubz{Y|}Z}GXvrLa3sud(E1mW=#t=w2V9>MN{_x*WYT zcNH2MhSFt;&Mwo{=plHhMJ;u9%dSReTuzqM+4Z{`ouL(3qO*H>HF}6bM=uP!uveoq zIvY#s>?U80zDDuOEYaB&zZ!i#6df#?q^$xmb#TuIze7RW5e ztBrbbxw2hcTduE{{pI?`TBBSq7whYdzMgrT=p5m0rPersacH}~=ery3nram5c|kmz z)lOxt;DPDRDqQuRy0;(12OHsMMwX@#BFhDA{C;u2LqcjVfECr3P@JVSx?(XeI^j}_ zp4uB=S@jyS8^$w9q9sc*t{X8Is=WjUg&Q$R=D?ZiBPY(V^E2b__bo8(`Ua{*HcE?V zpU^-D-$qxQ)I3=?p zyjJhMcDU=FV1IETalvY2FoR-`PYPO>@)V+vXCgD{IW${yb3|4zNV zXchV=D_leh-k*w#W#E9c0930zMQWb2=26htdiy zi|TEXLemy$ez*jT>tcg6$lGGJCrtsBVNJpdPwO{e;qMC}5ebxxJ&R#5vrW+o%YVQY z=^)eU^mYdpBEIz?8;woA(P*`s{)b1L$wu&x9kj%|lDfIEWU6i{csck$)5cU0ZzNB1 zg;pe(V6f8VO0?B%v$Sd9ubqX1+*JOP2|&R32|>X4k%MoE8IE*>(#pdl%FF@(FFyIx z1jWcPc!2R`mFF8&Q4ChQ|Cd(e@(_v~%qpZTWl?1Ap+?)vQ_o~#rJ}J{9;PVyt73~I zoXZsDu`(XZ>Ugxv;az^%2>jq+YzMCNxQ{nve=*;TB(HjXE5&6Ukza_~i~BH=p6WGR zOFkyJz`Y%CN!|rbv#83P&B+g7Ququc5WN(MF1_Oo(aVwO@{->Wy%LEoum26vtC8r^ zFd&8Z=HgfTo^sv(M~cPI3s+(7#N(QeK{B>>#22x(Kv4PrzH*dT*miX1mTT{@}Wsb6hA}Di|WmapMqwHwGq`Vpdz^GGSqYma@ z1qXB3P$T&Z2RHVb!BFNXn>orJ?lB*FFy9-)-$aPvvF*0*`{Fzucc-hWR2{(l<4(b6 zub&CfG?&KUhGPW3CX9Smh;zP3+ol6JiN=M}-`v?VM_CTG9-R0xN7>9#7VOleBSNAp z$czE+1f`8Qg0M12SuyP=@G8z6W#fgVIeumgcKMI>L_Y z=76L5%s*Da4dz^WC86!H&ytB8a2=fa$HMvE?AmAJlk1?#&G!#||J&ff4hCB6D>nJh z1tTn}jqZXi&*R{*%lu1<{h7H?WB=S&*wS$Uri`{6h`)lQ5X?c6g-&n4!R#%tn zTczf5VSQ_>UTUsyR|=JWgq1^di>+1*>mC^DR5G{Nw!d4u1^flCB<473bKQY`XHb}$fyqjIs*m&BBMU&_H0J+s?mwr zIwYe$fcZwc7dT6{uLX1d7kTvjV&v$#gr2G80sbvy zy=4lEI!0~=@H9iLUq*c(bbOgb*Y|*%kdbPD;&YRn@Gw8!nZS(j8HUjq?Dl2H&n&t! zi>}O~EB_v@YnlOy2qmT)`;|gw(FI-u3?=MXg_T)!WfolkhZINnu5Z{cr;t}>(Us0Z zDYNKGC$G$+D{{EXEV?p_u3l{wpS}Tpusy@E#J=UqEV>HPdP^b+$S?^PAj2dkfDBs= z#4f|u3}Nerunid|840A#0G5IBCRPcQm;XwjQWyEUQW+Vj)W(7bP0qI=i>{3NfWZn% z+sKeTR7QQ!@NpE-8(-3ZD44mK?EY5_PXWecUl>{%pUeJFcC9Z{Fec)08~E8})CbbZ zm@*EC)8&AgKL=-~U72YYSkB_iv@4@N=si?sXpJ4AVxujrMk7TQCu!o8vb&n z+$=3`6w9m2#ntkbU#a*_e|`HA+IUr}8>=3e?vyecug`{!*Q2IBST9!AGU@}um_Yau zGU@|AqdrKBJCac!5U3WTD_2H+zI2=mlyXgE)CXba*{FqIMtu+{D--(#X4D7C zA+2#neGs`dHm+m~IHNuwbUP84cAKg^qdtI5Q9{DxjQW6GWQnUoDmI)^A0$REFmNkm z)CbgW#K2{cQ6HE|Bs1y*f_Y=`03HFVgg7SLjQW76Kqy`YkTT?ANf}0Cp1uwav#%n{ zh60r=T)Eig6o7kx#=Hj@(7;7?9MlI<+SqcnvXSXdGu>&^=SB~i?ljY#+IV(@8~CJ~ zx=eSP=}ui6Pj99>&2*=X3>R#SfLzr?aXAP}X{I~Pbf<#E*UM;QcUrrCMjM;a#*WrT za7G&ogF&*~X0)+MFC;VCSf$%Yr>cxLw%+N~A0C2dV!d(S>rrlV-3TqCjU|R68Eq_r z^#?SopIzPQA7b?1j-oGnUaP(Bcl>t4_jbFzJ#V-E(C>JS`tGg=2)ka#?;Up99-s&{ zTZFr|z6+G^-LBvD)=I?Bfn&>Hc+mWR z<)A683ht@R9_0eq+--sP=4sjpfkN+7aNW#V@dRxyov)+Qln!&{02Uj+xe_#o<+?@3 zIdMf>PhOvZwjQZD3%H9;sVTZ+uS3fCoF+Iaqw{Lq73sMuT&PDyPtL`3kWpaFJ;@TH zeMxkw1a)u#;GZ*kp1O`;$c}I_&s{W5Q*_5(Z0+viIklX!__Io`L8vHl3Y7uS3DA^a z=nt(SKHyCoq}it_E`&>v$=d`?o@Btd- zePo)X-P8J{jboRar6(eGWK-y*a&4>*C+7~089|G?@L zE=aUaaXH~1zy9m>-fzZunls<0dlFwYyuBDIuw)j>)!X%Zu!UX4l=Iyt49|n6q*j|!t5iQ9wItBa4?bbc- zptIlGcQ3ma(FDrfc>T?30JTK9m*C4tY4qzO>&~w_BXB9_Y`crM7tj@TlL|+(cC+4T zvfb+HMr9SZlpdk#K;}a&DO+Xy?#=yM2LsGYzvZAY@~TTe@$6m0^nlON(8yL&gonH6 z(jvO#)f-K3`>+j7xKDT6z0Urw$4v0t(Px~c*+?rw+iJeYX7}W!XV4tky~7^hP$uqw z*mKXgQ}@3{ddls?cCWSPFKvWD`JX{g)voBBB6P)*gF_JfS6VVp*QPnV^aa~gXr3X1 z_+hcREPgn+b~iubzJn?ec`c%aezif17K0Y#2HSh|$t3zCm?b`-ifE8*6HY?w>z@gu zlS_!%<#@Y(x9e%!!ugzALSJB&3RTVXU^F)mVbryIs^+~gKbQKQBAgP_=2IHr9XRS? z7oBD`5eL9|gx^DmhON8`5^%@JCpPHPtzr^WhIO|O=R%fipq5oO6Z{Fuwf1efe?YdQ zLkfLdigCUf9_QE5^K2Z$r(tS01dY^2=N;|;D|zH%G#?ptC(sy;cO^gB=Xt)*^NHB! zm+~iK%-ELmvGA4@C zUklK!R)D-Lg|Z}LqQDoV@L+EZ;tTfkVpYzVC_F}kF*-knGbW1UN(E4Ga!usf>t}+? z1FnvI4V{E-exu%70c-Z|k;iNYonXni`5QFZw@P@1Db99V_x+$#YU8KWn zT0tkSuA{OqC>#+-u1Lm2aj*{{_@FlSJHb?LzpFU{V?fcJywj<-yCgq+nhtT*;{cGoA##P%@r$Zt7gZ4} zqGClEKveIbNiHhFc+9`5!eSl5tJ!_ir6)D&oQa#k_|H_0~HW!Jj0X;UeS{*9P;32zWk5B)Lj53b`x`}Ef}tX z@b+*F7&9h{jEUmV_rL9Xpxy;l8_HoYU+tjD&G%U_9J%>dJagl)*>1!5;X?%I-3zkg zC-npY8^lvxp17JPSOqHMEn}h}_U@W(nb~;rdAMN@jOUb_?|IGGz&KSUZ86+X%zad& zHg$rO3zh5=HkRs;O$)Vp!r*l)tO31}Qjuxi{epQ}rk}V1854z3sZFez8e~j&+HJ4k zn#T9y6H1h#8|Ca-d`gK>a_56NqT_e{4p_%^!NwO%HzA?C$Sa!D4n(-7-`%=u?~+?1 zQf5E6@^xqAoDp;o%}A@L5Jc#y2qP8fZPGwcc4I9Sv#FOBC1auhfkugM2=vMOAq~17J!7JfSFXBbGA4>dOcegA?>Dwfh2_#}V|}@@wz|H&wbf`Ym)5o`tJT%w z8vOq#nJ9{@YhI;XDQ8R+852bp{oNA!%Q5tXYHaO&(5W9Bz&yIQ-}HCA@PiWMy%NG30c4B^){AgsDsoE40UHiG}$zm3hD#TUC4|iZ3_NYz(u!KztsS3-5%M}-@lKCn8I(0FQEgL{3x zhc2262{ujLo)~mT@dEuFCpve~vU4xQ$Y@k6ChPyXm|_O8fZ1iTfUIcRf0L5J2MK6cP~OZj=1F!%L9SV84*WbkN( z4dM;ZE&ibsU6l+zViEsHWv=YMjf(LC%>kgmN7{BxPt7@<$>tL`0%Y@QO&R8^ai(m|SCj8R2k+g-~D(4|JG#IULJa_%wpy zuCvfnB2ySmJG_ZXC>ZN@H_kLP(o^h0QMI6Ep@a(_6-tcFjf4Lo96rLr4ta1!eR0JG z5}TLhU?^kTY%OD44QDR;gNzc{aEd7E#>5^;*^Okbz5QH!%d6!o{znc{=W2EoG9GXu zqa^LAZYkbKHZ)THD}6Ci3+A3+1;o#UJ5;DQ%YLOyb^N$ztYdUQCETt!_&;@!x;n<4 zDI=0heYlM=MGT4YzjU;Lx>c zg#o-*~5{o)VZ;cp<0_!GJFA4*mzDnv450ca>hOwNGgM#!p9M z8hbJRoEp)lioj19V@0+Nws_&i=m=i+@$}s(JZ~#u&)X-_Rj{fET|kZ#ll;ITccgSX z`N&N=#7nOyo>mi1zooAy+xJEmN3!J!^`z2G%g!jWWi)S~!qKehh5LwWPA?p-b_XB^ zUPlSBhFT;N7phXWf(PD%e%@IXbhX)w`C_HTK@ol&zQIrnQfqi|ny+F8kP-w@qLl zJ2-e0nYFZs;r<0QTZq1@sY*{UBo(=Q*fpAAP3ab5r8MZ@gr>%xn6_dQg&Fsh3t`MP z5v#5tk0I(r<8m3EBJ@8v$fXUSRVJkUoRLiwtnHYNG|+5PXgaS;u<62jPC7hHQM5Be zG<%i~Ab>Me7%+Akjz;rM#A>3d&xU|S_p1&%ODmzK%hgpEqqPfag)c?k8eB{HCYn}K z8=5BSm-zUkRpYuR*8A1U$|_w57b18hM{~lR`~n7^37K4=+ZsZP{f49VgdpW4ZFa?2 zBCVDwl!|)#SN?f`T2B^rO?;;;d-h{I;EcBaz*PItC{=D9Gay<3n?(7 z>Nvhh!OD^n3>K`-TQc~Z)>JZ_jT6yT)zvZdL|GCuwOT*5+G?R(z#lm%ZdV)Wwu!dq z=@e*yq<2c)SO;3O%chiN)l0S@sbQdqh+=dKg()56gAfWq$4$6zN<+$)N$}7{@8fP z!x!!bOGfy`i}F!qrNipO@D%*n)kl4?&rpdSNTpi*%o8^)l`1iNC+n`Nw63nuk=;nl zPDOt++<>m~X82ic6EsA((rJhmEE=M`%NJGjub)SM4r+W=L3Honkll@b(CY1Y+W=R) zMVKqrDytis4hSJgq}B6wz-CW;Z?Nz8{SH_^!uMG;-SHcT9Uuefc|m9%Q5+gDebKDc zG+=$O1k`>5Vx#t(pfRdI{1s~D296x316%BbF7`Rq2PL<3RVX9oub<<9_qoMW&ItQA z3V-keG#O-~f)a?G%K6|aEkdx+$gTZAP@({fK5VeBeT1ZYfvbGSylwG$`Ft4oSh^1s zh1?SAq6CILX2ddpsF#I=ZF#gBjST3ETmTT`Qi{+M`nGfiO}2qjIRp9vuq@FX<4!N? zj6pzBGN7-XN1>ho|2FK^XCAW|&{qcZ)dwpggfNr=eWie7%7DI72rC2nN+GNa=xekP zi;S58eHqR>F`KE7yw(z)+R{N^YCv)s&{qcZ8y90DMgiGH^pe28I(a4wg8e zItZ8~Fu)0rLBpw+j-I-LI~q>CeDu^6pw4jWm7}L#wo9gZ^wbp~$Z*+LkDhwfUiP)Y zQon_!8WaN>bjL#hm;QbZa4mR>3cmDo2J=W|<6BK+Dbvxzd)> z)=gsDFfA2^F$?H5{BIqU7pNrALL9>gPX%ho)Cgc_MZvT(-JvJ84F5~OJSH?_vrwfD z&$jGu`Y@SBF>O8tGSlSrnN~;H10;~TVj#N!Xh#m4gm<+IHps5dPYF!eufff@q9X@l z9r?>j=BkooP_#0`WRTEcnE_z^0-ESBVmPj3U7P~PiU(lOSOda7XtxAc02YSV=kZ;m zLxY8=HhmAtAi}tUcMi=M7`^bz3bqUrEpF&Lme$rVryg%`s}z1355j&LK5F#i z4m8Nw6eP*dDG@C*BCCxvuiY=9)AfUv*V;SS_4nWww(CCUK1)mwyuwy>yHVd-TV8F{ zi_4Ym;@Wb3z3ea7H`W^EdbwC%Z>;r04sy|{d;1=_AN~ONGl7!xnUSSwgb4Q|7~cQl zeusqAUI41JFQJ$%4EUs5ZmLhX^s2b_2GFOyhU`{?tkP)7l8oy{?xAZh!G+k3m?W98 zBZ|30XBXPFTlrDK~Tyy)-4+e5Av^#8sy+7U^4mPjN@x!uX4%qDN1x>Ui zcd#Xc;cdzPs?V1EG;NSzQT!D49GPF4yVs{-8Xl>k|0w=B_y-_6EhG0V`}u0C+k*o)0o}x((_-jFkz%!N|7?Yv zK~thq&ECTUpB{K&L1Tw}9+In_g)&d&Kbdgz`JYU{I{PCBgQr7PDbkyOSOcr#iY)kx zo&7!bOE0XDo4rpbQgsvY<%n_8kHMTPVL{wkQ}M!jgD)pcO|Mqa2G&je;Ia;nUeI(= zY4;apqUm7-s9Mfz{#GJ`A$pxzvl!kS^ofJcgsKtpH`5maz`IbL%tWPwmIS;D({t5| z-V#G9*^LfniB$t+&N5?Y`G_MV1UBCxcxZ`?i%+n9n_Tyvi4b|5L$ftkccLa3IIwQy znK>`?1Odq)Eaf6Q%t12nQ4SFP+;M>A0}j7wmYHIp?TYMW*dW>5K>`Rt+JsL)#)Lwe zkpvEs>KoAC4t86;6-LfE;0B5l;mMZv!NI2aQ9|CsR8cbQ6#X z9S4&%Hm8u25Vi)=66S0Soy}Cec!;;Thc^V)+ax z)i%1U>>J)H!D=FQR&+Dff2~fu?O7n(p1bH$I6q?2%JV!b#j1n<6c*D^2BvN4Q=vF| zrR8LGEgVlRY!K}j_D(ONF`}Ij;a0j_8limIO1Fx|U%45HiDtT8NG$yZ4(1QkWfBaw^jnib^)0@3|54uRV z`!=Zyz8K9dXbXImJ&e{@Mwud_l|Sc0ja=KQx0|~_6%bh0^U2k!f?rcoh&xj7JUy8K z+vDKKri8EVv&gyQo^C2}Gq=y8GqgcH z@*7~N*H_`qF}U|^st?I;^02xDE^Mr_$5^=wXhsxR9Z#e43mE*&3rc5rs{~zFJ72Qj z$2!OUwSyM>m%-)pBss0qLgCX?&|+Ma@kakdEc51)?Y_xGF12JlVS>t9ooguq?+(Di zYDXL3v?k&eQ;q#>9y(C=C7X&I`*==d`*DN7+$MGJw4X96`?guR6d||zM&+q!YJqyVnwE& zxZrBaW#zh6Pq{ik%GpIO*~G$T6ZV=P_S`e>)cvp3@A;kE%yMDr@w>=9Dg_MKUSw=7 z8Cy#ieJ`;yc(qW~0RbPo8{O|Z&ZD{1-8-@~xNX551+$6JqMOTnz|HE2%m>_A`eNk# zy@Z}I`Foqfq8jW?be8Gy;dYIv-U)xIIHU}yHgX8~_5@GWT8pNF?^8r8X9UKA}y`1;#9V8^I z%m+M$TXW_Eoht9YvyZH)bQ<^vvVdF4c(Twd!5_0H*b5Sq{Y zx{7ng{a^!=sp1c~Y+#cMPGXG8o=XeCGOn`{qqPYQ4<$J7VqT;LP=#Qsx9RyWC(WzHDzO*Z8WHz?^iV zS2=V7GX&ekPUupHSyH^CWTM&%OmDt_==pSPkMB4K?az$I_pyM*|M3#~i*e0tcCEC&;a=jV;2yK5Rc*EBbM7L#!ot~S)O;G^ z%%QXF>A`Nj;qUD4HvJ9+pGPy1V6QkoXMO+(@#Y5r74Kd^b1W&{&a(iyyp1;5BW90S zC(f-8N*y2=@>mEM5BKi*Juq;+_0Y!T_pThvQ&ti)(kd-O4WDG6O z&lyqxSbali6+(l?(Ig553wZtMcb-QOEU&n7O4>U+XVhmWHhS6HFo z5Gfo)4U<@jdy~YCf4LJ&UWyOUB$EMng$aMQbE>SQ+bHJ#JEFMq$sidYpedFOcfyoz zr92!8M4jr46WSm-0%sx1ks2fn%`Cyee+IVJO3K1ag{@*NADLUwr!f~ZuYV9OtM(~4 zp}r?th5H{$M?}^PtR9B4CLEv(c#CfbRsyHd_&kbI9*OhV6`dLuQRb=0MwHBxUJMw9 z*&q&pju4sAnmw3!7j?UDy?eECBgEfXKE*M$|i0%mjQWLgsWR zGXJL)T9yGFNF&X!__~^eHb{fKEoOVo@0Az+DBiF4g^-8@xDRZoXE9G$wyjz@l0R$* znO3K_JFpP(tq1j&;v0=tyXk)@HW&jM+fgWxCEk_P&5b2fbxX0y%7G1Is)#p|r@2Ba z5=<~y>2f97WZq&*}T00b%f7A zLOPy`uoxLrf&MY^3LDL!sx*ZntD@Sp$*1^_O>Y zhUle8ba`86h+d9Fm-l#v=#@xxd0lCUUX4VTH<^pihEMVCdA*{+;9UG_-&3yJU!}14 zd7)9!PCTyp7$jqBM|=@m3-k($zwaEw!Qb7a0#WrYN7v1yfb9gYZ{}Nmd|uzRt9j&J zMlt3-PVHKL(!CmkRmm|wujTWx(HNv1a60M2?`d>O{sz4!%Z{@Wz%S5;259%* zf#2B!MWo`B7JLVSQc`yC?rhY1UeO~A2}~U-UoVOTK%pu_OUBT$F|=X~tr|mDjiGDC z&~*_C3UlU%QDUTx#5V`3F~b44w7?*4Q42nUPI4(Q89XwFx$CNDLP51F1T++|1nT0x z{He#_1^2#{2{jmgp9L<|L!{!o>s+X;A~17gY^WI1%;kSC`n#6E3lHSH^~#!e1%#gi9f;%o8q!urg1$%o8s2giGPswJM#j6wkr6 zBY1XcJBZ8^uJ3rvJmH8(y-??xebq8HRGY8b&paFIujfFqDYS=$j1Lv?OXTg^BlCsB ze`$7iQ`VFMGYXO}8o^x}R{b{QO(LF=?Ha_DT7k^vO{efl$=x(Omp9-V3(=YsLg2HH z3E!!WtK=Y~F%1KSkD<}ph~B@V$n6gPP8^MCi10=6<^60CnZ9QvGPTxD&QOWXJu-i` zo%(^_1-FyqrZRmxbk*ou7{3>vP&n32>(Jtp+I5{f%hoELH=Ho12A8S@cjg9X0v8rX zjamURl2M9obf#Q6D^gp5=F80QB$cXQ4|ZG%Wxgf%i^9DB?-0*i!S1n z4sLMU7KH8aupKa#SrHew%B8FugM=twP;Pw13koQoEXi|`yI|MKodkwR`gJ};)$s7; zWJ*k&Leu105?TuXFz>#jy;x=c%Sb{(%qXWBeV zbaqd#Mqj0{JC^9|0$+{378jk}?5oj3I6A~uOv;^DIJmZJ^2JMvErlE25~nbJdTbn5 zo27DjZLLyW-te3C<;t30UtV7=ZZB7w+lAtGX?3Gg?@Lv@iOvb0VsHDf-Jm|Z?K)85 zY`AMGcdZxj8Cz|1s;fm040YBr*zC^+Z1%Bu!+tV}{wOEBVPQFLdaLEN&xSW_;IT`n zk34Ov2Zz`a`T{e$WS>FRJP*87n}_XYz1{N^PhH~cn)V&A;7a`taqvx>5BUBXd}R@s z`ExIyS3v^q82L20I&vfWsF=jA+ioxZN{q2Q_Y@?#ss`1_W&xhUP&mc(z8SV{y^fww zP1gi)JvH_JD`3ou(R>6S=mZ+0@veX+Yy9(kpXU>?&ok>)&cZfS>(&4B6#9cv^i_44 z?D~L90mNC9rr}EWp!Acy}DQm_KshJp@~HfL4b5%JAqE zJj!!}CkNh>Ppj}W4^PM7DK#k-d;56@>>)pPF4J^YfX0f_iM?L$fUiAN$#33(+J5X@ zBBi%I#{Yg4#YS+?TM{PxaMTh~)UdFMgRkqs*J1&zw{mw{Z34Mh8gpi6XP2gcbQ5^W z+;NmE0i)YxbRMWofB_BmI}dztH*OOy6L75n$~NFiOR6$hx)w{jHi5=*&Kb=uEu&Mg zTx_@Qc?X^S-oAU;y@)1I4j4wJ0r(x|UV<+prO~gCtUJHzj4Zu`oNf2|?dQ>#)t*Ir zMDjjWDsAAu8%6O=Wf%3exF!rX_gMCsvhuyUbec5)NT8I11~?x1J4>I7l|M|bc+?q9 zEB>D+;=ghG1@skF{A}o?kbh+z9~yO5PsNTNDckY8H}`KHh$S_#TM^XWQNud^>m4k-cRY-C}BVYC>p6iBfoSHes3ljy1qDriKk z)uk`@>-_O!qx8i-(^TvvRy&r>^}kL`MYC3@WK=XGql5$ZfqL$rywj<-yM$pNSWRpS z13B_4k?y^S7HU^?@Wv2)S;k3I$T(>jO9AHYNU}rOSz2;rLuO*4kzrBW^niyXuc_-d>PSlUd>~)W}(ce})G@9*9;LO{{1!yTQ2D8FE1F%sa!m>$BiwNe?_V zM8K!$KL=}ebeSc-U9cgVmCO=9g}roUiJwAPnI(P-VP%&1eJy{r8J=BU_FqPhz!|ct z`u2u7Ly9VANa!Y2ZXZ)vmMbg@%9UrvNrMpGP*Z15tQdc1miPr{DMkOvIBCqb@&)P9 z5vJ2A4MwMvyU2m7R_AkSXpWcTN_~P+h{H~n&pk!r0u#!~ZB z!BX=r09gD68tBqFuKvt;3PlZcs~eT_XM(OKy@4*NT3Q3$xOZ#@x^eH=40Pk($qaPC zKCj*LKkT_@+^PFtBj$Ox55cZ-&tJ+6bOW3Qv#VYPFWcCMzT$V#@8;3}=AcE@j<4Ig z*RF#luJ3)&sULvHkXK!=RNRwjg3gZwL=`NlCl1^9+xs8175iF-5=^Xb`7;1;mA1Lk z`c^zG!c)ci_M&{R1W$D1+w<}vvCNILy|o(0{?DVx_RMbPDJqRcg|V!i{92S>OY&=Z z=`6GIL?~Z6LCkEA&(QPfSVK=R^}yp`XgfBZ*4FbGV(ab|%>F;{w zjbat=8*MokET+m*%{H(V=X*#s;+(&0$?3CepIJ)ZUu$<@YxFvLE~3-AfX-VCLeW(V zAoZjxIzRf(zj4sNJB8BvyA?}SaBeCTT>uSM9DiT}0W4)DfX*qQAhIb41rb8NP!RCU zxn;C+uTyWfz^kC^NuOY^7jwlH*#tOXw;2x~z)YMhpxc zvidxF#u!&?Q5qXSzujf@tc)!R%+iIqkL4UT!DpFDwdJz2g9duQ;t%U2~%k;`7!Bzp%MDhoxUC?FrcKgdwI8x-?eEx4Iz}xSK z4*us3dNQs+{61KkDO{4V$v4t9m7C!u`P5RmV9P5@o4DitGE#CE?CT^)U2e(tG%|m? zv~nS7{jP(q$Qo)TNFpfZx?GlVMN5XZyg^2`p=Y{i#lL1OCYuCxQ6=5e4I(U{G+KxX zizqH_3cAOOCR4z1sO zk3WKcSvf`|%A5(RJEQ_Q_#fxca(t%TDrHP zspPv~PWLXCw0wV7zE6gKt$PSO*92MbWlAYZF5&}zK>Lh5u6z6uo)8#*ZI-eoeqcK0gl~xkRvx9?6{57 zAGMY4tgA%W$)ZcTbD(;_a;f%WR3yq06-495j!=55vXdp46J_T#tbrt@mGDA?Fhbsu$gux|9^X58yv@x9fsKjiA9qj!686`!!HmLDQYDUyR*C4 z#omWR@bpcQ6iI!Ye5cWJu>e*C7Vs`W9`BcIC$X*A72AoOShPDZ46_N}LZTzDhZ9l1jy?O2v6S-80?O^Ll3X1Gtj{sF)Q@ue)Er ze*K>Au^V>cy!9gnk(5VaG;m3L!8Lx|5XKLP9;ZICdbgcKEH!ECw*zacf|&nP_SP4O z#0<77J4^7)R@Iz!NyvuZG#jWPdV|TWb+eIj-FP2#O`7V5X9XZhb#a;Ps-WH;B%wt~ zOhfurTitqq(uMDZTr5Hg;ztC=-V}vTVVtY0ZX_~l4g|gmW7ix?u0P5IA&GY-->suV z)=)iu3PfZhZD68cRyTo2a*qC{ZZpR*>6@o@nWxflm3S%>-49palJ|PtAmfoyYu)v< zBh5xTJ{s=(201AtDhL2G-j=FfdT>$d(p#80U5o)}ZE$ zX^?oyf`rmFs7WIUtU=8hNnmw9ZRo@>o`>B}8#-8jvY$MvTU24fg+s#Y%gdd{mU&SA zWl73tu#;hhr6C?Ip@m|rp}(-oYF@A*zL+I0DOqZT#)iR`Jdfj`OP8Qx+LW$F#gvpT zOvRKeC#JoUxa5K;DeXB?&Wjk>l0JnpG%p<|#I-7QQ(RHWm>gGB>ZrJ)qNk|7PR+aH z*{zV>@(y{M-Tc|PPwPJ$5jrC^u)+?6{;?$cTFgFbv(~PycVv}cvLgn2`HD2Ss~heZ zPd;ol1~&b^=WXl126nJPTwF4#;=0T9=Hn&+A=o=YF)W3C zLL9|Hh3$dj{~qvgK(!FqclS@SbSe-8f~j5kT&#vIl<}P*dLX)2cSPr7iS9%Dnzj=3 z^gz_F?ufQyiS9!IOGNZQG_dZ7F2oYuhYpsA=z%C<-4R`kB|26Oi<-Ih(!^-E}t;*qcL`(nDC4xQFOOy2pF5DL1X_$z$q^PFLW-aw|CJkys)|m*96ZC zKq9~|CXN!LBX*_X_LAh@=%FZ|=*3%+yJO1V3ap?C#1LEbCN9br*$-9nl~5XW`*u*Q zVb>8nuR2(bJ6jRH1cP7=huAG^#5y4PsrWJ@lQAgPuxl2($r=u^Th@rx0%MI##-Lck zu8$0yHFT!2@BzgkcFP*EreLg*$>^Fj)A(G&uEp#oYdFMiStC{+j5RVDgJKQ4?z5Y$ z;SjrJjW88ptdYqW6l>Turrl%>huAG^gh>HojZ8)ltXahtZ+1N^DQgtRYTYs4-$Yk! z4ys$82%7`O6B&PmCsFzx%8vy}L5f#q`r-!%)h$1SMFQi8jDMH>2)I6QtekvqCm8aD zfJ>YTE5s4~DI3%4guOyG=cvuemRTDq`!ycp}Uh zC{JV(1|lc2jySM^qv458$&Hf}Nz0YC3cj4kBn*TnvU0f7Jke>laXgXKTZ|_%2?ODY ztQGDwPjqT+98V;@7UPLb!a#T;tAjhu6P-RA#}i4R#dso<&?QekXAtS4oN#OBQ&h;Y z9rNw)FP=cup4|GfA^R+a(DNV29{z}VL3UdA8D!;zHUQ)(# zUL|9!j`Z#r`=8RXe=Q3u#Me;flOZK&w>uLA?11e^+iN=57#r*Lf=u4`)oxY^1thgmb^ae+e6XaKjTf>4|XG0F0 zW8^?}lwKu3JHhoETuGW`SCm*+DNXi&NL9R(L z#5^fW#5WcO*7_8&1lt&=_;zOyH(YHdcSGlGfN5+rK0N3 zC7#UaZ?-PWV@qx)q0fb4Y}MSAG^|xwB13R`xYnwL^tC{I02$O3=~9Gp<4GDWat-gk zQ}_}U?Gkp1<1-t%6(veZ&Jdntrn1wafqaM@l<%Zizk}V1X01}Qv**kx`KPea9#{zl zhGOFJuKKub$&3-VMG_IMaafF)&<*Q%v8yqHC~;L}2gu4N?xn6h$qh&xoq2&rB9x*7 zN>unu3!7H_pLB-v#;TF+Dt5#cAs$R%oDbiUR&XgAfXii7I}wl*$5^;0L6u_&N`^%V zsz%f40q(K{SB@+w$*NdUxs3bbFW|+lpsNxbIliFOm&mf(6(qjqEJ=_`QvXSiVKcFkNWz#QtXy|-chKNH(^W{) z=0@rnP+T8(bxE4@X2>m_>f&>#>i(Dg7&%niY&y;L&9%ll4S{VwVLrar*;;qzmzE0k zQoTNFmrKjD#oByhwoq>?fbR%lC!+ss=LvbTebCu z`Ivd$alBV6o7;{H@_ut`)mg5st->pZW*fY|=|nFIEw?w;*)K@`GAUrL*S6NT*POeJb_?=_HtTf=~u}s42=Qu6L9n41-J=_JVMSgcv@aUp<~2kg=9{`^Y1Z8*?lp0 z+wJ|!wD(`LZ$j_$Cs(2OpKENr0KGrgU|I&tsuAzek;w7?PvM_?QRE9L@_1rY%p(k6 z2|H^i#MlZCOzY{ySOys;6zhajU^;u0jCnlwa-td*7`P5H6ke`vy}Z3?DY}o?@!wvxP znSc*P0PYS&rpV2?xX=eZbWN!H?db0Br9)o#W_!vHAcl1bN8Y8cN96B%OrjjHQOUC1 z{$Og=9E0CukF*K4I=$Y8g-B8~ZgEAU>Bb%B!76*4$%d@ohuF$omSyEe15{QiI39eU zIg(oL}K?kM4s| zd-B0RHTjs}40l#*t);bwn06;2TY#GRVwj|=LzWW)EuR&H#+09`An|e$1|T01NW99+ zU27yxP%@LnwqGWy3(m}PY8`KD3vzLSu~$fnFSSTw-hr^dKQ(yj>p8eL85eEi|>^VJalq1koRZs z8-H^rHnm+G?A%e$bzULY+1UepCR5buNriBlalf)NxvHB2!2rW9i8kyS)$?FKg)Fq8 z`N4C@;SHV#Gip>Z=C=bU3L_C#%2Qw^)t?!3X%L+SgJ^OKBVd+}fPE^HigqX)bNImTF+rloUv68SS!x&e&!fcAyR$huwt@w73LY0?aqSF^!rkeyEUvVs0)4^CRS9sCAni-Mi=a^~RcaNCWl8pq^N^ z&1Z-ms$5VjtA{vJGi-%6OXNbRnWEYZ$9xnpU@C%nft(6eD~U5SfHMpl!Vsi_o(`3p zch(xUcB|Q1aiFssZdn7$E|R%W*|Jkxs;xWgDn{(+N+O485@;J0*2cZ0ZCoV_Av_i| zc*HaKWl{~b2Eh(ovO*}uH}^4eDcl@neL=Yt?l-i$t-&T-eStg{hFy z$j-60yh0XCO#bH6wx>oQJM%+h{IWYWy~d}e>zfY^Q+?#*9~wu)4|C{KysV9uEuc?} z=+hGVbRK-E)!|mdnX$R0#yuKo@5+r=!7!uuj5;PCBaLK#FXNpfeLb`IwG3>p z=!$b}ael4#pwV_*$FTd2=E~|;XEr~_Ceyh~mv7&``NHC>x1YH+_uA!~FI--{`i%41 z<*Tngvsgsy(cu$S>w>pxcII*k%jFZ6v!lx8Ha5YbtqEx?=fsi}iD^Nq5m=m}z~rS$ zg3?F|@G6nS;8hCUBUbFbN~!xQ^W9e|cVDH_eU*i%DmLDgky2ZPVFS)D;O;rM-he!{ zbt>iVN)ZJKt>ro=by+Ll6RbS=hJzzi6s%LumFJ(i^3sj#FI>CrP65{!Uwr1uZRfe0 zmv1~zqdiz+&T?Y~Ibb(-I`mpb-}BZ}-n2EpSo9~Yp~b}iS>Kl;qdt13_-e_Dy>-?v z5}0*;$)7g8tZwU9CFP4Y%yb182$`~04LlQh%`i%zGSQ>pQ7_F5lF>t_rSQbEkn6DzzSl9fFx?B=ebashU3t1!~mYprci z!@!`k)!tZxP1PpkTyC`LO_00PMI$>A>VOm(lGjDU`m}WEKt9|-IFeVwbAo-9DVNjt520^;i zu-Ft32KBJxmslm8BJB>VJ(RmC1y^$##s6%ke?v}#D~kxtEahduIVQhCQQM z{gnK(7<)uqJ!JT5tXkb>JmyX;Krp#-YVni?nB0zl=vi~ z&#DOS5T6#J`Lv*-k$_K-p*}p&77#x5ZeNu+CgRhos1A^04mluvJaQ7tCc-*&z859> zpBn_T&mm0f!;EZANJMBiFgwLIS<%MW){Cb7>nPhWRl7@bxy&jN+IF`jOxC_JR`sHO ziBK{9islqhzq{kpKk9I}>4)91;z%d^6qJiT) z!c=gCN9cKv%%fdV!UH^#wQ!6_IyF2>$htVjmtK@{ zoG+L*j_}1lqm}uxE2?*ZFS729@ue5#`%9?2VA{9-P+I27uB+E}jCyVNqF$q10Z3sp zb(kyODZIoLTV#OPL6z{6Wcb>~^-a_L94H*?^TX22Zp)f^;DF@FTdL21bu{z{Zr)5F z4|C{KysV9uEuc?}=+hGVbRK+ahbkh2r>^DD@4$H%#_ZjOcwi`q$bMu@AV2 zWoO7Kr&gCj5qT$b8{1o(+gtiGxtT`-?o6hzPS|1}nSDz2@4ZBx^!^xq! zjFm0pWh+?O3SM>rE4yG{M68H;Gi+JJdjb~sFc<6`+NzDTYNI6!NXY_PvWS!{q9sd6 z$r4&}9w|AGmMkMB%V^08QnG@UTtG@L*pDVT{I6bt8!Jy~Z>*fx+*lbiWo?A~%F_gb zn?5GHZ);+l74!4+CL#NqEtr5haN)xJ-rRClrSEn2ox2blCWMm-`Q3s~HyL)_Lk;Vo z2+9Z7F%A*pyJEsMkic)a;t}{wuW!iZA$SdVZd!G6$d7AtLnteQDW`A)P(v z`N>Qz9)lZ7)VVJ`bI^dqoAh4_{|-CPGHtIw6-B1R<}G!-Eb+0`ZCV1;UU_DG+?-Tl?7v69V)qV?8`$?C< ztTpdK{NpQbj^~Krdw~Ge2DlFMz%oY`5Bt|Rvcq(`uwEtyKy|Cvw&rMnZLe+37_TR$=FvRnRlbmYKy+>5o|uX$X7*DBonsDhx=niaXD&qMlg=nOO{#{+LxjnN=JR zvr0R{td=}um1Jg(a}B}giOepBjQ>i^D$xUR8l1PO|GGO6izvHF(d>#?Ff+kvYkueX zH6O*Vz{JIn>5pF}lwb3M;#Ya+`BjeMS77d9$n?jrd6Zw}LGi1y^Zcqr@hdQWF=YDV zR~hA3r4N2dgS{de=M#&tidsD^yMdWxLEm`3x5ht3h&N5;bNbfp{#%#S5tT88{#%#U zZ?*Me|E)_4p4xh;|JJ4TL2Z4$|JJ41RBgT7Tk8;Vj%}7;b1{%T=Z~dH#%od9c8+t& z^hdj8`m%~Vlg61}PIzsJy249IJQIF7;+gP75zmAdhIl4C5X3Wa{%%n}c&3kw<7eri zbRH*4N~DN~5-sAPM2vVSQFF}WtJ_mP*Qg#tcj)6np@~p9&RgKp!-5z~A~CG^8I201O{jTuW1*01WF7 z3^EoV!8~{iZp|(=mfX}n))7x6{NA)q1@kw}lkQHggvZ z|CQ)do$+5eyKqxbz^-eg4L77AAr2RP)rTU1UDSZC;^x+aiv-NAL$0OuY3nWYKCXr~ zK|oTWgp(yaDVjWkN<+ZE)6o@&ARx7H29d5rjC0XLdLS_YpiKI0%O2jQFX8>P{({uH z>>EAKdmL$7)7yc3U*GJ+Vgt(>c29M=Xpnl$qM0p$RT#<9rA zcI$3y<9;g~ZR`{o=+ro4enc1wXh8X zM%HnNx3IR>fLwF;Xok2}Z3W^7HEHHJh^>%xTF;&*7DU>E1V$UJwFflj-o|#j-hd?T zv~SE2vd>BzsdRRh90Z+yxw!%n_%^mS%+uycGD6Z2{Ae7G;z;^A_%dXtUL2|z-)RhG zpCiVyxp?zw^6{u{rO`>+jjio=i-wL^a+wQZB-S=Mokqtg7K$bCF!1AiOY7ORrceA; zf;(CEuui`%d-z&YZ)qU);bJ$iHDN(r6E2bqv1@|so@d+Li*EHTm<8VKaTb`-%!0)9 z2L$;yf6X9YI!aE<663%gl*WSW)Ii27Fb9MKL6*oPuKX@FJ2VVrePi>%MvFa(3J00z z)gZcureY`r-DPsgZP!Msuo;VHlZ*@v`4XwPO%&pr2su85X0eZuC*8)1@r`j>#6)L4 zNoL&2C8RPWnqaX$-6(f7SY(b|a7)i4r2|p9+!{1wr0Ty14nu- zRdg+dVm0box=EgLaaz!|6bfb;-wC(K({7s(I5I3qy@ry!LW-X7RVC}158o_7=#9_UlkME0PIlJC&)Pl z6_Z~z=%im56Hzy+mlS&PadMWmj7g!}vOk8rqHx_MEL(27e$jC-K)@V%&!#JBXKQLC z5SX%9;DIU4mmBRB2x)j_wNbyjxzU6)xDO48bOdvZ_y_Z*wKAKKjcW3 zQ}h8?hmi`C{FPee1bd#t>bTIId~Qxt2?fL}<#mW!RRI2DIEksmW2vuT2gmx1XB3&=p5kds-R3Tz;(4J**!X3-1 zp#93Kgu9hhL3@=|33n>1g7zt^67Et~1?^E*CETH`3fiBnO1L{&6|^^56}2<1`(%P@ z$<6xB#!BNe&u*`+L88F=s);@gg!>B`@UptQPhhhf*{p4WR?;#-V|=SqsLst@%35y* zz3h^u6f-zomqlv8z#zn5}V+qF+KqlA96rm%o+?p|<|k%8RfVn9}z^KUs$v zo>6d^PpHNyXI&-4%{RE}<~HXi7n;IxeysFtXYv+psj>7d1cj$@dFS95t_^uCU4{g? zkkg8lBnVi3$06%48sxA7$W$L#M+F0Y{#a*}Y79T|{;1%F|1cri);5hzjIG$_k`(sSa1?Sy6UF#jEJon z0~e+TVGWn>>c+>+^ey{wAnp@36(b6ezN4`X15>Z5imP&0+tF;MSJf*`S4YFNPPko- zDU2#Q2~RR*q^tN+Y(XLv0W$PI#C9v_V*xt7=!M%ew^c^*%zJikUuT6*C0L%YtFGq<)bPjFXd?1VoNHBLqbF9I2Eg5D*UKT~9!e{Rt)@U0DcK zv~Z#L2~Q~0eNad!x&rPA0z#OISA;i{q7l3!jaoV64HE*DfLI}j0!%Z1`3K~37#Se| z!kJ8^$QOV}2>19(fowFfzE5D_Rq?=k{C%&-)opC3#}k0``WYcY#U{c2U=;QOQa~&@ zPz~#273G*d9@T9YAQY|yM^^bnjD);!0#r%i9jtHuv%GH+ivZTQ3C-~;8hHQ0!3$gc zRhvVoe-ncB`WPWb#VA1^)BT>r`>?{Nq8-!6MzTJZwLPhi6R3L{nyl~jCU3-sgLQIm zCnze1@jm~G*XOFyDAeb@Vd04YLaK_OeqF~DzYSRy!kV8-POL1%K@SR_tolhpF$mqy zmkz{SrORtoKRM(U;=K#sD=% zX)Uvubo6`DsdfWCi$KUx+$bE9lYa5>BnB}|tElQnG`3F&%qL+TQMGpWAu(Kq4=iS~ zq9{pB0!=ZFO6!LWqCa=TM5gb9Y_%hBnfXhz}^_CL#!;zr%^f~kC_4a`Mk0cB{xO> ztS>97Q7D-+6gq&I50vS5DQ*}I3)B7Q+XaI4ytWWCk_e0`{IBQr=k|a zsR(h9)lV)CDjkCr2VDKM6R4jtL|R~|=CcE0=)=UJ8!tbVat#U|u> zgxd;k)GR~le#8bC(fx@$HB~Ohb-!vK)OCL%;C}ZbgsF%{a44et5sgu$U~t`!D~yE% zR?Z0GAjBmf{D}Dv>-~6VTa~qVpZ`^V$g6h1P@l(x^m`g1LPbYo@(uR#iahQR9ZUE~Q2%tz`@M^p?;?5^GZJ#|XqBCDy{p>sbiEq^((he_2o)U-BO-bi zQS4Nl2h+Q_LRUzj&}oRWw!E>6nA))3JR>!(J^(OYo;<25^wHeKTnt6#Z~$?O@){$9rOU;cV# z@oO1B@B2Kvn|r^xwd&k&G*?!)IIbwJzMNwVU*AUuQ0tuv|W2 zIXkLcZetUIO*A1s?wsRaiHpRvAk_$5i;Du2mnsQLBPqbEL=uBnDRhrmvHL2e?yJmq zU!~lAl}h(j7NV-ycvnVBZ4rh|t>e@-n{(?8aK*1vDR=LIML|Mqxyn?Rwemf|%Fo5Z zI^|q>{+TN;-MIe3wcD2$uRh~kUwrYIE4Q8JZeG6eyamhsaXw+Wdq;=f1@Eji>#&Q& zo%j<^T2Fa%*8F18pRtA(6aQy@Wgpzja3@o^m*M`kEXAF_y3Fq#RCCrTWa9OG`mJdV zm7;|350k@%`3DGcWHA4s6a-9Em*SU+hf=UIaZZ$%AW2UpP|{Ngmh@BtCOt)=$!Up1 zSsR6BqtI*=nvFs$ptKcGXay8n0fkmTp%o=)fg6~rI?`VX-$%**rL70l#A1;t`_CrP!|J7&P1YO*%!Ovet|HM6=idb3=*@+zq(sD35n%#PFP^|fAT7!?&MXl-sDx( z&g>%wj+|z$Sdh}p6~#K8y{yK!$&pJ@Y30K2Xwt0k9F46$c20Y#QghjoCQd`9NfUc$ z7rFBuGN+l^TYmN>?p&wNb|&+=Qy<&&p5+diZ%uBHM_zffhv#CGpRKFTOac-{5Ql`k z5|C8fJxJ@l{p&)O3DC)4}P}Ijh6l6ql zD99ThiaI1Y6!ozW1sRPT3i8H>q7Fw6MSbi;K}I5ng1qses6&xMQ6KwIkWt8?Aa8sq z>M-O`)W<#)WCU_3$QvJuIs`cs^>Ou7=9oD}G#38Br0e`@mVA95IedQ=tXIyRjim=p zxty<)Y#+MI6dZk|V6l<{HH7TisY& za;qLEM?;Sx6xO7PzJMH^_zOtZX`UdHtf@e>X$bC_HeV;V*sCoF__xunwcE7^_iXo6 zl08{#cD7a;5WKUo4&hh)GTRU+3#=gg5iPXb-dJY^py5dYbG^2;zP;wK0}eO#=y`IE z!Q`*i0o_qtNuLYRAY#)eL6JO=sS9W8AY5ImeKJz)v1p z$RuW4@#DW2Ebfz0#siov%zc-amR_^1V=}L*o9&II?fMqb&L(XiUGb(Dz?)=@fmwtZ z3gDU}{ENk(147l$0#tND>jmQLd`4$!K?jx`)UbZTAO{#NZt^!TK`2|rRA|0Tdu--p z`_BbwZ-->*8o63M5|pJ>X1eD`Y97&Hh$+Cif2+B&zR`qU;a_++FlrE7lg59B+apXJ znBxt*i7FT{6h>a~BQcCMPpDxLHPbNQEjFSAOqOBB?KDSHA)G=F&>#r{>;2oc)~Y#7 zKT%5T9@+*EKGrr>2l9*-Houp$wxavU_jayvNXuA1p`ipZnWc|KDvRss8lP9>E$zt0t6V(N0W;wNx zjDl@#K`u@(_Ih$BWdlQs+o9s(1RN<|2o)EHNJ#NwsJP?`AI7~DDlVKxW*%2}@#gU$ zq5iZo^NHA9t{K0jn0eB(s04OACcX_4r2|`huXNy{lbiW*V+S_==1y#CVOuxJ0-d*E zD+DQ7=#Q<&XSU44=Gfh5XeN}KEK^waj(Ujr3c2nd6eiQ%Y08YKBuUkC8DgG}Oi)xk zpBXjJMPv(Nu_2&XCbAd{1_GwSkqA@asm%U}x3iE<>G8zrhN`oWIjM`{r(lKG8!=Bo zsw4f|8<4l?s?GpPY)uLg%jEmuuwkr}nieNaW@?&~4WZ=oPm`}1erUAo_!Th6NVz0STflOlWv+n;Jdgr^ze z=C1-f7fpTSSqcs75Fy?#un`7-gBdmO9;~N<-(V4h#72;i?vS70Ghq_wwCo+8&Ytre zEvFWbLAMW^sZ4t2paG^Z`Y#3lTkB-h2O_6|=De|v`)^$o{Mpje4vefW|6K~iSjiHY zwr-F^A|dGjG9L~>{vje-45U2-5@?rN$HjXJu`3e78BMKV`o{b~i0A~eH|7yC>Xw|X z(S_PCA+M0buK8tvh8?i=?hBZ7rW6Q1&K6IuCgkX&A>&P!T&!lO%){f?gEnG4_uv9)>x7AME_zHQc_`n)d-u!p_1xz^mWxGYFee3Y%jj1d2U5QR^K z1m7k(_m=i_@=sMHGS(Zx+zSzhD1nfy>Filg7>+GY@KwX85_*W@{xjfq`nbO;!0_5^ zwEdB#zXOoyq{dsCEC|pefshxbL+*I<*=U`R%ITJ<=uil0VATju6K;B6tiw{f%w%|l z=I!wTi~iHo{C1F4_Rh1)*0U-vvx=dD5EI9$H_51 z?>xWCQTz(bT@0E2_%)C6t2`)vRd$|Vl_-7%rZ0v}fBY(={HpZ9FKMt>MB{v75vEtG zhh;Y~lPu^P&-d0iB&GNDg`B>1yZ_en`qm5mw=S*UD%*?ww=OMsYU`!`TbI@cwe|V_ zTbE{2we@mut;6vh+bqH60(=?0`^LJ%8+j{>X9@eUG|6}^O54tHE}8ylw@hDFk!R94 z^UH~I3{h7cR)}ZfL_$0h5n#kK>0+x7L&Osi<)o{waGix+(3X-LSmL2{{Z%}aND&Vu zTEs(%81Yb|=9tG|lxw5^~YTdkc4SI-wv&Q^Qm^*nM z6_z|7k~JwFN{3d~5f3rHq(J42CF z$ikr3SlDchwApbb7k%{>c>-q`1Gn}*H%T6ETyvLEY zHN8D^PCoQAMTifv1_z4RQW4Psfg%N|h*+NkMT$}pv0BfZ7RQlhIOk;-=kx38`TVEI zo3Q#R<76L%s z?%1=n+=30?@=JE9vQ%1HDrHZT30j?o(P*^WjU{KPaSv7!6OJ^VyYXt)BnMYE9C`?R z-_3KIx;T^_Cj_EMHDI=WXQOSxlj>#A13nh%EI& zF`!h4FWk!-m-_9CMfZe=D2S|%L?ts@=$4rr-q<2iFv*(Egx33^6>PBnr0etIhY`5zThq+&v zBV?bIHd5*AEI9~D`*L#yhVRDKhI!gNNk&Ne%8ggY;jE3&0X#lrr(PVY7~g3OWuGI) zvblKkY4Y)?E~R0s+KsL4w(EklFB99L0GUDF7g_|;uB}BxWUXh00k==Ti+91oe5j|CJ3^n+$U#q1k2+Id9#J!wrV ztue04WXSPfaym;0NnA^m=)GTf{j{snLC&qF&*1fd41CFkcel@A1>-e|S zeP!SjBN%WFNgh=wB@!@h%mQ_H&(B>M7|W zagYhUTp@mm>}QvE`Q_nJ7x&)ovRR*G9OV~={f=j@g@nQBk*h6oTGAU{vtc#y!e6+` zRQy_G~+mSng+x)I&;Qmop&70mvV{~!E-Dtp(-Iw-}eeT^v z^N?#0f^-U<*J&Uq>m<9fDBpJS`+|o#nL{pDvD9C%a*k2fiW*Z&Cs)0Dvea(qUYX&O ztEJheSY@e&s4WGD>0iY4ZhcrGrG&5U6U4Y}9wuY13c1CfWhuQm>#z~ z&DM(3f$LoLRcF1?UV)9zMFVa+OgdCjH`>jWW(&-FjC3$lP94a?Jt`#&+Td>(W~U=f zoYY!MafI!FYvgq{Yi%&@HXx+cIz4fKm%}vL7sMt#5{&+3R~G0?u!}CqFX?ZmY*#Au z`eZQ0+YS(Sf*kb_mZFLQbEmEEFi1`khSzKm-TGZ=f60FD=p6gI2APS+!Q}I#8w7$P zX|ZTR-nb}O3-OI;=FNypaADUGH1YyHnUd#P*KJ@`@xOpD2p~Z{Y$y|4=}@+P9MZsc zmP6s4r3YegJIn7*sXI$s*;zgj-^JA}HMJEtS| zojqL9>HhX8+=tWM)9H%SNiOjuJPq7whmCiig5w+Pyhz#DOMUEnR3zR{eOmHKkX<&^ zOG#o|(G_>zkeZjBHG&gjQM8^Iak*vLRU_DPsfU*L+%%%YDBM#I>KddTcLKAs{u;SE zrR93TBXpKtu2)6g<2tBtJKR;{u*%ZyXUXbYOH+#{53uW{u*zP zo=yQ1IBtAnl>B_!Q;S{Aw^+;)Xn#-VhhpCYRiuox?tT(i{g| z7M8Z*s2U=nO40Uc$lGq~K=6sK>(DsA-NKf$QQY%Yp_lPIHyvLf zy>lE-IqxQn|DLZ3%@V6ia*Rcf8=p0A4Hs{)zo^u8+Pe0k4TWysp|RpGhaI`rD98{bs9{IKV% z!h@f5|#7OV0YnQezDQWHnpP#wN9Fu%66*94>u1auN4}9TYilKB9X?SX~R8{jxtv zVb8ilw|_c`^oGN2B*@?Pl!`xQCMC5Lxdi#07?Xe(>87FHS(zzR7lD zWqS?Ee%AHP<6*v_iT&m%sH(M&1JMD=LCDURaahPVhh!n8-vu}Pn=~iTXmHdYHyAIB zoCK%J<6cj=9qV+OE3F#D7i$0%VvcTX!ez@QBs3z2G6ziJzwOMNg1~%R%ML@M(}19@ zt*vHl%^c1c5Z~-yLUftT!62byIzkWW2|XMnbUZ-lk;H^fW=!*l(*}-gtUD`>7VvIs zqfIiIqvmkEv9<=(O#pRt;5ePlMt$ZaIVz}Yw_0pToLz@Jm5 zQJgtLj)73t@47+>_d2(n`|a8$lRAmt`DNJ!?Kx!)zO1EXxl z99eIG3r)R4&SoAlPqHzKkD()(N6j%-z^gNWkr^C{k;AK-ae4v9^o%;D!{gZu|RimW7Qw??ALe-eM`$!4L>e=KMMALNRh{7-9wCZ zMNcf13wGk7^M#~EE2V|RMaxC=%Le%=<4fr;!d<>+x7XG#w^we#fQFo+!?pHG$2|9b z`V7gu16RV`D4OJ*j4M~LB9Z;?47(w&$H)jw!)|G~>1@I!efW9zJ`EBH`fY|UTpjbk z!*qtEHirm(afFb&_h~>#w_f`F^bn-jJj5RPHE2PT9H_Ng8!**SZ)-FC#q>}Xo_|*E z8u%f)27ZXHfghr4$S)2ZT)qW7h*7qx8>i{2HuGAk9qX!0*I~tC7BM8PFjZ_T7@Q^f)1~Gid=L}fG zXH6n$P=_;@%mdO5e`03l%tOvvqt+ah648*Ya4aYoFu&reyAS6{HR< z$@5W5a#>oEA-|8eyq?W0#4X9y@RHoO4tX%stI~KOb7TOk?mwZc?nk`Im(I~2^?uMf z`eWV?I!Axp`%$p}Z7M-*=TbRAq4`RJLgj@7g&>qPWH$L#m;ruSnE|SN2C&~x&q=E* zUr~v?${v1ajP4p?tH9!im{H5^Nokrm{4jlxF13SPaz($u)F}D|eu#d7AEIB#FVH3T zDX*VscRua?pxt@d`$4;N(fd)bzn<DjnFY5Wp&iEU8nt`#<@`Pj9v+w$=k{2jU%eJN_; zyDBYwFUt$xwYY`vdU)aM+rDrx)2l(@LdF`v!uPLXjmX^arof#2D@l(UG+O2)Ut`AJ zPlM2TJGjW@W6)YAjAua*`QMblR|J9o3M`qjG95ZgiQZ9FO7=XphGH7_w+8KkR&u6# z3T&VLxzChgA9Tr`V<+fjs(Q>j`hj=JL!*FUmWREtt=qV~3Vvxz|QBaqL?6II}HC~6IN0ERu-P8fL z^e7V0q(_l}A3cf$#F8CF0?D)ypp`ZPMAAlpI@-vmlQi;W!~T+C{il@VV6D zkEZ&uqSI4=5AP7^pFMEM@!@69`>Dmh+?VvwGn^0W(CVK#^nmI^I`8!870_d~c&*7j zM|msi9L15&QQnr%QEGALD0jl=D19HO9nAE4f_5SE`~c2TzG~QCG0dMe$WMaK{^N?y z{siBeJo0{eS{#bdZ`@ZyI(m=Xn;ex++D9Iyhv_MM-%f4hgN%FJac#V$e~Xqt_9_3= zSp3IHkLScR`K^0cMZRGy{>K3v$9*7Zy#{Urb=hrTEVCrL0E}iJ5!DuWoUW14%(A(! zzOmifnwcWw!atn(XmRhex1hb1V0){<_L`!-cX)f?H+L8O<{;%f^YDvkf42GrKi5k` zp5u27`_$#7+WH(j0Sq4lYKq@;ONNV=t0mq#J2kW~;H`&_4mA}+a6%OytDXml?99=& zaA@wqBT_Kbx?Av(>N#l9KV7wt@ z>MY5yn_x~ZT-ug|^(67lkY-H2dteE3#$ShHxr=jSOsxb^vDP7C)vHo1CH`eAyAl)ebcQBG_` LlFgu9?nwS01~!j+5ALv#eU zOut8S@{dzzSw^G(yUe}*mEz}P3#g9SW~%e{JDEBujobWgr}E{7%(X_8_Xqbv!TFQ8 z@x?vVU3DL;r0%s?%{hgdq7!&fr6!d!oy?D~y9pyMH=^N*Ca^y4#hZ7Gpz5<1;rNm%e8=xp z7&|r;g5N9f>*zuaHfy20_FF-^kvq(Z_u*}-GCW?-ivJ0{fng`x1rxrH;NL3C!DF#5 zic7R$>xQo+xc?J6#~;H}mR0b3+fGm!WUF9ySxcKo_r67CYtzy}gm+=w4QKgARr z`PzlMtHoo|fHn<2(1}0e{dl#>SpIOF0$u)aIJ*8@gKL8qL1M;!;(MY9<_K2c#X}EZ zVM!I#dCjLm)w)#Asu%MAQ>6FElW|PKy6kno0+YJ-O$@F_=|n&PS-)@~{6U@$SLt^w;SE+@HH2lJD1on#eFr z5ROE@+8fOAv;+0eyC+PN^1!QWK9a#aFVqJN?-ed-tt>-x{&%k#eQ_32YI z+Abh^;TOTna2~sI-G>`>nbE3TUG(b^=jHFM>7`NK&Tkj$bC0n}WPRT$h`%*}Cd*vt zP7^D>U$p_fpUdz&f9&b^5C+?3DDr;HfDI>&aO_4mzG>YVxcF}}ZxAa0ozH%B@p5Gv zKh**bg(`8(Q{;is7a=zG7`QFIPhyuYq2uQ4#k=P_@rhSFko1di;D#lCorTcNh2-Y$ zFNjOExX3qWtX%}qcR`){t-TB`2S-uW(P}(S`wV{6m7@FhyhM{rzfpvIhVi3g(D;cK zZogy7fA(0>67h{d&R63I`#kX7;sLEnY3#V(B(y?bT9>z!TC`A@5mJgildE8PavnA} zjpnr_xZ*m1xJLLFmnO-s}CXl|(%0#gV zqo}z4Sn|TpfmVLZUcF$mg;yRvjn}fXgsKmop!*0v zp53Ma5B@o${b~#P@#q~vxRQsFkWK+8;8dTJG1NIcyUB{J91s6Fm8zD;!!Xy{OB=Zz#w>NZXi+7w#lN&xAH- zt6%5I*+a4<#kv$K0;-67mJY3UHRbOt?dWmYF5Gyl73at|v4vVHV6f@JkbX3h_KD9> z5aEM8JEn2F;$0AVeJtO)!j8M>`SH=uvrtam8a;M6qoMQ&dfd4e3+AZPYZ8{+b-4@; z^^xX^C++B+?bhJiG!=5{j^cBtL9qIxNBe3N>GczO)HZtr?YR_0?=G)^H-@QD_o_s& z+h#H`UU3`GFG>YP#Vo+e1Ei|ioKwwgXU(7v(&yQQ6CMZhyptX@<&7*)oqR;ldCZ6h zeHq84B~-c9$lLhG<`SeWY!()ppCaE~-U~i$Q{+dd#o>Cl36xBkNW)$`Vn*`_Hgbg* zfBb2guvT1-^8ig&+j0U^FPflEUjiGqtORrFLvd;SW1KwNfq>3xemC<2lscQk$y=wv zOm9CrWM*UO>li3dIRU?!fLq&!35!xDqj7I5NT%%tx%3)Pl+%Yzy1z+h?N1g`dlU0N zD8Qm;?hsIt4C1k^VDd_dY_`n6P!TKKCEbfk6P`0Q_p>-9xDcmJwWEGrMfjjX0JjS% z*7VmAuc&N6>Job{6{1C_NDg4$qGF5)N2rW;!Og+R_*GensohORwA|2I77jPu$X)hc8TT zLyOT*p|vRGZal%;*f1{8k4syYijpY>7cLZ!gH0dR(wfCKw}4>p=|LFrt2` zpy+ES9Gx|sbUBR1PZ3W1R^f3$XFH0q^!>y=m%MHQfy(b~K`3(C1j>Pr7 zcHC|4RZ``mLlVBa;;zy{9Pw}}pL0$W)+_1Jtg#ZHk(@}PUwp&c^Ueyt8WuV4DJ{Wm zN=x9|m^H%H(Vpmkz8^l!p25G4OMz`!XUIJ77BDkw0p;@+cz+j#6T=E&rQ2LMHeM1w zukK<-A4~9*fjC;lW*`PLcyiE-{_!{>)KnXQib_%FB`evpI2mYMH4J7QOv4aK1DYl! z&$k8r6GRn!MTz61u%|bJ2(P=buXzixY~@8Pxw;;tx0nEYScKJ^C*W~sh z_!yd+u9F@|V~7}di_H(^p&_spPo_kIkMu{jQ$CswEUADJIZ-xi8(`V)By@_E$EG|^ zeiX%%u({`1q-i!RJ8A-UR}$gdl4fSNuN&{r$;YE&4LIwhKKuyKZ9DL1siaLr6ws2&BA##^DgmAmDqZ?W?jQzLlNBM(p8^ti>;$+YW*HYP4D!(A^Wxr5tE z6#qIO-r8BYR z&^$Cxfwvv=V2VgFIlb>0Cf{$vV(mEGnkNNU{~W=xfmTv}t%`KV2QmHG51GG&3QbyS zPd&fJK%;d9{`-1-=vySRNhe2R`N+FyKhupL(-*+^hW5(HLPx&pNeKp*Gz&jYJ40q& ziRA_LZP2?K*V+nSy?u zFG+111*@PxV7_Jqc8ePEk4jTv=Gw)OR%M9Pp!sL`Qo%YFb29CM#7GtO4Ks27RHLcX4#fAVE>S3J(A*u z)AbDK(P0mXXy#&?-Mog;qffx2sug4`r$d4#fv*c?ux~*pEU{RGmNW08lHp9YGg)Di-9=u0k2Z^%$WR4Qd~>`R4|TI~Nw9%Zc%rA@P=ci->Wq`Yc6j$Cu-=!Q;Z^ zJAT04o9D>l3#UQj(p&gkd6L{~Jd2J;UxB~J8)0*<8-%;|fXWId{O{ikSS6pyb`1HM zL-q|g_|Sp6S!{!n3K?p$egW~gX91OWx&#$zXK>U7D^xu73jKz+!|t>-EYL3<#_0fm zea{k2bio zo!$mpN7iFD&4GBEQ;>cCfS`P)g7B3<09ttm@fI_ppVqD9KUMyKO@TJ;&lzBj{#oR& z^ikm^sUj8;VgxZKL{Y*Z1an%h$J_H_%XAN=5a`*I)1CwIW! zlhMp{wlSEek79!s;h;&>X}W&2VEc@TT%a)uxbH`S@zfV2`JOx5Gf*wO8Y~6(Rq9bK zxQ4AHu9e^HCi2vjYos?b4Hd5G@a{1M2d3yu%J|x$J@r>T@PuDIC~r7a(>~x<#t%Qyn^jYKPh2 z{p7>LM7UqQ62&Ah6Qjvmyh~$mWy;(SuyMjEtm?f6%dW4aJGxWZmZ|RaMq&XWPaM&; zUIu)2SAnHZ66D3p^7p&-X{VbDI$xIHTe5C~Pw-_duX-x1DL28r&o{%;8Yh}FEQ1Ur z8$-Qe9A@dI;G5`hGWLf&owF?oBb;L^V-mJtp7ADh=`_GbiGi#6?{nBDmqB!HiHB!f zZNcuuG5Fy$6JwhjS**7bJbTwJup;SfQ%4&_Ec}Ga1FeOPdyYVw=0udVQb5)82XOS^ zF_)AjK5*<9R`7LQQ| z43Cdjq?)r2UR4#>0f!YqIKHAuxb}!X9GgF%57viJn+twuzbzS8xZD$jWUav34fE*e+cyW`F=j)@N+X>X!_i@geV;DEAnwW-k-kK6-=7nib@w;u*XVSI91(9Y85t zSC}{2gfBX60>wvvlUScB2-N%oB3re1@1|_nwf7&f*mW0XSSP{>?-1H4Iu{FE40xd0 zQ(~4L4*kX($+g-)FfAbzXBA`t8}^Z8IVM9Ssb-dCo?vq-5A%Jl;FdAb=(j5r+scRY zgKjo(+(HpPP2D9twc<1JvM+!O8)INx%`Nb?R79QI|QY+jkK4t(=V5^ znT2jc z&}y6miNzVz7l|lI<4_;+7ncX+= zr;AQlqWbSLJiA$pZTojfcqQaJ#%Jc@ZMB1Vu(+5Nzp4iFO%w3xW=-lkV3!X!Z2TWA6!Q)AeE_C zULXp3BOx$Yf{xK14nj#Iy6Z;)xDKv_!1Q`t_O6fp+k6ks?!1l3)zeVZt`XiO>4Eg7 zi|}{4BDXs9kmN_lLx-)UaN_Q09NB2em+P1d#HQJSZ(soF3wnlOQ*|M^K!HCi&SPyB zPb=H>|H1L?zuAM?*{tW#GTbm(0$i_Vu>ET8IO+0Q@<_9oO?_m_zua)c@aHY~IJ5~> zFc)wBTY=i0E}&^4!)s(Rpkwa`P@i!EY=Tpz;Esitah~mblnd8~Rl6AaZrBZMj})_8lfhC`b?8^0 zWPHAIwm@PiYclNcp;?I=!BO1}-CuTM=cFyn>!N_XTXz)~n?z9cAD`j=VoI|>eVZNp*^!OSCf+1Mi(vf> zQ+Ak_vG&vl!a0_oNX|4fCTnbkt2dhBsio1(sQnFGdNz{{8(q(i`fG8SkzG)$SC6xY zKVd7bq~ShUDF}rTaAVypm|V1zrUdCh>ZZN8^6@J+%BGr3PpBugHSGeElw9y#pM+aJ z^?_B$WZ=C&Q6ZpN*rL}AMaOHg+AvedLNqb(!7v;-t`P4R=<^L%5*YU6lG8TFG33b! z+%4x%(2`?O$RRw`+J?@@WKntfZ}Q|+Xr;`>I(MQtb;fMCFstQVZ}=X0%-+%Tr!IwUbu$C ztDeA>BMLaC;j_~o;S<5;8`&@=@dDfb%#b;OBwDA1z{pjX@!&EwOzmCB1O4WZ0rN27 z-+?K(`BD`7&p&}g9$myfYmI`9*J{ySYYBwYS&$dE9#7RM!nmOztaqm&-5427EMjM4 zfZcK&@7e`kd(7xA@e6DmHGyrd-gsVVbX+7HHLV(<_WHQ z|4}&qPlG@+bsuz%+=r)q$I&*UF8F%a4Ccs7@{k>`D?{tIU=}$fd|6kF4o0T*S;STL z>4OgX|2c%e;`d|K*7I;lFW&BL{)5=?3rUw@mv*zwm|B2e?p5slA&XaqoPKPSUB6_-hRJT)PN# z_szw3D>SIaxMWiNK8^|omXO7#5>ef1H>1YdSW~%ncrZl z-_>~DEFYefUB%1;gYkgRN@14g3*tMc0C##D;t{h%kbl}h&W@YLd_J53muG<;+{$YoP+)yyxPc zXdPNvGl|uH+Qd?Ri}2;U&kGZh(goord&gqOzLgkKAelk#q3a%9Xouxm)c@uo5O{Bk~+i@8Et z89`agCG^qLHgafQG=9+6Mf;}*n1_xTjX9G{>ZY1;CcB;_#dSllkq-CK*vr}{44u`a zesl=$g;)KLaA3kn@YT*nz4LeAoU$%euK0)IDU*19W&}8|aYV^4I|R|aHz2vjo!pdq zfI08q2|V_mV((__vjOd9L0ID|@Z7W$_72VSt0@V%B3uL09E$NrKp*BWav&0R8gQ+7L;AcY0n^1Lpyo+G z{0>TmLsM@OR@f<&39Q2Lt3-I!!P5}*Vya-@_Cj(Y@fduq{SV)T=*T=s>O(>I z?qnDU{DlVk>tX5(6PCYg6gNI|4K{^56ZBp6#g5?N)V3m=&AHQpYi#rIblwd9OJog; z{AW%VM@d2B?ikp3T@F_Vd!mbOI?4?a&?=NC15dM@qwoG?)ds_O-j`iu^tN>-JsxJ~CKNf453`5-?wqVPMwAdE(~apk&nkG`Rt}R~#zDM{P_U}h ziQj061$ucUR9<*Nyss_6u7tCoFJ;f325Es|SUc0To-w3fqP(~FDooFAVG8C_e1yL| z?$j$A^0mo0GjKoZcGtlVx4jUtsEMUKxCrOQxWKF{1GuM3jNWlxheUTjj!|~uZqbuL zWvv1j2Za#Zs8)!$D?#75L}Nwa74#Xmlzp7XaJk8QwoE39w0vrU!dd0GW!haZaGlD} z$xvRC+{ThbXX2NsQZz(;KEB+wA2Py+@v^U7=ysuq*-IR74)>ged6qjs*X0r`UNZu8 z|GHysbs#o{6=BK0>zIEc0P|P+gP4Z6@Wz4ReE8#EIHGkLPT3?uO`j(TpGg6AXzf9> z!x=a&UY`uQp2d~tq|kMPISnarMAP4+xtCu*d@C`>a}keNTjybZQ+*Mnj5Gxe|0ptf zy##$K7sVp(x1vRBB;0wB4^_J*>1d1j=(?@~R5jDc59djE$(dJ9cI+VC^K(crX~)f_ z$8ng|3gWilJyb7n0Ufx=EQa>g%zDK7+5kKxwF!lrexTR=By6i=OtM3ckNRx@om0-? zealZ^I4+ctmlCvJ!vvnUHVKm#q_ES*UeM!}2aUqv>=lk<6}^LG1#5vmktHx z*&{H8Y=aNqW%;=FGce3dtvDL0nCpZuLHeQ0H8P{;V)>!s3 z&ICQ=Re9+Q3E{Mv`{2P8MR>1Ti0=&wVbzZD^zs`EJnC?QOwxXY-oCcZPu?w~>&4=+ zO0rKlGSdMpZ1$n;&7+lXTa>8QZCem2u4S_lRj5VpVeq^;hJ9F>3)va-xY!R7{`dKH z>{j22kK+zQ-!5mlyaB9}Fe zLxsg5!Pp-i@Z?YvahfRuJO1r~TDlkQ1wHUo#sCuZhWd^N+3;5{PhfvWnsn9#;jeuq zIBmWro#&qG>?GY^x_SvE4fcLuH_`Q@iTi=`y3^L;@KFZBDlI^AKqV`36tl1 zgt1s5G*w}sCV+>7Tp)Zn-RWiG$g9%Rx>q2p6Dbh_sZbzxapo=k<4j$9^FJalPsQXL*# z`5brdG66Ts^vd$3uh`suSD9z84FugQz~R#_vmbtEVUWpD^{Qc5LcfFT?bptizn%fI z`UE?);RbBG>_iI-Y-sq1a*U~$q3x{^xN`Sj+>=)#G`l*MN(?6BLpeM6PdR}!F$)|s z{61zD1=E`52za(O0wQG*zXX3H6LOSkK!mkWB0!OM=c#eE(WdaNbSKVx^qHk4Ri`mj}(lqf{dsou*&u{lQkIwAWA7LZgN6FIJMBGES)MDoG}8(ZnhLcH)>* z*D+{q3;e72icCKv>gt;Cju5S6TJgb%6dHj!sNaT>{ z1#>|tW{iz(TXDs0ReB`17_*-JzdKOIyS6t_#byL7$@v8fURc0jY@N_MVHkaKUzv~8 zHK21eQ(@wDvYks?+$9y44{4XbKEBpMicSfJ{^ffwEeA zt~O^FuD_#B!-C4;XW=|J+3^!zy4?e9mqL&hjw1FIKgq}pH}2hUNG{19LiM6;SZdQP z7;g2M-TQHl>1vB|6{}fTUDgE;H0+#pV+C_0K`XYWg`>_>aS!*zf6+*7_ zJ~HoHy-@1sWTZ3H_=O>sBHm>Q8oi2vItLE%ZfRtAm^ClOR9N)g9;@zKLgzU-{%ZCn z5?6i(ekfkYXZkI8=MljZovtMF+##q*7{Hm12LxtXQ|T&P&W=tQ!R|f}!D~LD5ZK^@ z(Ho@bqwY7t_Fcu;ievDZ{4P|@a_8GOcncRi*o!YSJ6V}rC=CmW1|ed|*MQ*ZnS4taB-v@Z_gZU;KIKAp|~c^lH+%0?`_s0}VTdUX7_U6{RB6qouX zlWe;Ul|HLQsJq`vFbTVX^FK(_y2eJ5SW-z=rqz%GlTzTuQ)L^SLe@rFGn5;Q0 zf=x>r;8SuG`ES)G+#FUY@Uv`$NJB+VOm|lknY?fx9krJ z!$>+>v)wTHCgS(O2~2#`S!}o)#fEjyfD`T$KyIxp&rFe{84h!()TNC-fNlXX)}(tW=hRnK51@FkKS7w~3+p%*{|aGXj6_+>IAI z?fH`V8vLoBg`m`Y1d;O{Al=6v3qsLIQ6s5`teUROrH1<4^?gTJ zg{v=wu6m5m+U+1a_O>u{+j5*XY8YHyJ%*Ajb)x;lmS*d#;kN0vY}&$noN;&+Xt$ro z{n}Be7IY43k4nSt5#h)ltN|naOK?9nl!(8Uf-U2V(bYH+f~J`8oeLgg_?t&8LZg=j zxBVc8td2t3?yETJrAs1D>|AXO&wo*6C`DDT7kU3dn{HMA zjJiYF#5#$;cpx$k1AkwKd$RRV{l1@s8}Gtx6U(9e(_9oeIGRV<)WaON*=&3KEGShN zMYoLF#5}xDV%RV#{#!YiTg+>N2UlM}SzDgaqP?G-I6ImdEWOGkM7NL$yK?cCdly8{ zdBSX;y#Pf#3kMqJKxOq^?8;lj`fdlI_O8Ep?ByomB}D}qyYw|KGmFNR&#G{hs0Y2J z;fyug?jbuo#EPuliSG{TQybf2qPul6XuO;~l<8a>$^yJ-W5oiRpsJ0M3rw(Xc?`>( z-U{}EvzJP7JA}>>=3;7+Y+<2-=Xq>10-Pf63;P?wA*9qw zAiwS>3bQk~=$9zGspd|njc$fhU)$iQxf5S?cmrfU3?+YCdfD|?56Hpi#_*%d7<#X% zLG+Fz%%jH$)1y?m&yN@=`+5c);>`GyW7$IAM~fiBPl7)a{Q+pboD82iiV74`AYoM` zxcXdz2IW*xN;M{Y$2Ppc%SrHd5A=vEXLo~63eUc)g$bYH*wrm@STNK}zg80i=f?_k z^7bB(tec0%V^+ebxe_$__9ZA?y#>occH@6<>x7$w@8FhzL#Tzj*yx{|p|mp(WM}>) z7sx+!3{wGVK^*=|v!T%y#e!=NZ?NptB24|So`t=-0ypF)&;w<6Ah{s>Hcf*u<9EK zQ*+!+>_e^4Y&IH%4-+){09RarwzrbX71=Kx?C$X2-2wlF|LH}h*Uc0jZ zI_HOwv-x4LZRQPdU6qOkN+R%aeiQDzB?0i@B)ZJd5JKEp|oG zeRk<2c=~L3JNSaB?OF&nhV8gu`g52+jJlig_^>K{}} zKx2gD5NGJfM=z4Y@TE<{eS5U=gUUTYe_Hc79mG>%8BCJD1+BNOXpTl5 z>AhVE#Qy`i`o{$tG>YNY0!dmHAVuYOf5XylQ&N|?TWz9|!yYpYr zw|@d(uj+_itJ*NO(+Gw~t!EYwx=8;^H$MK~2qEkP%mp?VXYkX!aPx z%9r4>{a5kbAA6K^QRCO{>>wJ!t?a$#b}YMK1|iFwh)aW_~O1)>v)fOqCSo?DH@d*~pOP&sQ zu1L{M6Kq(sV>O!?;6tYW*bf~lJ=kr>*!cMunf1H|tezPG#hR}L;puPD^xAJ`b^8>! zo{1$+yC1`ezJq9d%#qr(1;B$jpV-KcPXxCf`tsQ#FVX4dOm60L8ryoO@R*Q7Hf)^* zw~O(|Ah#M;0R*`)1Ye3&V_2;tCR%Gy?eOzNx=Rria&O?L={r!{Ef6j?7C@xcNu1<= zjWqu&Kut+WYL~x?b{ucR25TicO7Si zjkx8|Ei_AgOww*f^B~FVf&*f=$iFdL;j!;a{F52L&-6aP03}VhUt)`ID}+qv?OWFF zSq;_acHlToXJ&KiHq-bNLf$^jB_Z>-z^o(Duyw+6u(J?YLqyoEq%KMb@Vyh*zb$>a#x7z*cG(fdO7r0 z*bD7DBgv9DMIL)bAE$p;rn$SL$*PPh^6ckADt)&X)-93c=YIC${FVXYbJ`lFDn#Jf zp?56#Cp5?`=-w>=CDXDE!lk0F6k2rsS!H~s;dy%S)=o5#%b(gEz%@MR}9%cH?Q zYi{m&8yn~r43!^`cc*;>i@afct?(PHaea=5CU=lQsK!^h7jf0DLqj}sD><-f4Ww;q zggFTgaQxv^D7^3sN1_z%kqrl#(HhL-Sw3zy2*tquUl=uZ5oC^fjn?<;u)Z@6)P2*~ z@K_;yp4toJR%r8@ce8QeODry$k%m3t*O-&j0NyWpMMi3##y|Pqba&Eo!M|27GMSA8 zuhTE!+wVj?Dl(g@f6HeIVF4KAc?kX;*oSYs2U+2;2>Nc{Q&6@%$_Dq@VVTZ(;fTTW z=;>a~7T;{$`)&b@p%esl*YR z#*W74KShMuc3%8No5WB*YzlHs$5_g>qmUvO2yaD;F)}I^{<wiQlRs#Bp zx#*pF9HYO-33YN#VpHp3vNk~p&emmv&XG#;bEgHYI@t_Z{17h}ok8;Wz z1<7OeAwB&~kEgw1!3VN25$=%*?F!Os2V*taShd}rPi_7D3*qHXFR zDRoHO3zFc8!`;f%F$*ELX9bE%EAv@bj>CctnK;`&5NZc&NLg72QT%I$wwaUBJvE=m zPgw^u=eFVB2^P4v$sFg|r$YA@Lz<^ljmjzqNXT+u(0zWIy%)Z~9q&fd*(M$MhaE)m zqW8kRm%7ktQzFrrox?9bv1FM)?&7f_=6BY(Ntjz%4rh-WvyFM%f&Ujfl(D80<5p9$ zAlwUl@*Rcp&ySEttKzXT%MnWa2)tkT9gW&Y!Hg;Hw7q&Ou9VC$Ce;g#Uv%K9ra?xm7K26q3Apjm3D-w2pxahm#SJ?ZAeCrvZ{HunK*2^V8q~rd zF*O+eGz#>6o3ZMy4(P~sNonEy;F1V>KPDtuK+M zj?wV?L@DT1$xu8xR(71Hva%43G!lmmZKVsco~Bu7OTRE5x22Zrxv!X%Rx7t z?HJ}2O;pb|V@GEi7+k#~ocr)O{L~-Cim50K9TKYO1BY48sSEhZ&5~A^r-5GL7f{+W zhseF01qWi~=<4vJxX9)g#{8WIi}zNMy39ndIwirzIgMd&W41v^Nid8H8^JWb=d;^o z?y&#AvqIuCg{JdJ2zAwi84309;Lq{Od#w}k+MFF+MAw#n?v%mlM33)IYQwQ>&0$q< zFzG1O#(8TVGwtq1l+#&*`zj|2MjSp0Q!STa(es;V<8up*iXzZG(i^Y6*o4)pL9k+v z1qSI>vrRsd+&v&0W|2~6y73k~$(;tj&pI%5^RxO)MByXHscD1*!GW=aokHd#}*i{lkJ=>~& zW>vS8N!mpV&&uCp)+!&Fzwk9)elCW$L#l}N$}4bq_aFwC?88~bDrDn{cMv}L7KTb{ z&~iyx2nx~Wu6jEK z8XNKD^Bi*LP7~3;8$um6G5q&OgnhlT9zI{%4Qclpp+T=o_&}@~sMH@;;8O#)X8mKo z@{T}|+$$)K^nf@o1Nxu!R5s7J5m|mDIM1#_F)-lMro2X#_1V~zsRE0JX13aD4O~7Z z0TNE$X90yTNbL!K-Wgbfi@R5niWR0@m}`pGOM_8%p`G)^BPzlTk<-yL!4$)e$k6+h z7PQ!JXs%Vgfu#cn`0!0)_%ukCS7-clQlA;ijIQ;tr*RbTS}h~f8h4=bb{Dw3CKnUF zx4?wzVVI$4%h&8qgg*09s9s4i;fMxKoY=*3oo7Sw$%SzG>`y^~-D&*VC`MQMufnPb zNjRmEM80=K;;mn4zqu$p#V>3@~9#Eeu~#f@hXCV&k+c&|SI@HqD;n zbVyzw`qIYGP`)4K&-s#^^P531&jA(h#V~d6tyr;KlTk5b_8YG` zcwUqST}wOI)z|>DMfZY-jx}yxe22(eiea)+KHD?819y$lg~QqxU{vR6mi zm)3-X+r-J3wj!A+nQQ``X%+Y~Gy&JkO$1xF0NVdmz;cqe!sTmHbfvQ#9quxRCQlCK z(mm$9zoHAKNH~z=6<#Rnc>wM%zAqRNrApH*GT_3ICdiquLI>U-f{b~L$c#0>{s-4U zqF5RhEOH{DL*91Ng9UJGpaN8Mu0z@5=@1reNgIF7h3ZP~=mH`~Vf5!z+*0arO{;xMP_LwkPixx1Hu#WM^SHDQjk z_Yh0p8rZ(57?!XHc(P+E4|rw98>7|_`LZu8-R2Nmn@3oqULM)I-WMKc=fpm!nO&ns>jh1Wh+-1s0ZY_YZR(D{{P;TAg>;_52gw9pIfCE`u z1-NA_J$ytT&B8~rGh0f8yUSwXwcH=_(6&l2Zi*(4EGmYk3ni>isS)dMM$@tN7jT&F zF`{pKlsFId-Rrif)1zs_>8*+w$Q^S6L#ZsDygkG*FLK3iej}O9sBbtvWB|na5>aUC zLOd=uuq6#=;Aw3>j1)PDlbUX^aw0~RmJ~yt*IitDp&CplJ|n-rOyobyj-cxIpLl6$ zCCgMBL1yHBAn7AZ@%zh1U~9CN>U+!`A{?Xzg&*2s>9b>SLV7#QS+p3YnmOaxtJmSJ zaU7P+{|1*VQikSF2AlBeH90xq0Hl@t!Ci%gU^TTL-p@OV-o}-v+#AfUuem^ylXLNd zXaQbZxEb#}F$Cq{cW~oEGK`It!LA1xn5(^vdfs(Lr(<96!@oSD?HUb9FUOK-g*|Bh z^&D=myM-mK4?)Sri0Yoy!e?pXj2@WEQrlPZo~@tI?@=snt?-0bt{LE_AB&O>l61ay zCOS1Qqt7~y!NR>i$n)XbaIbEIaMQ(jY>kVDl8v`n<-+^8u=z5Ujh{gb#2=GCMe@}A z)ji>ND;H>Kk7eI7J`jsh({cBp5z%<~mRWce!(r)EoDwTbw)PYsb zwxUvaK>8`JTor)>)?PFzAy#+=;%R8^QHfZH%d1glWPT!qJj9*d+g5&=B_tm9=(~ zKQ~XoxnfOPYTpo1skZZvg0}7eJ_J6FG94gZ=ntOyj^TnBH}o@weAVuw4L- z+WZ9Dcb$N-2h>SfMFi}tvElVSg=E%^TolbyBRhj^fZ9!?V$$Mtdh`R(aT4Y8gEH}M z^=PO{?1HCqx-hnX52{;w;z(ODI2nSRaY>**^E&tC9KO# zi?$34Ld*2!Xl<~6D7S5ZI@Q0#w5}gNRfG-Q8^7bM)?-4d=la81H&0Btl#Xgf=V4Zb zGk9g)!nIyMNt;i(P^-olOVsqZ;ssMact#0#lnvc?5>JM(m$9tRVLUzI>p-_@NTa^V zMs$iQBZK#SY5YAqzM?r5S6YtdaI%yv3<)Df!xnd~8~8o^?=^!J2Qr0y&NldM*m<~J^_PkG6p*9N2ZiDZ>U_J|5Rdvd z4E>9QV6bo|i@CFt&k!Gr*GG%u)ODe-{Q6PQFOvt6dS#SdDoKa!L&&(FgZILq~|9i8<5ObF;3X;fP7QNt2>&h`xOLZw*o)bGAmjc(XI#ZyUSa z!3hfrhu;bE2WlkIV+oR*t0_rJ{)qVe#Tv;k+xhKNw9+T-+A~Svw_?G-s&@X!3}?al z#KK9N+9dIHFP5~Wt+bWX4if};5`s*%5V0!!BmSghA=nYhm)KNr+TSYb3#2CTi$N#8 z!IS3!IHZ zB^TH~1=Fks#Lq?N1>4jpfq}esyYJUd@$7CO5&2GX;b-KEJu0UOYP$RR0qxU8_p;c6 z12dE))TpE6&dg51mL08<*+aRKioSYrjl)jKvKz0(^rwr0hkIjrgNCgV{{B``L|TKW zac_&nGa+$DB4@e;514dM*(4?$N@ zzG%pGiEY^+TT+y6D>3sg<(r28o%D4J1PzK)Y%gy~;72YzAg+3_c5)vz@r!<(kjyzh zAifxrA&|+q$uHNfpVZ-4(zPc+ylr8qV3ENI!SQb8c8mR z#rN3f1haD?-D-t>QJ zwx1_!bvDh@5+pV<+J8izgilCKT3+Bb`DOy(x1+x~8l6Dw#{ z^6gt@wSUU3l049gmpFJ-3dB^L;OV<#l0KTtUlXj;ej#XWd*x)lY@n5G`)ZdA|Alg) zXz*K|_>J5F@%;~zSofm>K2Kdm(%d|mzmKVta7&fiNp6M2tJ+2MdZb&>8sOP(H}i1Z?$~!zNTk3POI} z;>Sy`6^QdDu^2~<_FpO&1R?%elGrqL$)ab9?VpCO3uK7bj2#US-k)5=!^IG>4|Qt z8jH&}@M*bOru-<%NUS1bBsP6#D0VPY6Yo(q^^au0lq;`2eJ^fTKPQKX; z%HmwsgS> zGj5TOuj}%I^b;<4v%(A_m=$ok0fFV{*$IvDm}&7tf+w zk>7MhSzJZui|oel@Dv>_L{Dgc{>WpdND0b{&99a6RDTWf6xYq*Q$D*zvkoVV9)>6J zcdT|2mpgnBok>0*N;{cB$E0i)ou0CmpZwC9&ms=;>)P$bH)6^}r8g#cpH?XFwdN<2 zyT=U0URRCyOBV;B@L%Ed+e2PJEzy%)1^3W#CLam4PdjP#;$-+z-ken1s!Rh7JY;G#{CzRrTC8ODdspDi`_}Z3zVh)Z=`vEn z^OLM1%i$JL{F4k`wT>S+VqncT|Kd%r*1Ca**hZrFj19ae|K!9b9rogVGi`XuwkPS+ zYCgOJ!aC92?j@q&hI^u5jk)5;A|rmmhNHCqlOfTAylmbFaXMZR$`u7{99C+4QoBd^2gyun;LeeHU3my0i1<6KFW z_ZrZt)jee8VJ-=7+rW`y@^Ft{HTG>bByTWO$dX0_+9 z;hB%>OuIQO_53gv@Q2cSmF45te zaiTrfn&9=(yV$A3od4yz3xAfWs>s*Qo>%m7ov0)`g*U%YgTJN7kf;7+gm*AMmMk{A z3Y70;@|JFVPWGCclb?fEinibU!83fo;m(k3p|zqf!N_&#*rLsc{CQ&$yjk^x{Kl(7 zCQB;FyMe~!In6sXBkUhZRL*AVub4q859v>(Xg_uhTJi8Ob zdlVSrfZvz+Z@&fbb57QZ6{G(0V$45_#gUHuRT}pEq1s{b!^S;gJr7#^mHmZBRWSG! zhm<9eqBMS5vl9Q({{4K_Z{|D&xk*mUAGRbkSWVI;vX{8Ln<9AsRENL4sYiU(q*u)A z-zgG-GX8z5$Ksobc`&qXj!5;h5bw*0#rIY3kjUE@l(|jkz58Q;R*wkDgyMMkeZ_py z!7cyD$FhaIhQWK}zu#+M_ij(ZzQ~OpKHo%to4ua5(MK1?w-sTH8`J2Q8?EWbmD#vB zr3cQg*hbGeID=Hm?!fi$T<92fwD5Y77B5mri;}M&V_1$)21l2X&9#x+OwRl~hT&U% z>vEwb;~ikzY#T{o&q)7brK)h6wYTCCYwnek3^U$TmZW<&(?n&JO~sBytg^PhET?D3 zZH5{d)(p94Rv$HLncYQw%%s)&O!s0bmfMxxj888ut&ia-=61u!j6+R9R?4f+F&<}Y zvzC8|wSKHv%iL^X%1oZ$X7kBmiq&uLF&ka)+sxHBv#g&7cQE+}YMI*_nasyYwk)Z# zm(~{oPcd76L8jFyO{Tr79V^*?BWo-#%jQu&wrMH4z!(Z+FzSFxpy?Je_wom>8pyLNCBciPr^PTgtD z&VDz-M9*^Ag+~&&)QKJ3hqn;-d8-Ha??x@o!M*Jq8E4jW*XX<$l*$#p<15>Uuq~(YWNu{GtAk(Y9XN=Cwv0t8@X?>6+xW5yQGedVKG1?v)vGY3;*$YGOaHNtpa-X{J+17idSx+y= zv%ihAIWGM>*qmRZXXb@ua43ffc9QpyO?}l8_9>To#?rHDHYUM~ zSuphhW53ZUW+|`A#)>ipDA?Z<92uO*NBg!^>lCSz*dD z7*=Oq9*$)_Si6K-IAeuP`oD+P!&U~29f^9(zWd)85>tD&WW0^}IRdb|swNiH6`f>R z3@>J?&D(CZu}8wlcRp*i^LH~tv-&AVtf$PBb$Gxom}y5St}SJ1p3>*qr%z!oH(kKt zGtaR)bbfP{2TeE&KQyvy&&6|FaV6_0p}as<4A&#~IU8 z^|^m(>&=9B8kl#@n^+3cPK?!9iCxq_&Q>t0Vs&j)W#4}t%yK$*ih1d> zAM+h=D(6a27CS_t-DXX!A5&LFo%KVunzg@fI(v$2KKuH<6APaOE3=e+OqiZr4(CJW zK`U3~OqSh-sjP(-fsEUB)|@?S7qL&Td}TBBpB4Mdm=iO-lgS~EKDE-3mg6op&|yQ> zLmc+?Ib5)vVtaD0vIpw|7^%to7|Hg=+<6Yc+@`5g?3rJevio<&u%9=)WFMA%;{-eE zvR%DSay~@T%zlP~jSPIpKEO_81>7@Waubr-p27ysvBo7<^x~7u8*mq+cR-pU^(Bqj zk>zdkZq_l@rz=L*W9kMhZi5&2NFInO1NcGHbky7#ZgzNB_8zpaJgE9lZb8mr5IDLX|rQ0_c79AI&9*6_p;c(6Rg~>{9ye_ zH?msTL9i0MhHW0Lw_sn*Xl1!8$a5->9%a!ll5OU(R$J|5oMwho9M*?FlX`lyhs`a0 zIYwGwKVxy=X6CI>4My_jc*gR;i%jX_35G&Y5##5LeaxDJi>)ixOR>i|t2mc>9f*UH z9q{%?HVS&a9ew?LhH6@#PCeas1G#!Wr#!#dVDY+(RIrl)>MREBr9X+A-DkOPv_^@0?%ANd!-k58h(MlhtpJp* z1%}I{Vaw&S-2DOymF`SHomZEE3T-ny?YlgF>M^}7+}8(9`4a#%wa%lk=}vfrT!V|t z?xH2BO!&DfmTPv&9A{-7LyUnj;j#P0#FEt8sP*cV>ebWd>gQjW(TU`xmli= z(D1@{ov+~NYp2MO3IX{wO`4R-aH3yp&c!i7pW)0y-H?&p2XbHj!4`9i$WDe0RqGN9 zls|eCO4aeGw%C~1*LIbo%(4W1Z-*!|BPKFlT}WJH8G$i_wQWVBJn*c)4w=PWCe~KO zaW(ZcQPA~9!rtA1Njy|V--dp06^$Qq4ZE%)t;|L=1GIB1zYS4~KEx2ZH;$l_uga0o zxk_l;6AIQj6ru{R0BW`LVY^iCqey^;u0Th zbm0Es6y#EwN$9QpLiv!(K={QTV!E3lp8MrE$e3MBoL4ZVguCJp%Qzjlir0d%MM4zu z_%rBI^hVP~Gthy69QbAbL+;d_v#?d!6%cuQi|}-A0bUoHj*{ps^iJU2Q5vescgc8*a1{Rs1o^$s(cV_MfhZZ>l#kEmzAWjbXJ>G|_)}DZ? z+CG8}qub!w^;?C9J*E(md8z2htD`t%o-^!x*2s~Xww$t(ZA4w2BJS918`!x(4QF3y zM$AA|BQNIwf+UGNGy)2D>j&M2a; zTN9wj`!^LlDvMuj^@j6TW>e?atOWFzSR!=AQCz%rzVO2F(_l7TN0|&<;l7v(VU*`A zsI%n^*jPJEc@=B~Vc`>COZRbMR*4(-e{BhGna_Yz--g0?^#f>%jXs`#MhUOH@Cf`4 z3dbq|%W$YvD&@RxIl?j&z~)+bXQvC6m?5mk+JsXM-+%!Ink`-*PGA@B*@(GFnHu@d zKxu~~(WewgNQnNz<-|9X&HjMqo)gpCY(uc)#}lN_@};Q#`Y$T+51Y1LWl1kIs3SYt zrAR_5hg|S#CC*8W#bs1HJa90B9{hX|RKGrs--SnE<&7WlIpe!zl*dl8Z=Dt?rL~im z`WT0vrcNQBR(yiRw~|3gI-o6-I!U*l8tz?1Yp8Gfh9>*mz%G2R-ksXGe;i~f&%o=%g~Vdahu@c!qF;xU zvFVK*pl~;n5*=2dp2)`{xabZ_Gn)`j`EnWDj*7;kU$y}EURBJ$X#=7j86ywhyWrc; zG?<}!86CRd&xHjtDC)I5Z1I|f-p{^=7VhweHPI)~bTGM|J>C+oDw)`MwV!Y#?KMy? z3PGu>E8C{@?11-DOd!8l1g?sB$mGo?Q0=J*yA9_Ezh&lOjy?yz*v^C8LN|CIMu_4z z11#SpgSXC*hWqV;@cW-`cs#nEs=qFU8nW(yKcouQi2M&-(Yb=n{M~SY{x_I^NRK-G z%r00BVI_>5Lq1(1Ou&=9ta0G8r z$wUDNy7*XVMN{CU-ePnvVj5m?JOT8EWP{P;G%DIy08FA(F~za~EiV#@_yTt%oS06q zRz@JPu$*d|z72q@KL|Y7fl6iP;-7gf;7`m4LgAYeaH)1gI}UN*`!-VU_oG3{-$bry>n5;t`y-)4YbTg&6Gw2@7omhT#o!Y^1x%}1 z1_r&`ICTl@DE+M4$o&3x6mvr!>@uUNg(1&~FjP)4|B2AX_0iP#nI_2d`x3Ne&vIhK z@)NaZSutvv{X{rtQ3}|ywjT{H&g05`s0NqwTexkHVhAvB(GY4pSYycl8%up;$B7P>!q?#8WC9aqp z1<5-AI=3p36%tmpwQ+Z~CxIEyH=*2{P^jy8gv+%zC&qCdwd|eNBtsGrvq#SnDSau#J4p>? zyDtwp8+~sTBxMrG{Q<E_a;7v8v^By&xFs&6X2|# zNWjrfG<~K$ygT@g%O4*AAB}0sq0x)jD(OWE7x`H9E*JcAv7#7J--s_Rl}O==gc{zz z27J$)hM%(66Q(;ZQ``Mz(W!$I;Bk3Z+pc+^sV&FvQx1z=g@^Wt(Yya9_v0uB{dJh& znlAiGnYNUoNC$atQpYXe6Nb@&r%6b`dKzxH(M&`$vcbEs44{-0hU^|{z|0qNaA@sP zuC(rB;%2W8(%Vu8EKB-;a)K+|eR&j+yEbB%S$b{jv$Wxt@J2%RZnxTQ;~7pp?8J#Og5t#HiSJW3o(SHt0v zYZ0xZjuQLE;o0$6oUp_O&$MeqM7am0@l`-{e>{l{(x!qpcL$+=T@BihRzX#sa|h?H zeF4sbaPI4VA2i;ZhdVw`71ql3pqRwX)ag5`Kw66uN-HYooIa}{j`Q?q)CP-NKy%c;&A)L6H2rD{{`|0_^80g< zS;u~$_0|uotl5h@^DRNW*B7cw{sj2$E`_N#MmVj}4sX3_fKi?ooV%=#TWK>3FL)g_ z+5AP7+OeaQs8ySdQ~p|_huRb5jDJpa_3nPGbkv3nU@4QDnNM)z^c1-D-Vp7v-3)GM zVUWw>m(q5jE9ju#mvG&yTX17=GW^f=5u@=?L<_1y=NTZ!Uo_6Jnhu&ngJAs9n8s|V&2m|h6PXX@tVv*UQF;w_fEL0z8A__dOQ`_tMxr&T2 zFz`?hEnno0u3o!H44jIk_UM+PH$Rq9;pe$%C99RXE|US?op;1WDNm^hzX%kyBZ*+b zM(X)!8+esE54Mr#(c$AyIHlfA)a(|TC=3LVqW#A7xn}|?sChlFAp(=JRCft{;ygSYa?)Z}mPKPpRi(3sS z(LtSTC{)8onmnMw`xE%9!#Fn8`VJon7eI9}mo$46PklJ|E!-U8=pEZT;~gRghGOqyq{FOL!9K-7G>U)+`g=CtA_pi)*NE zVnF!1cQ&_EC`RRa43Cis3Q~k^ATyl_WZz zbcGb#?;u_L6X?JN&h!S8PI9N%fF854;vIh#MDsiE&}%G?kt-{9k-caZuXkbzU1hnF z4m#q(%X3>S8YNooYN2`Ddi}Jty8w3w6=fxBYlKubbR=50MJ1 zl3`wG1HIwdC-P230aY(hp#3byh*cjxfhS}Qa&6hfJ@h^fY}}p*W~jD7)t7Oonp=a5 zmp_9Wp1Q*G7xEx+K7`P&uV~di*@66PYVeiU`7q$RDdm$PjRuoTaRfHDTS%k3jn;4-N;Kqq}S!a6{n* zm_H>4$UpRh6_yUb#dj0lcmEe0ndeOW>WhGSyN{!u>gC|VwN7G znc!Wj7fzE6M*1gjfd}#wj`Xbs))ASg;KmI!@Ax_Bwu>*^o!o)E&w3zWY%d&J5QE=b zw#Un~UZI)!7m0iK)6uRmCbqq3L42#uh8fHN91%id&iouaZ+#+YzPSqi1`A=y6*c^( zq5}SGNQR}-^2GA5d%&%m6X^YOJE$D8114Yo0Vjgvp=*pX?RC(L8xh%CLKWh0VU zld}c5J-Y;sPwzp~r5A|SWgH;)bQ_SYIi-;7?!?{IG)i#D!JpQ@Cpqa)!QuaUah{<* z7Vmlk_1@kBW}9hjYVC!0?7dIsC?2GZjx40_={I1+?nCE3?82MfZ^PC1C*bmhcS+WH zNS8nH!!0+yL5pN-vaE9r2>Yi*ACP^CWGZXhOxhB0{q1@xXUY`F-E|JhUWnr!sJVkw zRgVFepj9xQ(T=`vlf{;HD$vd(75&omg~9H1)L>XSmHTQl$c|cv(s#cG_GLPtdfygw z{j3Jc-fRFJpU1(pKmQ@)&}j7UY8v{ttQ`!_DTctqlW;6o#4CR4!}pQhl;3w#s=&+= zs5pKDbJW&jWuGeUct94SuC4*+&)DM!^(0bGe+#zPUL~@13GDoJ9_k1`Mkzmv19#P} z;6zIa8d@Z3{`7S>xHGJe)?R6&%mPAT4ZH-#o5rXs2drTkvlMOJ{g?P~whJ<5Cli*c z2e9FXY`9~Y8u!yX8c#h@h34y>K-P61z@=anWN>Oep?h{Ku`RP7I+?ve9S6=~XF>!L zyn>dIjb$yoFNFALptZxQHyIRN%z29!}fPD3v@eP5RR$pqE;57G_l}C0&1paKBtXDLi^@ z3s(JWOd9X0N3Dbro@r}FJF0hLCrXu^)8~NHn*8ts%}wO0vQgZV53qBIB{u9X#yJru zaIL=zz4W;OCWci>gFSgj`qxxS7?cCMUM&N3LcWyI2RP7xT_upf=j<=!_WtDRLDUS+~vF;p58l~5{o$S&4Wd#FKw7`UC5vm z>VvR)-Yej4-9$9{lu#>T8j;ccI1qbS5iX)1qKPx|0F*32Dh4w_vh-1yeL0`2xco6v zYVja0RSCJz(kft6U=mVItU(GL@6h^>7pT7%6|uRJ25SFz4bOYA83h;?z=G$u@YHXU zr@SQOVAm@HL_T*rA<3$ONTnOLXlde3y+Dwg*^Z`7n86uGt)ZrMEvkKy3sZ_71EpO` zgtuA@SbvH@VK0^wm3ejOSG*UL@75s7`7M+QLkV8I;t5vdxq_TR0$e-9hHG!-aWiJK zz-qA~d@<&PN{=C^h#G-&Nh}d}sS)+~<$|X^71(DMg@3=u2OJQM^g8bWrew0Efm;q< zd|(^i@F4*+4OhebwH!iq=RUmsd;}$s*-3dE+5vXn+>OI~uY!Q^+wg7de*DBK3DvG+ z(e776xW^wYA^RlrCjZk*EH$V`7ys_W&oVT~2=5Ir?RhlmKCuL!`^ml~D>fV$+73Z)K9F z+++w-w{j{eZVj+m{+NpC4g~6^4xHFk&0x{BN~(849ep+_13Gi2phbIyfOZ-NMMkNp z<8=wuv3(Y}dHN%AF`kYM2FHl8K^{6YF&6-4Ec!i(iFw3~A=#CusrO%c3HgI^;C%X5 z6x?(P&C*y;6W_X+>~3_c1r_Q!T`+HzD3&6xK79gF%lL!R!^Tu(u%*YzT}%vreX?hK{NDu=ptm zqWY;ltMWJvk%~w-s)c_XUIl+@PC@2_`FP8NP$K4f8gL5y!maN5F1!;m177soiZ|zK zV9#n5k@2!sM3oj>Caw?Hz%UZOHqgc30&YuG}S`|z2BMx@(nD&e%EPu{?-=Z+gSA7I$GAq&NuTI3- z4nJVOtc|j)y+oZ_6Uo}zSzdP^I&7J+=n)XUGv*taaSQ`x99PU9IR5ugz zZd!w#Iyux0kqKIoDFsKGC7|WtC}pUA9f+$8&|8IMVExt==2|NdT&2J0f}avh>3&FP zSC_&W@1LWCkuLbuyb^H5Q5SvsiqMv0CsAAdT+H6P07X5pfj1Yu#hOcG!8xn#`10B# z!1>o|)L3T+1Kgj&lF`dKfAr0;uhSK^L7@5=<=l$>mMZG4-FjDF|Zp2`~3luTU;Dcx*osTDMM@8 zCSZSi4f3bQXW`S|G@f^2A6oXti28D_8lAhd5FIN|gVye9ykOmM`svV3ta3MyzQ5ZU zUVC>LR_)1!-&K0(QNN?Oa>^>W&QgmkOZY9^5fca&?KXk(EF-A=xtYG2v4{+dAAwba z$*@a98sY=Wu+DfT^gdgR;O;WoihBZXe(w$fp9W!((HSfnZXW< zAx&J8QA0K+Q(#H)JY4*F0s18(KxStokk>cBPtBvSzmpVfIyXcpyiEnh+D%A(&RN_O zv6!+vD+6c7l%tl8X3ASG1Ft;R09G#5g62t=(4D9n;)Ah0F#UcU6l_t0FH1{W#cOBd zS10S>%$$De$CAa6vXVwull8#md!Nzk2W{Bq;xuUggJ#WJAPeRWYvI+Z%R#2;4ESh$ z5>~0!1UB!}PQ-Y5{Q2q_moAdSpS!*iMdvMruUzu6XY+jAQ~3ZXy@~?0Tlb&< zLJlwMz+85M5C(SG;ccOjc- zdcB+u?bO!LE8L7oi(PD?ihU05Hakq6^_GG?7fz!g<|gj7Ma$8@z2V@(&QN&ZMG<=6 zFzIWs1(13WN`=}6!*umZ>aFM!5!>qLMVs^Q1XeR;IX63b4r2!w!JuKrUjCVr>MeS7c>$29OOD(5f*Fa5`Dq}q|AsF>X+0| zPv_l2ykax7_Tec?x^)&(mHvl_nMvFoUhlXYorjU%{K3|rE28o4YJF5wtO2g3l>vuk zVW>qf92VUOgO@JKqfFlrkW*HGY8B<-xqrK0!zMp?A=(apTg}6EikF1Wh3nwd5L2)+ z$eLK;D2MV43?bj)5wcJC5AH6N#?|s&$fLf7EKJ-6PhKmdAE+{*)AKScXmut#8UxAF z>I}M@Z`)_^*}{NU_f(^6jc4KC%?@PoqKCBq7C(+{nK7aKPeP%ehe4yr8l8@Oz|oq2 z98EhqKon{_6Uo0!k(JVAVs+*wRQ3G_GGWGp1bP;Abp*CU(bTUKN`V*YwZaOqhPRE^Ch)Ay#+wKhv0pJEir!jE}Hhq9u}CZazhtR z=0Etl$ft#-j?m|kF4L5{vAzX;T-QqN{G5f6^$Y4h$Mx`$Oe6+U1IX*y5*&2qeoL(E zX~J~x26!Jv!K!OwWc=A2EEt%KHEl0~XKlZM?Uy9fcO1YB@fW}!cW65o-$g{Zw{mf( zED^Tl9NHvj4aKqroDeN#49G=q8q4IUnD|TJZ2EW0t3RIFUgSY4HY|ytPDYLZ9&HeFcA<#P6iF6+s5TB>l zBL0+Aiqxyab*g5d;8_MFhWzoRTu-bBqVWf_WOQl{35-_`g1xz$;nqFM*h4Rn=qF1l zBSthZmVF3RpRL0GG_C=ekQ7*AtWG^wi$w;k2=wIKe_Ts@89Z%IILz8-3jFAF5Q_psYzeqmEVq+&>|S!a!OL(DLJju2`qEr#6+iP(5V zhz@QMuo@IH{o@+K5|^=n1UNS#bEj9bFLxB3l{H_M{(V&kse)wR?e=Vg2>-!uwg4z zdvybTFhw7Q-L`~%j>oXW)oS$HoWMVPGq?vjwsQ|u+Q2z?cEE~>Ce*K*Lfom}hb&Es zfl>PvptE=%N=T7~5>Gd<fsW(s{^Xy#`UY!Vw8{dO`PhBaZQjnW)G<27nkQ z;py2!RLkEYB=ZtL*svP4Nt^|I`!hhkcQ0jGZ3&66O<+og8d{_6jp{vwM3g|9Ug*f@wrbfV}vjtt&g_FTHFYc}uV;7$6_UmcOGmIbfqc@LQ* zw}JNNg}uGFuEu=mA0x4={AT)zy!O2Y_W4g2N!teTZoe_4-fH_HN1bXS?Ug@rf9}L>iC5y*-;e{YRmZ4c z9RZNru1y3tsGykdm%t9Wk0A8|2fg{?PJEIvr7r!hMw`#{P%C$D26u`MqR}m?)ZC7-}gWP zOCCntw*ZU8(|`kM0vAb#15aZSG|mYJSO2TRQoTx~jNuE=$vFjYZVW~`hxZYCS}x%6 z%Tn~em|d{(zCSuNr5#*&?T-Wh&ckl8ogmTl8crF~N5d3EkN&%l+{%_huC@s^FO5gM z-Rgy<_Wxl-^~>X5n$~#vHH3n<4N=wG!qKg&)wt~TW$y0ly09f=G5)8r7qLCgCFnX=VuU5X`J*MU6(sRQ;*jfXaa_wmZG zcDC!w4A>jt1h{MbiJI#woSbcwF~{B!V6{F%RJXE?R53h6j_8%pyZX1%k<fO^4uQJ7dh3Y(fewh=`poTJmyEz z-l5vOjUC^~-dq>nSgNvU?dS-;_q?36bMGSQ-#@|SfI(Wrq?IhpKFwW6n&WYyB3k`8 z4#c5Qkpd50Sri6RjGUl?vJ-L0s|jhI zP=lqyg+zVYaVQ&5hTf>ZM%_O9&=;ct>Iu<|R9>t?V;}>wU#>&478{{LJi;HRdLUX9 zhBfk-;PX!=Wv`t8rE;2J(gG#C)Jzj>uQ`jJNK7E{rVkYEt4D_xAA^g?W^TRD2!XtE zLC+B-T>R}HS2fU&y38E}1AW&yc5;(goRK-)b2kyZtG+;NxAO;)G7eDpTu9rCT?vRc zYzgBmcM?T&DCjuq0&GVEP!D?$Wer{jUDq42{kt}7ndJ>iPvoFr?oXg;wG>a;myLBc zoW*+o(y{IKQ*eB?7x&y)Htx`$gQT;&F zod=isl)~$U%O~TxBKoCPF5Y^?5_&94hTW$%aL4?UQ0h|*(Ehg*c!g}C_4)dwmh(xd z_Ma8Z`(8w;+NYqfdQIH5GJXW_G$G47OK?*9UgF`lU>Kt*Pd-n( zDhxAgCtU5!LE_(P)E;CG8oW;nZTr80)eh5;#C-u8Z3qR@m95mna%T`3@(}5i@OsCKVCPT~h4#jPCD+cP;%tPPIQgJx znGSAH$w0lES5l!9lQu@x9z2~YOKlkDfq|9*r157C_phcrC?(U-u!jxxJNhGKX7`Zd zO4pzn?pj2VWG^CY^U=@&IZ!xmU=kgWBZdKJ3}){nDb_FZkne{~#u zt=@>f`&|W3`V(PS=l>YG5=SW7I4t*7A&H78M|Ni3_nldGXO;*_hoqEFDoMH&T@q5{ zELY{uog+C)c4wCmNl8MKq>@fP-KFU8`4?v1>$!f9CUa&2=bl^_%C7wLURXB=XC4vJ z^rA76|NiJi-((kH))Z8-3)Gb@w7Zu}-Yb3-^Yo-Gwj0k7s6{JCaGAgEWOAqOvskF)+I|mpo8Mn>T3eM%vsgdpV#pB+UC=#b8m?p4y~q} z*GY>i!f|0qWitC^-C6$jx?QYA;Tnj zC(`Qu#cWaS7JB9Ll~wVAU;N-b`pgln$#lcvV0z@tUD|5!rqG3dh1T#Z=l6b><9F*` zWQQGbCUx0!(RI`!J~Ho(eEwy6hSFNTu-PavU)MF=ts=~9M3&biu7UC&3AK1HRR?%)wXW8{@ zXNt8oz4<>5u*}vBcd>@aDY0+)HnGB)NsJ#ajPc6iGfMt5#kKMmn0k&6^ZnQ&bJc}3Y z6)h4!|8Y$yQ}bzp=Q&H?ak66%RO5_GsR>gfb5fL8KaGC$cOAR<%maE$=S*hNZc~P@ zYQh*a&JjO2lOmix@r`;6*6=?%zviWxbPCNTJeIa%U9oPWt-wtjAy!x2EbiD6)~ftEiB49BXG3-bxBP(BC5pl?s)^Rdti^}`VeOAq754_0XyRG-)lk+&Nhx-A(`jK2wY3wmpvvCXE z5#+|(5UnKqX`ez@F4w0wZ=S=yjB{w?=LI60#xDK_4=sk>7*7i|;`siTA)W63TlAqd zla{&snhyS}&F9rzVb@trXI4$>6J2u17E|Y6(lymQW-wSoe17$Fe#2Tny56Ojt_YW6 z`imNQi8hAffWS^h(@MZb%ywi7FWR&ASGLgar%o1c|IsJ*ufv$1PWOc88Z~L=iS$JdKP54Pi8R%tkDdzVMU84P z7es07y%YuUk+G8M;RacY&#y8CX2rRZq>zbs{gT^)(@FLcaoqunxy2hK`(pn{ss@V$ zSzQH!w45mxLDX)^mfd?K|K$f;6wL{#whPaaytRp!G;bG3j9z9-c9@%4ynm}{VP6tr z(KeAEKR)tNY)`%uY$$v$Ftan_GnQJ|M=D4yuie8UP$*&`pUKs{D42cB3X$py-vYO1U}v6 z?>nQ&MyRUNB$nkSChTccbxpgQCu7uf=9=FX&aZE18R0mxOEB zd2~~)2J`G_9&Pj~j|shPN?(6>P7F7z2~-c6(m&O&GL0f0`$R`&BD!@(Y`;lf68OfG z85v$Cx&+GUtOpy#dq^dg=K8c{Ev-a|B-LgTFxFEUm+foUcndLIL>@KaY}q>p`~~%u}K`ByMoRd z-Or3K-^jd;(i8I(o-mCO70goN1g|XVF&*=Cs@TZfpNX5?${bHu6DX~?%B=O&kQ~%@ z6uP5tj9mFG`po0g{EaqO`I4LFjQ{9$al>J0!SwRQV#Awu;zkQQ3vVA?0rObDT6#ki zbHrF)f<3$;7#Qpl1oxX%*Zql=-0cSwJwVa|vt&$=Po9%_L);>F#WtpBgd@!+7WFB8uY?dM+NpW#e+!K*K{6hGpNei&O0djy!#pt-=9LyzCVXF6`jfPM^YbAALgy)SB=om277x1s@k)ot8^~Jy1*Mshs6~i|-Q2 z9^Oxne7(SaTmDP%XbmN~r7kN`rg{Vd*-j?>U%lk5>Rra6C00VJt`(HqlnS)|%ayo= zLka)%xFkcmkogdBqOJP}Us**TS6ncTzkx)ClMM^W^x^&$N2a1xWM>Bhg?3*fARK zcva>k{o>FP_S&$nNd31WoAvF2sGgg}ZgZ0G0&ySK zzSoWYiLIn7f9i^FEj`CBK8p!QRlo6(^?cwXKNS;+bZM35 zI<{)nTXwTa5P#i~1EQDmwxU_1hv~OZF3>0I4zqSoJNfatox-w|WWL}C*eRLmsSk#=c%4(;w=;+_Vk_)^lS13FZW{`>osI2ni090eHl7Ncjr&z zt^FaSuiE8`dd_f|8Hs)Tk4fLyD8C6$(uqmT#)uMH%Q=m2_EnnIcTg4=6>MkYziZO- zO6Azo3bW{_SXTIKP)A&J$5@1$2Z%CLzp?wyE6~SpycF5H8;JQ~R-$tqO3ab10&#}! zN#3zyW9DMmZT|U=DfE*k-s~c+DSXH5S3;S)dOYm0v$%Mt9PQpbnXxkSVrPuJWt|p@ z*-h@ctnCydTJF|ddi#hc{W!vxO?4RKJ8$eVf9$r2Klj9MS}-ucz7AT*lMS?I6z#i( zlXNGsuS+}l>yj_?+Sw*{lJ6Ae?`jiT`?{t@@Z*h=1}zy$W%^#hx2=yD{&0fiNn;c9 z*!cPc|3e7mpTr1?{#z?qOUX!-9`;JS*0eG8qtb$bNu82RdnpT#jxlCIm9)Ug`5jY{ zV<}kt=Z>VyCQ+gw+aWo#u~d+}=``b!=qs^vUm`JE$cSTC+6w;hFH25ZNAO#0beTOq zl<0yU%}x=oq=QXcultv@2^IEciyF1-*t+eF{Oi|qX@R$jc<6@*8wX?9S1P1v%QIhL z!IpLWxIb6uYI~ z@>Jo6i95ba2KY5zjk{9Hiy{aqsRC@G_5J#czeCBqMG?_=d&1v4cx zgJ|&=M*$mlKs@vI#3WJWdWLsnF@5?AlC1k~z-XTV;$uFe%zq1hFeS|q;?1?K!iJ9X zl97>7+RZYW;Ur2+%5E$WWEx2^2jVOlNkA53)c=T?q9-G1tW6PbRPPYOmQM@`8fdH3 zN8;Dlc8Krj%ZO*!-JxgwwvzaS81duOYUnyvnYDX-O*CtD8NYhpT;atR6Ikh-hVbSR zeYPwpPjt)lB>gi`L9E$b%3gkTO0@7{i#heHU9@JvlP z>B8}gbmr|l{Qu|!^y3`B^I1HHKWoH|9xx4LYp*8oE)+BhJ%?}6ryqFWvL(65sl^4< zwdI4{W*tJbPlouOa~P`IB@mMPHYj+HJCSf*6KPlvqTZ6*;9}!!v}IrgrV!YGJvpg> zj;1xCJtt$~y~IrXwqyf%W56P9yjhb3&A$Jo>X&+>PRnc5vV;nF z_&_{zHy$MsPEz&qnUo)KinOs&;AXFNL)%B%QPAP@z!^ zgEwb$UvzjjLwn&f*tU@0i9Pq`@8Kc*6AIGElxro*3%U1Ig$5OmpPrph*q~jssU>D8vQp`}~VI zR%1i>^olqG=5L9ml42~L4Z~8a_rYi3i{RVA7);acBC$_*95;I{0{u_5;k!dKF#XVL zAn~;#3`xC&fB6suMJb2Cg!ve@M|Q)h!X!MGBZXxP6+m6`OAxwVfKP26#(j25qZ#ww zf%>&?@Ll#<#9Pb{^mN@Nx(}+M-@ArEd%}F;y74qTXi_y$O#2Ls#zVklZ!^ffS_zw7 zB7ywckHj%Bik9Y{z&1%KpiJLG&@qj{?oE~utTELnMjWagP{ks;o>v*iA;$LF!DGH{h#Y`3iZq^dU4aX8*N+S`m9XOTiabBnu!QzZx@Kbph`slxz+lr}Cxi`vC zn^ZpeJYNIp6}Y22APKEksiZ89Penhrs*o3K`_Wy+K=?1Ag^VPUSzHMaKiCWs{e|fM%9})gcqytn@eeK8PymVb zX{g!!5|-R|374k2z@xn_2(9!$-}ZW8hO?vb9NGD>E;A9|d5MP`xaJOiF}wI|5K>NqpQ8bym$&_+ozs zvFlk9m@`EJ+bR?>r|2Wdb}k2HZ2AsE_m!f%xDZU)I~5t1zlPDLL*SCqa-^sej8uAi z@vY1D0G&BBvdoSFBrhht_7OX z7tnpz7i6-oHM%mf|G(eZ$2~Y%nmpQ}!PQx_4Md%YM+c17P;=&bQq0rET)!_;RCj?Q zcj!qzI{N7~x;rC`#HZ4N_7<+=4K%hud$qu-kRkQe+IK*=?Nw1#GyIKP32(wf=xhWWg}dD z>nm6lG9TWXtOmUT8bNIPTr_>6jUa36I)32M7cg5z2AuVif*Q77K&oLD_%wVPmq}QN zzN9&ze<6p!vDa?I+q_gT`gcDhWiBD^`@MKg*bT_$$RIjilQ1l>Lpu-Zz-X!u>XrQk zg!W{De*XxIcM}7L7f~>Cr4XCvy&iT2iqL?@Iq;l)2bIkbcxf~T#DwjG#mD87APCMJU|tb{5s1P=mWg*TAQb2}G>UFv=*d=S*9b1iiW{fZ&T6 zR{1>*>HXG4??SEM);pU3u9^tY)K2(T_c+I4Oqx7m%}4$%cIfNX6KIk9V%U@(V9rj)PDRlVy#J*u)L9g6ZudirE4)W3+-+S<%D*(^_v z94(=iozW%TobHgR!Nb&t9XrU#$;D)-poM$+YCiY-N}9T|^EnxH(U?p+`iMMfD#w-L zy`moMZQ$nBsBwG0{6>PlXv(fMkg}Mfg?JvTsqhO%l;iIR%;`S`RJLO>F%({=P~WUGj!YHW;9>SOEQsWx=nZEx<&54H#02LdTZR#Rq3a!snLP;Xmta zj!f}w;FNHcGdjndSlrSK3Nznx?0+VL9^3QS<`|bddFa73^w{fh+kwWPv4xhQd6-`Tj0+ zWY;3tyj&I;naqMq52T^vr)^-3+zd2j{UsDB--3L>Bop5Bkw%a6l>q+o51KrF0c9Ry z@X-?pzTv###9mxKarSN@F8;`!?=-uOIQIP(Hhc0V&^XB!PczK`vR~9;$@Fz##=p}* z*0BcvbJ7yiT9u1E)_(*t&!&Tl4eL0)_wR8&eBxc9-e_yUM?dBe`eq#{l}ocz!AN_C4%Ahr;V`& zTd?JFS10Zj4XB(Z$Jtrs1CI#pU>3aeVnY_FTPTH4`7`+B zo+l@?GJ?P=)a4L$Ypez00SiJfIep!Q^X&_WVf#^hs;dQ1T!M)(O@GKsmA`IFwaDNTR-h{L8ms#S}fHXTAyj)SrfyObJDw@H5zjqcX_8 zqYjUfdWvlw)FLcDYoJJD0$frn=PtRmj`C;>B@Hv@Q)$<^I$$oWPrH!a-+ zc^~FbVf(gH(nY%|wUL$NzcW50Z}<-SX6DSDH`AV*x4wZ~BuXMP!b9ME{xs5i8IQZQ z<_SeDiQsB<%5pzyZbhlF)2WK)IBGIF4?c8VqQ;dCDeo)(c%Ty(z4#YKxUQLxf0An= zjyQXBvPV^j^`pxP&NTr#@jxE7UQdN*jm)4&zX>RI9)p3NtH6`T3xM<6$wc>sYdAJ& z1;d!980T>z@q6`M+=*6(3oI9*CAVh7mOujKYc~?Ves_UhYY~)~uEpk2IH9XQKotHx zkIh^lg16>rg9z*-kXW1|mbP}{;r<$Mi+w1*T(Sw&KI9Rb&AtE)Z7(#SmrM*n7Q8sr zhO0%G5v%*G;r0urC}X`f1T(If4o@cujcHfFMo$|;Wf0)fxa4$XojZa332%`*CgR*7mtr zx_gI&=C~IGN@KZzcv#7{Hfs3cNWfP}ch# zUT`iJGq#(_(SNrVY8G4qf$r7#L4R$`w!xFw7Zd@u{&s_gYgd7#lWmB*8`csF*G&Ua zchlfe<*A%A(W8WvWET;4Zw~QS90k@keg#E4WRdD{ESeYa0MGZ=2J71sz_g($D86gU^UFRCUk>^?@iNXOyXX~k&MY%g?(8;0?9BS@R@zkMlL4@5u)!0{uf%$DhF(yJVd?5hH+~j%W~)FHB&t6sb(sT zQPe7vBh-A+elvNMKr=u8F3K_MJf&*Nr#i|txuv}_W|9L%+`{5LW{NSB%sw}jldCZw zZt!gMD;y;!XXWTY)PEqoF9bZW?LcLCA-T{z z94&adlsFXe5qznPK%5ijz_Xtr&_sCw`A%XD5%Ue?p56n;9+;A*`^q?~TbyBOe+YQN z*T$bIiqWAlcl3Q{9BdT(6K18KfL{e4k(CU-PvtLq(RL1DMfRxFY7*)8cK}RXbqXc& z`RH5dJ9x%ipWLnU8a)ct!TWa?fIZhLVDPpr$cI;i9xWD;tw-*lo~ON}M$vgrUM7!R z7WfSG5EE_}%kjt?xqdLCN#8JMjfDW^6uQ>pnQ-eg0B6Zwfg1k_U7GbmpnpDfBZrP!#$K)`P!`+Jg6e4Gic^!*-^jzh${{hiRIh=8Kr z6IgEEAsEuq3v9>;^y|)2xEKU@#S2Q>DhsQ zC~O9CHZ($ND$sS=TaJxtImoe50%f%n;T4;Y*1K(letS!Z<_b%6C22ZLzj~Xf9(E(| zB(~s&=GACP=Rd?0PsdJ}E+)gz8o;;CUw|qsg1w*miLD3Iaof^N+?y9miY6t&e;tm)c@zw>cK(t^cco!wJT!*MNCy)qTqa3_)1#nl@5&6;Gy>Uc_h`|DFekRY!(q61+WY20D@zjw*+z;bG4O$YJqHpu}_% z3vnJEl9`7745q*{ua*X!3*@0`1`g6wfOzz zBrZM!nO8S(V>4ImWzpC0%n{1Y+F&Y$e~kqESZ$dM9@ zU^1os2Gmggjgsrm6Gb1B@Bsq@vP8#|EC`TA>to}<_7%(Ea%V%dFBL~Sg3Mu~<8So+ zUl^Qn;4!w^yBZvw=Lp8syz!6ctl`655y+lzi4S%96K|iqgXt6NcpL5mDr8oJ44(x^ zSMr0i;g%oR=-vgVEk6w2$2yT<_X7CvQ5){Czydy9^%Y!kolV+!IFRxcs_<+;B@kRa z1O*aZ3#wo->x=fVq>p;U^rF%TU(OIGX2QU~Vek#kaYsM}5LX#KWZ)UTn_NS?b5 zy_r^zrsm5~8r|~R ztY{!k)y^VugR_K(;TaUGe+&kcpGEe~%IJ=-I%w1T4@f_aMn?+TL7xK`Ji3+-ia$(X zD!)K%NGlHweLa9Ve~cq88#JMvtqq`|=kNrd@E4_AdVp&sR3RU8O)|aB(e#?hYh)zq zAZA=S3|`Ygupu@d->&(RIA@WL89wKe6{iw_XN3s0UU~wzyX-=H3=g4WYu|#QIVE7+ zvKNrjPoZLM64W+~B4Rq#O{J)NM8#8Sv@t`Fv7 z8hezHtztSV+5QFM_J6r2vd>d0QAg9Fdu0gkOM2!6XEFSNwDJYLaY*A zo#1F@fWOD<;NmWOqu1%p(0C{TWqz*$PRc3J*km119oRus>#G3M`y0`*`M=@r{y>xP zb_&n^g26`iI+ipRy8 zg9dHja5Dy$1!$s>2gk9E4hMm%t36Pkk_8*92Z@lZZ@}uADbOk%h7QRLwtDLZ+N%Y_s8->V&U6>*@CM{S)_MJJS>xvAj)tC zv1^Ymvg}~dme?rB{4jtwa&tj+u`|k2fCbM9KzX+hfQ%{HrWrp&VcYh5uzW!yn)*5wee%|XTX+8kw^jyoe;Ij_i+>bT ztdBL!Z2LpLzId73p16@Jaa_y&^*n)^zXMUt!V%w|9BjF9^q=X3Yr z6`XNVhxGKk!S&cQPL=&FqspgCaq)yyXan;}Y?l!=GxR6n&pf5#-!I@=L_EM`W=6t& z`Q?~%xD;wQ)d3vIMaEQ2G58;?8!J;K6PW^zc(C7W1_LMi%Lkj>_kt zkGD2l@Y(_nG;ASfkB4v^V%|cmpGO3QD&T&rKcRlj{ite93aq^v4encwf=opt)YqYd zR~DO)*o$}+_b3s)>UxPjG~~jH?2G7M?{XyjpAD2GV71e5;|-SZ0~ZJONfX?)sgzm3qZ+C` zKb+h}#Zh;)($S5!wWMdxbn^J=MDBpaO$1i7AQ<$WRB*CFmbpFXWt3LwoULFYiTL3oyn}W(O+(LSRjc~t5BH?*? z5Ja_eV8os|q)>ewK33caCAF%s_Toyya@I?9#mWlGddQ#_^Q~~~m;<3#u^vQ5zXyH4 zWs#oTiurabO6ct%PHJB`MO-V}f#f(6;>bQNP&p+728>veCO#iwjQeE#o=Y?F&uJ3c z<9rTmnkS7;&AW%fr&|J%<{)_aZZ8qlH8KADf;<)`xd*!pBzbd3xqFqqBAtDuRPW_Eq;ujE z(x+NTnL3>#cYiA)p9g&7($`zK#jCBUuTC;l*!Jb*z2h>}>73Wpx@~W$XCBYF*#(B& zp1YS&QG6_lF^P(FkWXxD(2M1Aw!m zBepp?#%2BMKrW)-p(Ob ze@uaO7tdqnJHv_W6=AS}I}4nBG>!1Oq6VV9>mc)dfa5ZG2tV@Z0wj0;#Ev!9BgY@g zpy9U~FbtXl-@D|49Y?Pqy$4s&HtPpOYJW2Lcx4job9oGXdh$^7>$s}Feom~r)#UZ=^%MFp#g#H#j8445NP`_TNxhM# zR^4;wo=C{2sxK!{LWP&qqW72JmIftC|Kc06!r(WYHRwpyemur~IbeaU+C&kr*X~w}aAU$EyixlE zgfpE`$}$Cx$MR%!|L`2(ad#u(zn+1+?Uv#P#;qU>wj#=^Wyo{Bal|{#GpK264Z1Kb z9gcnUMbiIi&aW?+Lb#lcK{v0k6DFM#zL;J5Sgl_3wpH z;u2xH_{dtqDSsE9HoXZm9N|JO2MP(@)^aXizE9*YpAWa@E+r1TNr88#4iN@hi(%vR za?YX3pTzsqQm}Yw6j+v%iipRF@GAm?!OCk-O}gQjZLDY zP8pMtdu6G_LMJk1ogt~dDF!`Okl`jq*u$*&N^(!(Q*xm95}8zELK3v63Lzbjx{c*-abWNc!sV9_JyqXfnXrP$>LehhCiOhd_ns~8d zGNo90hRiRn;p9Ct0C{fjiLgIAuxIyWiO$R_Q~MYed{8-xjnQw2mDVO;O+qc9TmBa$ zMZE%-Cvbn@oGQ-8OAm-WiWMLyMhu4!nFI1)HeL+Q5K8sNKtvB2 zc2WqI)3FQ~2h7D+CSJh)Z97LOt?`FB+Mn@RW!WHrxh{snsXZ%r~Xr+hI*i?dBNTc{BjOm=uM^f0rYsnZvoMsYH&&p2J$k zFhCvmg5RI3;NSFcnEdNG9?j*G^cF)Px8e)(gBQ{IcV|(>x_UG;^*z`lzXZ%)69wh# z649o0!|-!eA|BPQOw@Y`LFMdFsNxcdCePjqw`KLCmaCtzou>DJt8pp7RB|wk?14){ z%8^U=5@hMFPkE`{M`Th05vgiMDc(1R`{P~U_y-60CNd8N*hr(I3H=_0pCdD8(PYy5 zMza0-DJZ2;1{tYHGSamidAH7iMI}7AVtWxvSgcBRP3|V2huD#Y{<382lR1ct&IiJ{ zRY>#87RU>H4C*S)$^K7C=#RSr=KM*4EGxDLd-nxGm9_tfvU@G2^L0N0zgHA^9b}HS z_|qJxL;I2K(kDneelzUvMd*+H5az0@0+P?a0rmQ_WOlYTx?=f*Q=CiS8PsOjH|;qF zwr4{3)?(-|_+f%$9VZUUWnodmM3h!`*EG3X2WT$Y3TxITVt&7)P^@1k5!!ncxQ$MR z{hMbKPbXEuQmON}Z^JpX*jx!5wx>{saoB%g} zUx$5|^_93f>j79>fpE83X@twc2*UeY2Oxc3V|BBBgACUp(CFez*v=%;l*6e&-}D{P zHYo}W6)zxOyILW#bTTplDa6^f5wJFK35aQ317AMvLuE(aK-KtKxV+g9d86iDxZO^WF|VjrvS~1jWN5jb_REC#YyUt$vU%J`zom3JY?3tSA()+mCcrf z@8X*I%9*Lj7@5siImVS@BF*514C>C&ddi;JWR|y~-R$C`e9Etlt%nlW4;^GC7VC?b+;?dnW zu=(gHwz5+LyUQIX^xZpfjmkw}+D`+#zi5E)_1FyQH#t!I8wFM@iN{wy)4+qqb>J=g z5aMEFEC~8?84OxI$HKYi2!$_3XzTtk;(b*YsF6#DS;;DR%M)w3n&~CFX2)SW?N#8x zJ1?Qjw{7@0g_ERGeKbsZ{SXF4+5k#c0c>oUjdtHE1gg|Ja`ve*5Ioy}{IRj0B@6aImP6+pEw2%S(1!{oG- z(8!Z5Xj54h2<0(k@~tbx*nD?rUVa-&Y_gzi7?0#V6p$J1M?hg?D)@V|5jMW_M_bY* zC?Lq4Jg3}@q=SRWUoB5e6HgAJ*{K^rv@iy@UGT^BweQ4PH!BBrC>EM2J22ewxU19= zCk5^r*$Y(bh7>CMLIHW=LFom(HWmRl572-m-i z=C0{S+#P>d>~4(?TAJ8JXde6k!n?D3N5Ef_0GhjW zpjEdtavD}AA|-slJy!y%L%TrY^$S3e^~tx0!+#O7iIVp9MIv^eZ~vVhBeMr^Aeq3ZQ>1hq!;z8uFhU2M_9{ z$TJyX;Nz=AG`w#Wc|5lm%yKC~Rv#9CD_4)>mHS&z^j1sq)2pvAMDYtzcWJ^$WNLs! zG7&Is#0GAP?L(*Z8SJoTAyKqe3g#-jhd-jLvBS4zi1{5UaBM>{CJHYFb4LIg9d8D> z`3hp6Y9`l+t^X6)EME+dE;x$jJ{v*p5>v7+@huY8YLVHWFL8ouexQSwmB5h|nfOi1NMiGk zv#6IyfN_DIT)p>p)ZgYI@5h-~Hmj zirW{Fe8yxlZk;NDYiuAxUOXV{Rc>G=#?8dZPzWx$QPAKx0G-K>*oJ#0FdrKvHnr!2 zw!kdl@o*(^XT1yfd{79r^p)VsscnSOvGqhPTn8V@mqT|Z68{&!0LTT}fuM*Jgm{Yv zLYD8L>(fyDGB=B;@cBZ#K5PgZ*4}^{p*!B9?gx{q*CM4+V>~u66Wz|4@Q@h`z^0qS zV3VON;gMMhc^)J|zbxz&m&MyC;~?HUD+Yn1R6PA2Hr zn+|Zwkiw5OJq0ce1S;583-3jBayPW@rSxo5$x6qY+^{Yw%7nhc&3_wC+PXS&ogaRp zZWkWlzR5`Bn(ewu&Gl9_JLfh;mLvlz>SP7i^MH+6S7A4K#m$2H^;L#aPHdpkLwIIG zRyo}FoJZWD?z`MsReI!R-^pCR|7*Jruqcvl3rHA7;t&OtAQGm#tE#&P7>1yzAR+=v zHjM)ef+Ph*L03T}tVl3|n81LTT~sod<07JB&LXD8ydo;%>w#r|)2eU(_xL_&s;lm; zIk!)pdwaTj=Gi-{?RF7*9%n_Lv{2w>A56l>#d4rGnMov6rGWapDR|bblia*Gb*Sxq z6Aw064)|$H@psB=frrK+`1=fH!vz5o4*s>^q&AP?(Kt9gNax>!4j)T~VLs~GR&mhz=XcaN?Y$Q4L z?SWF`S)%BT}cPi z)~-j~^fP45mOOOKsR3V|c@J>UrJ%KGX~gKNhcMH6D7kIuX)pv0fELv@Ft>IVnH0B* zJNs!R956r^Tum0>i*$|2=6gP<(_j^}AD;u}20aB{wTfs&c_}Wt$cxlIlZ`5>kD^h# z3ea<2Gd$C94rLwjLX$*t$TLosTvJkpvd&%vcZzdC=b&|PUm!plhZZ2shE}qqI1f#H z#iC}J`&%}@vn3~0=|Wq@Hq3ed7Ln$d|)X$P@S7 z(T#YVJV%_M2VE_Mn{G`bho{^}3JwP3k$Wm+@JT;%_px;d=B}VSwe{%N{C6aZOCj0u z9w_IC6}f>GLk`@kOj$fnr|(=JLMb2o07d-wBR6tn+AxPdwruqoT4e9gn$&`@! z?msR3w@5`q*pN`>h5@E?L<-zoqYM=yqVXrICNtY*^f;43FJd2i4VwkmpQ> z;TM)4f(o;hpmS^+ciFH!csKJI@!^#^ypU}{c-`HDd#!CCqS9gr7|;l-vuubMBX9Ud zKqGU%VyrQ9BcxK-64ApEC&7x?9G)i;gdsz?HM&_GTIkUN}UPD^AACH z`J;G2at;3Og%4p+>H^b;^syYgDF`2Kc>!-XK8o$nXeRo3KzQKyF2ZG=K6kZoIpLFP z4Vh2=vE;jNCw6uXhD+Nt34cv7YCA?@eGSWrHwWdgGLOs4czl|7q zFc}=;%z?WN)HQ-dD4QZvFgP+{8r>Eeb_l%<~~t+qXaysQvG zE*sKFotQ1AcW!@z`fki52QM=rjUNx7+MnmrhR6EQCWcz{OGk`O(J(|IeMA(NGM;*@ zTnZyM)l;&e3u*ml99%vml{gqXhFEBw4x&!Ff%v)%%czOkplP2raY_(Nblx%t8*8Kq zx3g+Qj_)?`de1VDqppE@=l12A2D}GYMPE>OvILjzFaWtZdW6onUkLe2Z(-mXIoP~S zmuOG{K(?)hsCvPI&H?$*wzLx)KY`0!Z+!(v2v-towyKc7u#2H$%P=&i@jW~?eiw1+ z)=p5`ZVV0QUxf4DFT(g@Eu@;AOj0|Xz}1-PNDveNcDAnq;p&&*xcn?~z$t+K&{%{U z4wM2T+ntfONjzA!@ILH_(jZwD0dPA$7)}o;0;XUwsS)9Y70qx!2hX^Ia_(--;AIp! zb;KC7G_?j^*gX(V*S-kW!$&Z)gNw7L?;w@67NUr94f1`ZIrCnfC#p?pN3m{BXmf}$ zI=!Lmvq4=$ryrRQMXo^)Y}Io~`7>=VQ@?Lm0Jvhz+h#tW5HD>3{>; zwcLwml+d$N(y(>oF&O#5)QVqwk{;Z|rk}l6qjM)7p?oa->2{|;Ds)5%uc>1m)#kp7 zwl3&T-%{#N`@gEh5HL+?>$AMB`fCk{QKkm8+Vt}7V_ zRPYL8A@6#ho#b_`W%Qi2%V<8Q5=D#F(+}^J(lU$2;rR#hNUX{PgEN&;Ez1|m^HuS3 zr+q}-pKO@rz#|9G1+aPdEp%+TK6ycLGvJxB$ODG%M0(8uqHD^2LU3{liW!-Q?!R4* z&(c{B9`5AB3R)dLx}-_U6wg69m?7G^nFCX`i$I3*Xi^8ii)o`&sBmxrG4!`gt}w|G zU2JpUUR*1SKWh#F8cKOs^Q>_Y?0sd?Hu?n=+-}8W4jzZMIHu^gOJ1-az7#Iqp+yKh zw&SfjaWLGw5sp!L3(gg&;Whr*1br%)5RHEfaw5|~;k68KABx)@x_q4LJsTyP@mXTdKOdg8i3#K+DYuqT7Xer>+xK|8kc=q0^M(V7>VeLMNye3wU~#Yb&<~Txr-q6u+s!bD#>i_~&Rm*EE6Hx8fRU z<@uv#7IJ9UwYoHlAxDu(kj z-0?v|U)-my0Nv$;NyhVtV_g#&_p7jPZM`{WB};w zyADsPk+t+z97XI~xC8{ScMy^J`Q&|ZA*@-Hgtoso!T;EA392{0hpZ?aI7`QpT)y1| z45^bMFEw?LDHgO0zfA=-9`cw-lJl;&+_Af@bbQrF58H2EP5t{7SL=3Nq zMyG2%$z7vTaVt#)`18OgcqkGBmT3CS&Sj3Q;58jba$5Z5=T5n0|5_N38^=2~G&<`G_j^S_5g);XpI-m`4 z6Uje5)Q~m0eyC~l{kA4&wWxZ@;%U>itv0%7Peld>TYNAM!21)b5y3$9{ujp_c>fr zYATn`Wj&;3V)-c8DUi;6HJw+ao{PWwScq;!j|SJU;i$c6I21ikv54Nf9-6$8gL@jS z5PQ%hXu+KKTfNx~F0@Od`2h!zl1>znIrSB;60{%sQyYnUcDgW2+8+8n^#|mkTI|vx zB{KilPSmUNPckG0%z^UP@!0w^@k~r(EFTWF=TUKnk}`wILSo ze*q6H`je}`odg_LZ^5Sx-GcWOW#FxZ5YOLM#GKJJ;r@Mp$7e?CfPP2@?8@DNt7^}{ z+S=q{%wtIvEvP}ajqheNvOkh%OIwIz?v|A@)oQ<>u0Ir z%EMM&Tn1dm&BtQWRw4Pl`-y#jE+pot-vni&&*SspVOXTO6t|dFhQ^JoCR)?xp^rQN zMCdT0^xKpvfPQbg$`-r!KOEm&~? zf$iJ0pk2N_Do+>>2dhtk&&wMjyuF)8>QWiXY*2*$2RQy%N=| z(t%tmv*1DvQ)*IBE*xf)NtGU1h)N;j2{fM$crh=6GL1$3>MGST+dj8|G%gc9DUT0!QNWPH7TBcV!4JE)SG6 zJ>e3+0_eB14xf#e;M;g!c(LX-aOz_h*mbHDBt%aq7VpWy#4S?93K0p*-*|#eZi9&X zA0`mdVn?h?l>~m1wqlWEyfNjze6Zkk5%f$ui9OVt1|JzN!Ly#Ff~xEF(9NgavTWoK zsPoi-u&6UfyQK1nyEa>3$~IegdHzwDe9aVAw#yL198VJRUbkW4&^(}K(N1h0*GMeL zl4Y*bxMGBnA{un>7hFqeCvLKEC$xFgN?dqdKww7ts90(^ez3g-TD7^N6E?Nnmw^LO zzH9`MLOmd|*X;r`yzgN7Pb#s=&k(m&Jqv|&o+BDwKY^XIH-N48OX2b42{19y7!2KH z0PY89iW<%}j5ea?|h<%Oh?c@#CEb^$#?z90Qk zD~HY$45t+rsgMIRDk#CcdDM*(0Xi19n7W$dNGq(qj8_S@(P@PQTz``<@IIXcbXH|> zrIiBk)Van4)wTehJ2R2+-);ssIR6Pv4a>o?rE+j&O$Pq%DF^%IZVH|}42Sn>6W~fW zS?v6EdBQutk|=rRg5%k~@Jc`zn1-bSd9N1)`_&A*vsnr0tZ%h++o2BX=on0a^BecZ z{(5AI1rf2{8Q8^_EVNK_B|d>}fd@x?;67FCM^>14gK*DX=o$A9q`A=!v7aWOlW{Dp zY4sjrx=|y_i77(iH%nlUe<;}Uq8(65T)4(!(`kH(c_eh0ugdKsuwt(58h{f` z&2a6hMQCpobDg$J0jez>hCe+ek2#Abph=IHqkdBe*r~r1xQpVz;SWVn?f~?E=J277(NCClH2p_ z;aSX*%2n1udTX8_i~E~N-l&b_(2`eZn1M7}BLAFdIH2xmoSgOiBVb>Sz8QGuk5LAF)R-A#hJ2Ch`>L7m1 zeVrwKIhF8G$^{+zH{gpyGVrbcY~oDW0&L=)RIv4|0^F6cflv!u4ZP=F#V?Om#M4)% zfseoL2W^(E+(MZ|Y_N_m+-1tL73w6`>|Ll>eKa|_I>FIg(4XjErCK;wa9!7}GHZFV zUD;aQD*0u;HCt{e*@03uJAF5KyT(AiZR>lRYBM>ln!NRQt2<7e73i7eSGNh>Y*zFS zt}d<^QWLt{RdD*ij_RXx0&6I{LR%4Ey86~{QPrZ1$g0d!`vvPSFRO8_n9pBpoKbDI zdSunR&gWHuX^X0C+!OhyJ6~5_ym7PgWx*o>s`n``TX(<8H1-`o@!Imr5Rb4*bYXhc zvt7@swmT;B=bQKl`bSKydR;i8x-_=F%IRYz|H^dznnkX+_%|ur>fQQpsvlMB+rr~5 z)ssC3S2sLeU%6$|K*5qw^{NSIu9gSw8gVGh35W)8^W&OK)rQT<$vDEa+E1K<-TKq=zPTH$}&5 z0}P3}cAUN?1iy+ahehQI2O zTg{3&ysA6d8P!vpkl?J!WPSsU39=H5s@A{w#ctJw)m70__BDaNq}@DSOFJX|6M~ne zJNVAyeyv^`(zixiTdMj;aANhoOSN#~Y zr=8~Q-8c$4hb8mE;@{HG%H?@)_dTN98(yH78*Hct`7h|y$}D=;?M(W}iw>$^^KzIY zT13Tny7NZwUqd~5Z$iJ)n$HWA!%5eu3X*CmptP>7KoM(Z(<6%B@bnL0a8AQfN>r49 z8jaVId0`JxnZ^dF7O;UlpqL3a1ni=_!X2SsNglalX)>BIMhE#97$E7O^{CQq6)C+- zhCH9Mk$PCWjJYQ@kV?Ngj69r=p}H7(o{7vuWcb9EqWWy6BXmZfIltx7!kaFXY=kD9 zZZ?XVgdIkQE9-c>TUOZ)I2_K;OE9uYyvVh+o^aJ>KPG2u<}%#IicYp|oTbO_XlC0s z78}^=oJ!%Fb~@PRJy~QkF@1-PLeo-POP?R&3i%cH$*8$)y9_M>8+Yvn%l#OnmLZ&M1z ztJg)8U@ld3K^4t1JjwGbR3qi*IP=CDI`bM^X`Z@UChayk1$B%$P5H{6r~UT6CR<`x z(reUR>9?C2$QOyW)EK7%-tgp^bkzEr^h2u$yzJ@=DCE8cZ8hGR@>gDjSgx1o>+6s5 zw!FDvS9q zE4yPq@yP8Ou}@H~SGi2>Y3_v@kIOS_Ei^aOI@~F?QY8A=G+vM3g^j#uJ$18_RjpzK zFMzk(3VobP7YmYYcBW)oC9iU~$-j2U`T{OvvtwX`bsxNy*SSR5COyv4x<6N!_hhcQ zjh*HYo4NI+R$eW~tRD71Y5n%zZL31-U~5pAVho7EqI+147Jl~xh*Gp(ZKj#+CR z*RVksPCLr)OSD_7er$-8l#!EE_m7d&0ZY!7;%)3g8MZD z1cTP+7_|QhgU&x=(ES`kzt1q}{WJzPOP?dp9NTDdB=f=zII=xfZbUylP=+hT-Yvt{XBlyHq8WO9i-v5LF-N|e;&BX* zO*jfs;`rF`;5fD+YamCzXH|TgrRpqGj&jd{UYq{|?tf19?-KZbvl7ghRb%qys`+}w zbjR143B(|d0u!xZu}~rcX1xu`QaA5yP)x(3w?Q+Zme^VL?vU7Vdm9ovthXVt!+RSN zJIP^6k!HwCaM+&+genVg`U=D2B`jxFbsU?+f*ciLq*xRi9UeV}SqJfAwkiur21SR6 zX9+{XgX0-5B;zG2CQ2L~KO-uR89*@u62y@)!HhUC;-bl-IcmZg(?YseNxXQPa8{(K z8@CQygT>>h#))GSB>w-B4d%u5W7zjKGrMiR(SMC8py$uOz9C{lP(m3`uddW7pcZbAo zNN+=8=hWMf*bVJ%^pdSbB8kI3m&D=!B#F-dEQv0h&$85r&m__HrzKH~HS$}suEKKr zTg;gZ#Sp3g-^!`UNTgbl*-+F5JiJnUM$LWzziKa60Np@5ATl*wqmr9g! z)c5%5{s})vf67mfAMw-kYkvOMDM^X-Z*!0tOYf)osmc(@tCv8C36o^Um``_1;P|ms z{G7&)bQr zs}A(o`t%euzC8tvUr#~f&lEJwkN}Rf6|)KCNLw?TaU5wIW;326&1W_fIMTLkdDcXZ z42inw@GbC)f9FU&^mjcAKgpaw0zZ!9^HoHZKg+Xv^e2u#>-mvyNuVSJ8xsME3Ng?n zl^ugwQrR(JC6yh6@;{Xw6C=q$CODFTOjINTnQ%x3PV61{W%&Ctof?c`cZ|O5{9gMz z`TILv5|7<-Iq7?HDf~n(Cx0rJq94g+(ARSLU#BZ2*1ye9W-QmAmdo$O`pZ5(lZAsn z&BDs8kWVwJB1_DXi;syHMY0uHp&Ys3m>JRW-ACA!v7GX`W!UGI;s3Fm`j;hB8b&as zVQ924Vx}l|O3z8h6prXC>KizPh@xPR41TSJfPKeG5k)%xd&61wO3*@=FrWYd`k zf4@veX5U8il?;Cy8~vj-{?Ai>h3zM&MrQTzr}+xmcc!ua z(KM4d0bilf_zTTvmBhDU*!+9{3GgTU^MB<3TP2YCXRrS|(XsuJ_lX?;ucFiAz4z63 zd#Ap=Fn?IEd?`zkwxmWD_avgw-n6grgK8%V8OrMA7F~7ZaMoA;BO`Q n9`*aS({5q>ky8nz@E%B0KZW$Ej{XZsp6ym1bI4zhPVdnhdBm$ zf=vr^^a*w4U<4X72WSk7>4h)l{2~hcCHW@CE<^7br zEGi>qOjw@+)P*a7`hWrK?z-tVP@@S68kxWv6S>#U5|t4(#jlYSW*f6c1B>bZwOej7 z7>XjrCL(anh%;y{*dVbLH>yNsB+T*afu|&e2^)dh7)_J7I-7vnEQr>&39RkLNv&0) zGQyVlwLzkB0yr9VZEK%1s7WAvM<59xMQ95+K5rkJ$iXEeNkoc+#%BvJI9ArY%$Ug~ zBT69sKx3t)7ibrYY2LYaObkGf%FMt3W*nR#CIu2;1Q7>5E!@Ds$Xw3Q&IK3a(|;q? W!^JU^iytI}AB6vI=vFCpzYPH1UtRG4 literal 0 HcmV?d00001 diff --git a/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs b/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs index 382941d9a..299337cde 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs @@ -84,26 +84,12 @@ public void ModelWithSelfDefinedModule() [TestMethod] public void LSTMLoad() { - var inputs = np.random.randn(10, 5, 3); - var outputs = np.random.randn(10, 1); - var model = keras.Sequential(); - model.add(keras.Input(shape: (5, 3))); - var lstm = keras.layers.LSTM(32); - - model.add(lstm); - - model.add(keras.layers.Dense(1, keras.activations.Sigmoid)); - - model.compile(optimizer: keras.optimizers.Adam(), - loss: keras.losses.MeanSquaredError(), - new[] { "accuracy" }); - - var result = model.fit(inputs.numpy(), outputs.numpy(), batch_size: 10, epochs: 3, workers: 16, use_multiprocessing: true); - - model.save("LSTM_Random"); - - var model_loaded = keras.models.load_model("LSTM_Random"); - model_loaded.summary(); + var model = tf.keras.models.load_model(@"Assets/lstm_from_sequential"); + model.summary(); + model.compile(tf.keras.optimizers.Adam(), tf.keras.losses.MeanSquaredError(), new string[] { "accuracy" }); + var inputs = tf.random.normal(shape: (10, 5, 3)); + var outputs = tf.random.normal(shape: (10, 1)); + model.fit(inputs.numpy(), outputs.numpy(), batch_size: 10, epochs: 5, workers: 16, use_multiprocessing: true); } [Ignore] diff --git a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj index 58c176e82..3910eba1c 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj +++ b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj @@ -65,6 +65,22 @@ PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + From 7cd829288de2f04b701ff03d29edb25a4d151844 Mon Sep 17 00:00:00 2001 From: dogvane Date: Wed, 12 Jul 2023 16:58:25 +0800 Subject: [PATCH 05/98] fix per_image_standardization run bug --- src/TensorFlowNET.Core/Operations/image_ops_impl.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/TensorFlowNET.Core/Operations/image_ops_impl.cs b/src/TensorFlowNET.Core/Operations/image_ops_impl.cs index 0ced407a8..318b8b142 100644 --- a/src/TensorFlowNET.Core/Operations/image_ops_impl.cs +++ b/src/TensorFlowNET.Core/Operations/image_ops_impl.cs @@ -102,11 +102,12 @@ internal static Operation[] _CheckAtLeast3DImage(Tensor image, bool require_stat { throw new ValueError("\'image\' must be fully defined."); } - for (int x = 1; x < 4; x++) + var dims = image_shape["-3:"]; + foreach (var dim in dims.dims) { - if (image_shape.dims[x] == 0) + if (dim == 0) { - throw new ValueError(String.Format("inner 3 dims of \'image.shape\' must be > 0: {0}", image_shape)); + throw new ValueError("inner 3 dimensions of \'image\' must be > 0: " + image_shape); } } @@ -965,9 +966,9 @@ public static Tensor per_image_standardization(Tensor image) if (Array.Exists(new[] { dtypes.float16, dtypes.float32 }, orig_dtype => orig_dtype == orig_dtype)) image = convert_image_dtype(image, dtypes.float32); - var num_pixels_ = array_ops.shape(image).dims; - num_pixels_ = num_pixels_.Skip(num_pixels_.Length - 3).Take(num_pixels_.Length - (num_pixels_.Length - 3)).ToArray(); - Tensor num_pixels = math_ops.reduce_prod(new Tensor(num_pixels_)); + var x = image.shape["-3:"]; + var num_pixels = math_ops.reduce_prod(x); + Tensor image_mean = math_ops.reduce_mean(image, axis: new(-1, -2, -3), keepdims: true); var stddev = math_ops.reduce_std(image, axis: new(-1, -2, -3), keepdims: true); From 0cc25fbc35eb406c4f7e93ae9894633c03bfadae Mon Sep 17 00:00:00 2001 From: dogvane Date: Wed, 12 Jul 2023 17:00:16 +0800 Subject: [PATCH 06/98] =?UTF-8?q?Add=20a=20function=EF=BC=88get=5Fclassifi?= =?UTF-8?q?cation=5Fstatistics=EF=BC=89=20to=20count=20the=20number=20of?= =?UTF-8?q?=20label=20categories=20for=20the=20image=5Fdataset=5Ffrom=5Fdi?= =?UTF-8?q?rectory=20method.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...processing.image_dataset_from_directory.cs | 32 +++++++++++++++++++ ...eprocessing.paths_and_labels_to_dataset.cs | 1 + 2 files changed, 33 insertions(+) diff --git a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs index f42d12cde..377ac4de7 100644 --- a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs +++ b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs @@ -8,6 +8,37 @@ public partial class Preprocessing { public static string[] WHITELIST_FORMATS = new[] { ".bmp", ".gif", ".jpeg", ".jpg", ".png" }; + ///

+ /// Function that calculates the classification statistics for a given array of classified data. + /// The function takes an array of classified data as input and returns a dictionary containing the count and percentage of each class in the input array. + /// This function can be used to analyze the distribution of classes in a dataset or to evaluate the performance of a classification model. + /// + /// + /// code from copilot + /// + /// + /// + Dictionary get_classification_statistics(int[] label_ids, string[] label_class_names) + { + var countDict = label_ids.GroupBy(x => x) + .ToDictionary(g => g.Key, g => g.Count()); + var totalCount = label_ids.Length; + var ratioDict = label_class_names.ToDictionary(name => name, + name => + (double)(countDict.ContainsKey(Array.IndexOf(label_class_names, name)) + ? countDict[Array.IndexOf(label_class_names, name)] : 0) + / totalCount); + + print("Classification statistics:"); + foreach (string labelName in label_class_names) + { + double ratio = ratioDict[labelName]; + print($"{labelName}: {ratio * 100:F2}%"); + } + + return ratioDict; + } + /// /// Generates a `tf.data.Dataset` from image files in a directory. /// https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/image_dataset_from_directory @@ -53,6 +84,7 @@ public IDatasetV2 image_dataset_from_directory(string directory, follow_links: follow_links); (image_paths, label_list) = keras.preprocessing.dataset_utils.get_training_or_validation_split(image_paths, label_list, validation_split, subset); + get_classification_statistics(label_list, class_name_list); var dataset = paths_and_labels_to_dataset(image_paths, image_size, num_channels, label_list, label_mode, class_name_list.Length, interpolation); if (shuffle) diff --git a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs index eaa762d89..232f81eb5 100644 --- a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs +++ b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs @@ -9,6 +9,7 @@ public partial class Preprocessing /// /// 图片路径转为数据处理用的dataset + /// 通常用于预测时读取图片 /// /// /// From 68772b2cbdeb431a432617e6a5e8bc5e2b2ed754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Thu, 13 Jul 2023 22:51:49 +0800 Subject: [PATCH 07/98] fix: use git add --renormalize to make model files binary --- .../lstm_from_sequential/fingerprint.pb | 2 +- .../lstm_from_sequential/saved_model.pb | Bin 755111 -> 755111 bytes .../variables/variables.data-00000-of-00001 | Bin 61038 -> 61038 bytes .../variables/variables.index | Bin 1373 -> 1373 bytes 4 files changed, 1 insertion(+), 1 deletion(-) diff --git a/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/fingerprint.pb b/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/fingerprint.pb index f6ea8da23..c37cc37bd 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/fingerprint.pb +++ b/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/fingerprint.pb @@ -1 +1 @@ -沦Ʉ%̟땐͉ Σ(Ћ܇}2 \ No newline at end of file +̟땐͉ Σ(ռ2 \ No newline at end of file diff --git a/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/saved_model.pb b/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/saved_model.pb index 6fb7c3f0e8e4a35afa38cada60f78a6097b84501..618c800eb45f0481ad202d25bda12a86680eecc8 100644 GIT binary patch delta 2303 zcma)7O=w+380G!H`yL{e2#Sp_@!iJf&$Rd6`A@a6-B?W8DiRbPNt~HGvk;>7hc2}u z8X>eQL2fbV#zhwCq7q&Qx|N`4Hbq256c;YK%0}oy7drReSl@Ei_s)FhoH^%wTkA)+ z){kC8n;}Fr!dy_mq=*2LhyjL>2GS5sQXCTlarW_aWoc)$dE?b` z2r39LMj4hGf|QQzYf12RCA)hfSkCs3H_6Wa?%TsqDo&VW2x!hEKnyYQSC;L?waUyXubXoUQ`zsgTJszG zyMaCNZROUI7ny>4c7CQ^x7&m2pKE=hL>zM=fMAXciWmnXi8L@WcAz;$RAo1Lb26J- ztDLmGFRMSD4Vg>~L$L->=7eIdfFMu-tPMm7jS~^W{}E=xo3&u{VeQwR*TG!w!&Q6r zq1vT^4?3Hpt7ex_aLBHosiC(*A-DKBXGsKeE4_OH&gePO@8@_?5S#sJ!Xx{Y&vFIsnAb;162tnt-S$PAVQ3JxG7 z2|y-+Ksb&-K&*8tWNc_*g=O2n+<5Uwm@p0ri7|*{mo=n$9yk&KrKO8ZA_c?pfY9Du ztw-!1qsBzPpU1G6f+;3$ig1C}3Ijzn*Tm3-=+dN@k2RjPAN<@{p8e9B^NF zu63c;_vl)Ck#$`-S0{o;?c;B>u1|VFE6JCq7B<`|pB^X-ur>EXvzoc?IBDPAY2A;# z12<(|nSMX(CH0tVPKE8oBVKtMJ2dl5`)}d96{d@px7V(=x1aW6Grq2hPT0NOcAem{ zgiP59)@`~I5Bk}`neLNW@7Kv(JkMpl#cE(bUkw%?^QxS2J}wT2?AguWdg>LTT=S*T zd!5nCK5Ev|^|D)=Ek3v$>l7hMaR#{2!lN?+Faj;W+9uGZgkv#K%r(C) z{&r%@SeSB)A+SOw05hY3;3flxN+dAh!ese{gOxM+(l4c``)TQqW@yQzTW^&<+82b# zy~`TAi}}Ym+UZsnO8Bifk=*m}vMP=I%Fl0=zIZJh)E-d2 zbtLHb+o1k)%Au_Sah-GJ?vgLftP+v}q67rkLJ9;!#>++(5*`^-LxsQox%^5i1ex2> zDgRW+zqwdj%;)|Zn{=mo<>jY>3@`%H{nMzNoC>1$$ZzxR&x4g8+EZM6T@g%ymB!2M zeGG(5HAtA0BGt^|B>#TBc-rm#r1I)?OteNSQ_7n@^WGA~0H&A)Os5&;1}mcIaHd<2 z)t+(px|Q*%7+NVUC4M~xl>keb`kC_7zd>Kebf$GSg6zXPm1P>#>QT{gcWFnJ&4$vM za6Mk_wu6DSU3|6r`*T5{*y&EJMmKcO25mQOci$iDz8=PZV diff --git a/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/variables/variables.data-00000-of-00001 b/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/variables/variables.data-00000-of-00001 index 83f2a2fc81958c3a0916847f3ca7e7db48524b37..ea67db4f4886a56b01b78846fcbe2561cadb7691 100644 GIT binary patch literal 61038 zcmWh!_d^d}7p`b%XlZH@Nf|AbKKEP=v?Q`6BO=+pp$HA7lD3M{PE-^b)#siYN{Avt z8ObWEG73?=y??;{<=%6j^PJ~-j?KcAe6hhj`sn3*2(Y@r)iX1AZ+#_K+HAyemN!nY zpT<4?wRxjk1Y2Y;#Kp^^ct~{^{Y+XgXzn_C?nfQnyFDAWwBDvGzXsF87O{Mt=U<#C zUcs;3D}~+1i!msrkT$d!@zd=)`Lt`HRC&y8{y=RN&(*%f+fL}S>SL3kBc~Z(JQ_}a zJ{Uz?9uk|wjcfR4XFJ^VTOWG=9YJ05?U)v6%x@MHvNMasXqW#AZcsZGcB#%KFL{l? zs6vI0IZF81wkU{t7LQe$kI0r5Q7Z4EgpYU;`LkgHUzjPyPk2qBZb=LIUfDr5&ioGP zyZ!~dGUidE569Rd-TNMLHZ7r9M%CE8R*ag>8->s09O#(AnUGdDhF%`jhwh`TM1Qm* zUDPw1HjT3g26U@|4W9UE%Nj(d zlkr{)`B5oDi2vIL`sR(~$QpHi&nO+WAFbd~HPyKBpei3%;K-lf@56sDilFV3B^UlN zrs4N~v%8xuxn2)|!hKbqKWDix&0UG-?9}I)?a{cT{4PxFb*9ci8YJ!QDD?e#6oUVZ zgURQkVWfH+o^Cmhw@&up-yd6{ika{|r*ybcXD_q+{2Rn$b!oGM3a39MXkL{yX{gnK zSLz%;BtJl}yDuOw;2VzJcTsqcf5NwwS%MVvf5K2zEjs7vB>H<*9s1=-v;4DT&?}%8 z?(SO3uSu_=u~tc}eXkY2ygUf}riL)NE<^sR%$Wb)<<1X;>T^3w%6*hYL3dFr{Ce4s zOHA+M(~PNf_1*`dBIt&X7CkUO*P31p86Zo#6hXlw0kO`t>^FDS@@`l|`4*@&u(Q~aepBgGc7afVfZ_ab5vuYC*E7{Tk!48}kxB-7X zd2TcHh%(jc+=T~B$I=qi<(;0YL_bBGzEhB)_cfZxU5x-NIdmO2<^9H^D`)a!bE^b( z-SfHc*927h>Pg4XH0SNN{V}I~EcF_HiIq89VvMXc6U2YS8_UkZKS^~8!85?jZYsyi z9`yZdYZhEQ4^>>WXjk=DY@c`!&}0^d&*;U@RzmK4FyoRF6zKB2c|3ZGIhPO}!EY)> z;7w``a;ZSQRQlk^*wH*zz6o1%KV$bEL#$W334T4Q{H~83UA}7&WM@2pW$|+GGpY;v z4_R_|>p>W_GNe}@%W$KCEEMURMHh*A(D9NQyeC?hR@kcWk{8Z6uiS>+J7R?YEt^7z z$$Ep~o^rvgMmwV6U`t=z*~07}+=PWXE$A<%NkdE~@Y?m#u+-oJ%&F}s5vNpXhC?u( zx-Lee?u+r@`~rcCPYjI9FhzcDH1|2?%@6Lkgzy|d3z^ewMfhd-Gp7hTP#H_7Nz&zq zFT==)NwEISbUO9TS8}KG9_%Em$>b|bu|#wg@BZ@~PYg=Zg-a92+nLt1WwbGzA@6ZS zr6KxTOrgJyyV9|$llkAqzhL>g120&H13Vyfs*eaYJgi7>p7ns6>N8oEr54Rp3&gWI zk!;}yQx;x2gSNHTK(GFM`k^vac;85dUOh0JZluPPPAP+#62>&+;7&At-HLNJjOAsE zHE0EM;+EWwnpZU8qvL9PW71YktjdL@GOvV>hW^9j%GIcuodo^+MbJLTf=={u=PHKc zT>Vf3$QDGy3l9UVU$hy6&L*Sp##}HkMsQEa26M$>_)0Z~46;u0^S=!IB|Dxj%3cmu z)BeL+i8##Lzl0j)R0<38Q`psi+1TcC3s#xEft3R*NPy}Lx-2*tODt}{>TTbJh98~j zOX~z|A2@>kd=Thvo3buMd5fwGqi~d;I?A5jdWkiXOA{=%fQriQhF7I5xEyKWTh~Y(Dbhl4k#G#jq6oGtoy0(go(efWaK2+GiwE(H{v8lkN$?q<9p#kMuNaj`~(~n zdy4brzr*�FaG03kk_kaT%79SA{qq^V0gn*-U|@-X}%A0u*0nhLCd|F1yu{ou3Lp?!lm7mV+D(6%41d&rZj(OZw<5 zF$+#lu){^x7unoj?fByHd**wx2^PBPv*9Pgq3e$p_N+*Pn{PhC%zqc4x!3cU{`dO$^R&qN1rPY%P2QY#^GjrH=)%+A6ln)3pR-# z0M9wjVDE%rRiH!O^^GDQQe*JNVt~SLnIKJdP-(d`4A{Ei^xinQt@<5C?vjK_ffG>8 zTZRRu&)|EXyF-7<8wOvBU|ab}G`*pXQ>iwc8ZU}d65g@f`RCAQa2LP(Z5%4vd|}U7tWKP!9iwRI6$o@LB0(0h31`60UQbr&2yiJ*N+1nMP^lP+IBzWLQCwBNawnfd*M z3hzjidLaYLB4U{I;8}R%zZ5HL&x4EJ4j9%V!xD?)u;ce!{I%m6%XdBrHTvUleWD6q z89S0k9=MBqdkP7=V?jC^xA3BtnL-c4G&Fv0hr!t+ti~dVe$mcypu_u+x}#R1 zrHQ@ZV{;}pD1@M7qzDddKh4-GS76h{X>WWUM#|iQ?R6`Jod&-#eYq!^)f^@ohpS-e z4Q0Mzo-6dw3^?}c9w?qp!SnYo;bWbVIQF$8IlFThRk73Ilg4|{ke7KVy!{8IRE_DA z57Xh#775y>rOEI4Eo4dKt#H+gw5o2@;M!w_!h;%$+_4QYULu3__ITsPeZkCePdl1U z$;Y1(VpUx31vzpr9%_FnK*_N8*j*yehD|qywDjRPt+X9{#j+s&K|h;wOOD@slMNN? zqaY`=sYs$Xh;|TjK&cv z{*WD(0H-~DNJMQmc6~FY8+~TOdFCZvKk!DV?l+Cdeh3z(1U!fN)h7k9 zGRY*bB98sm=_cA9<-+*bS4>fNE!FFtMLZ9D!TRtL6G297@(fxogs@Mq95+}8R5v~)dCso(=^FP=lsK_2?ki#YvHIYjJOjK3TTD>AH=X}@Y14i@gm z*-uUA*Tr){{la7(x+noGPf7A_Jz21kokkCzcm}a689Tg8nKs9XLa=BpF}YpLX6UMO zjk>Au>SY$5J-r;;*0nP=DG}bH*n+e5%~9fCnN3{o6X+P%Eh8d+N}j znku~R6AY(Di%?a4Yn+{O5Oy0)#;4o*Sd`>mZ1)alOZBLry|DsjxvzqZ?GkjX_9=2C z(~|Gjy8tJm1uUYP!XC4$uuDD!-y066e(1UIY)N&`Z#Vn z=MGxDkl|hFvLG9|g8R<+0Ppq`fIxXR1}DdZn4cT(P1B@e(YpA%(1qL%yC;w@dj;~j zYBXK<4_XY{j8E^{Vc?A$xN?;(?Dkm7L-`wgQesAx_YVTTO~sL>C1k$eBer6{BSigb z5%gV@=bP{n7=3yo*ta_pjB<*htcOxv)kf@k)q}1fW`Y|#^4L@5blmuS3w-X{ghpbS z#II|yK)J0H1s^Hzm)?p3FyMx{gLva&7?~oa30ChqSn~e2WQUyv-PivJK2CiHy(QTw zZXJV3Z}UOhO^){OTth})kf!NPpULlgmC&MW46%);P`=_jEZjbfmuSn=^eYb`JtG1W zwmH!a!{ack5U^Nk4)jD_g1Nb=m=fg5TwPm1e!&r#nJov;42RxYITh9(oQlnXZ9hv@di9F~Dr!GH81v!8g{)5x+Iza3xKNNka~b+H9mF6dAH@$xp=M}0t6`_bP(4_72)_7DQr*NLzWSCn`rGh zB{W*O1byZkq30cC>QW@bpYQsCtpyq2+<$@_gtp?{AHyLcvJ*Szt>qVP&ES#s^~}hf z;_6z7>3O3hX!xY*kX`?mG>%G#=lOH!F+W0{-IL{`>-M0!j61BewWjV;dOSNcis|%T z!^Ou|a>*<;FiD<{Yjt0+`}g*Mibx-FQ!zAC8zGEo(`PT=ErxY-|Fe-Y89~GP>jjz; zEjH5|P54q*Y5v^BfNv9C#>L-PLaK=|yK+UH%)2N;byR*ahxHfmPhksuORU7bnU-wI z7cY`Ma12KL7s%3Ujxz0cqI7zC06*H0kJ*(eaM5)DmMF&Jo$@5(1}x!gMza-#~bg}T9{$C;=;m;l3EKM5m@QrP`{ zW|$cr0=M7qMw_AlO!fZ=?|)cf%MD2`rjbvkpH-vdgRUSyd`L=-yNt1+X0rX(VGHI^*+ z1M9LcR(e=^5s69bg;(5;Lr=Uazy50%?dorb)a*b!pOnX1OEciB;2ODV+szzrc~rSL z5b6{43qleCaMZK|7^L)y)ZDhBmj^};{l`q0HuVk|FTF)R*c5?5i74G4(F|G>646yY z3p`(UleK=Eg-v%u!TFg8yEU^MmWD;b_!K2-{%aUrpcRUPi7xoGT@AJFPeQekeRwh^ z8aqapvlo7&>F0l;7-Dq_SE(dpsX-8|A2ZrU?WsP$v^|b2v_DBq8V=&TQ-5I6`*q;E zObY5bJn`9}AaFaz>MJcYDYEF-9xo~ek zO{yKR2RGb(OrF$;fmr@~pi@$C?$#nW+VKgOUjGDZyPDZTxe9DLB!z}A6X8(i2HgMV z0_>h2gM$Ot$QHXAT==MgY2-HJr)PV?u)>{vOk(Vm(P7e=(GFq@=dsAZ3iip|6>nP~ zM6K+7Fm~}tT=n!Ent=+H{ZR>nbvKmoh}(`$s1E$GuYvyFOdEIHVlMjZcP0+ake3ehly5Q3=iVk<7`W z1`SuXpi#qH*c#xC`BN+5vG)mKa6~iq>FJqn59)a zR3`eva&>K9bHDOOmNKe+FuV^85OQx_t=C=jU zCZw_Y{XZb~)>T~Z^cn{i?7_dpTbaNejBtBahm%J?i(Z7IgqtBDxiF9U{GmFe@x zXQ#(nUB^Xw1g4aXq_CYDpT8;rH%-*$F^OB? zu=XbiJE}$>Z4ZYA6=O{PS|#u$6ZmiI_iWq}6JhmAZKBe70!~ixCu62DoVhj(cRDmN zvqh3n|09WrJedVX2@W>1(<}w*Uh6BR{chgmulQrB%6iY6DJL?zp|jyS@841 z0Iuz8We(zDptn++S8cwCLF5QOW3(K^I}}Kc^f3}}^ai_bXNX4|?cu*9HEjJ^iny}UUQ+PgYJikWJiv_(wJ=IslwE4yjMg`I;<}aDa85)( ze#KF^c~}>3Y>32tp>b^C;v;zcauiCQo63Jbeu8!FeZY58#$OA&NfIk66-{H@FzQARb3yEn%Y)b z_4YD6PI`|12h?%b>U(fDA`n)Fb=ruaHh*_a2eeexhdh~c&~d#KGD3c%@UjuLDgT4Z zO|!@^=ikEB>DNKZs)0!r?}IIq6Ja_T0C|Pm?83-7^yOj+nmMP3taLcVx*GbiR`IiN z-V$F-$w-FWCuzV^H;_lik}>pvDjA%$31hD5k(v>Ocu}t(-?pq|6N9Uu-EaeHB_9CW zW#YW^gdVsggyXTkK-BP%rR~Nc!qcX)EcecN!QU~r*lS}#nsg<3(WSerG4=rFJB%bQ z%>oE#qw!>F0vz8=cwf8*>L&bvppJ6*u5%i!|0v*>buEHX_gaPLpUCo$8Hw0>W&$6x zW(%G-5a%wLmSh)~5qK%TfShyR*ch#5wAgnH_T^kAxjpu5_;WR0SD_3mOcIF3l{&0l z8OS#Ve+9o{TU;z?L!R}5o!porm?ApTrp-1BRX@zc+l~*wS#2(_tmMD6oVS~-nfDdr2{}Gcddw`V< zR&aQ}28Q|HAg_Pb!VL=}@YaxoWZa8MtFzI6LKPA94Z+<;?o8`l1{^GH!&M8iaqOEU zELoh$RBL`g)fjg?m-q;7eGP;6Z4Ze~*B9tGCxsI4mC?gknQB@w5>s-Mb)BBa-Y5lN z#hyt-Bds4J?(PH2^fEYeyMsjBeG8g`tu`K8au`p1i}@+9Syq7_5&LU^x9@}t%&aL_ zTeO`;?BcK(C)3eSw0T0R1|Ntjfg903A=tDNBpfX0#eh3#*0KQqdyvY?3x{<2WHrn^ z`i2$Hk>=9BhVg8vK0Mf;2ZP^6a-R!NF!Qi3J@BKCX{q?Ko=Lf2EH#POTKvMz_d7|- z*WZv8ph33mAM()i9;5P^LcD*?kjzhw#=d_csF_&H_HMj^HS>SNj*))_L4PNs)x)FE zhf?TqR~nAG{GTHyfcDm@aNW?GR~=HvhH^O?5_b}>EepZaty}R!U4XF1rV@?Mi_`kl zulVs|DSX+zklDOfqdj(ZuALMAyV{qw-_xecSUo1JA9J*1jbbC zg|Tl(aSbj_?UTZV-It2cPbCzkM_W;;lLesvHyT7#yUD^Grb6Me7o2fEJQ)=k2US3_n}81kE$6jq0DP5sR%8` zVT+Q<*}1{Eagi3rc26LmT5hw~ep`vaGZ0QI9wx`w{hY>j*C=3XJ5__!sW0l82@=4WDIFX4mW>eP*^+j5!rwpqZVRt zixQ|3McA}UA4{ef(l_N5P@j7WW>}vFrOu0j&5|_`D_%#$4ll$rj;rbCb>Bha!*BRh zkpz9(o?L8FD5kcnquJ6fftQmzyvVqZRtoP(&e2KK#b6S%9eLD7kQ#(%NIx7Nx0x^8 z8bWI7-Qi_rhVZQTF+6f~s85x|;&FR1Vp*L`Qc~#9< z9$iyi;Z2S zLv5bxfpzIe67jD=nESFAi)F@Qt+FarQ5;VjL*=OYO%=!qpUS+}EFvSaN1*3=Io?xx z5O8ic}f%`fftj%{xjQwgut8GvDy}cmW2C zxrJjk^f37al`u{=N$~sWHS#2H0B)yEqxT+6XV<--l3fZKSnwzV*QVR^hy$g>_~%Ky z@h2F~rp`h6FAv#3=xnT;AP+V(Z)0P;4*OQ*!%UtHqRqz|&{b9kgDXmK^!IV_Ns#78 zT#He%tdV6+GlUSonHadt3QDy!;OnkX@Ys1|C@bD-_~lhINV0`FCQgcXWZcK zqX@X;o~_+U zFpp=D<7}~aIyv%m0(Mxr3THWqV((Da;nbKKL1@(g3wXb-YJ26$=|1~9P}md7zgiq& zt=p2x#?N1|@7|DSuU`N`_6FocVIWR-Cb;gpfQy`1g+5sb`4yhRi55|iY&V9L6{{|9peEU{xz9j)a zn|+CyY#+Y0cn&7vI^eMS7DzvjV@IkN6I~KZ(haU)p zK3a`phthCn{zAdQS0X%h`7IJuGYV_t_7KlkQMg>T(dOYCGgzp63zFZc@Fu5zIAi3& zugq+~M?opXXO{@Kh)zS_kRe{wEFnyetrcbsK7>aMFInDS01- z@fQ+M=InPk7hX%UoGj_jE$Xyue-hj;l%iGTDt!1FZ49xL$9?w-QDWXKn%R9sxJFx^ zCgN!l70JC>iv?*&VucX;vZL-aCbl z|2!8BrBaDY^-kjER|{9IRJhHlpQyTe6GRP^k@agk@!D$#$aEXeFAU|)BJJV@PdiqF zIrBl4rY&TQ{CUWHJQtYzkguJjhjEtLsP#{t8!oy`)?U1ak+0iGh2Cy_x4CRccld;o z-sZSEV;^xp`<-=bi|~SwS?pAFGiE)PfboU~a9%zc>TJT$rR)x*oj6WnCo0e>zn`E- zj3dvzbeuRZxC7D?jOCV^(U3j?02sFNB!DZLeu>M8Vkk>X06rV38hMGsA(_tDU zu}G%Zv4SJ36OR)plYSZ1S$eG6lUAOsV_~i?&32%(Pkpp9+-^Y zcQU}+sUAyrq{5n~jX2fF4?@bbA1ZO2}tcl}$aw%5S< z&;H`$)5lrW@A>ey_Cb}N)kt>vu|IaHWaFsnW>i;u36p;g91KVlZgu|& zBs_}z3hW|(wn;+vqzb%evKt-q8_4H7Cj4P%A?h3)OXdt)f&n4NAacPejJ7Dp2l=k} z`&=cYoD;(ykOPO}B0Ox|0^wb?crwZwen-D2E$6R5Nkkne%smdHWA$i$dZF;h!xb?4 z(;Rl(NLg?q^aR#LPJyE_k)W9DW;C|g_Qnh#=el1GI4Lz4&-9B}CbQ5Q* zKB(X$-!E8j(;SWO9i#kQDnpNA@H2@55e;v6+HXfV{RCqckDv+O^>BJg3rhE-B0L^W zrwqRfr;|tV)5iYbcQ77D#>B$&^n)zS`wP3G{~EsexH6Kg3^Ro9hpt1BdoHrX`R&ta z;&mmOUg9$71(Y-z$UavI5lO_A`ZjS;z zpu#aQycLo#2i*?^;;u7$ndGqu!5X(LT&GQ_)r&Slh1yi6UX-q|pMrI{`pfU4)2{b$gPq2Fn8l7R%LLcHNcf@s9{aAPG2bD-L}HsQw0IX{>`xn9aJ7is zzb?w19xL*SP=sfR51<7Pv&c+8#I-K?Yo-W^*V+Ta`+hNxiDy7#q7$t9F^X@?6VTQdfeC=5+9=f?i*r{Ajg#M54_$WzK)yO3EhT~zP{Bg3V%mZ1)O6+xr zfm4Ty1rrxf=i;9i@QYUYLbsa$vP(>C`lTk~!9(9M=-WjYcc=xDOGUYA{xNXcc8pyZ z^cEVAGKcdW&tYF#9{YB&l3ea@N2c@+-gbQvtlA`l0V<<$$>w#ab$JB2Z95LK?q!lY zu0EJ?yhK>&R05Bi#`CKsZ{YAF2P{yYj0?KQu=88fpl9|R?3T@9(uam~Tlwj@%Iy;? zK0Hq7*&GB96e=e_`(?b6#4o4#$qp#E#h+c(qG` zI=wgoA!qNSP4_8qzg7#Q|K#D5S{=SaCLOix@?m6iI;gE40O^_;v{yEpR2-2&=Oo=R(BU@D-l>QqmfAtv z?nvt9zZM0mN*LB$fuZ^{Apc_~+IIcN4t6|&PE`Z^tu+lLA5Mb{E`G?5p2l$|YVBg!`Q#N?EHfl2E*sc}qA+-3qfH|Y#XzxR94hY} zhHKVX(3vle;rUK+q0Yw1Xu86bl-9h3VSdAK^+rRe+3gIGV1tFyFPX*qR*<>fj7D23 z0N*SHC;#)z;Z-YhPYA@}#`}dY9HJr0Bn9Fn3#WfRp9%?9!5F6}N7gW z$>N1rLaz|5ZxJX%=0kkdV!S`AhJCJ@$@43;dEj(2Uaq@e(A+l?=5}*7c*GUIHK@?Q zzh3y?j4l!$UIz<0hx~1##BIG|@$<`}e*5b&vFO-C_PHq11Wkhb=M1o zR5U(xUreIkuZ5hsCz!f%A67@2k!hbkLW!*nc>JfsuLei3h-YK5t0b9y9=}dF=2jZM zah=7Fd*5gG%MOCELj^d$tA#J~ETLn`P0Y`%#vNHZAtZ7%xMSl zNexgJ^qOgiK4vz-;mlVd9*&x163w5%@L)j^#{AKyT_2?AzNBT`$2FE6x#@zRM{TZZ z4$#2d&c$G@bBEmj-iVpyVQkK{j~JAuh2v`OkliD+srS1aa3^`NeZMF_uvH17JWha7 zK`Xe3PNSwtd7yaZAYR#10qa*!!xj5pVToH6neg;FS+95nkNPWMfx=L|^n}B6w?;B~ z_Do`4+zrcX7*iY!M_B(4cTbIi!Yzb8C|xa36`4)9Cwd9Ak0+w!uE#ih!$|lR*UKzV z6|!r`55QcDY$(kA2!C?M)5#b9!v$nCy)i=@-+Y@*TYDa(NAnJ%*E$bW(hlR7G)=y6 zMloJ{8V&0H0njHs6+*>huyM$Ln=~gI)kLqLV1+yOSJ=SgRtt#!V9q+P?t*nX7QBG; zqTeS)Hl_C?&Wqa$-}>r6xj6;iJ6lX2u=NLL&l6abGZ(zaXc3d+Ml4EIhHq+;<@E_w zENop1S@e<##A}V=z?D>@5gLro6~b&Vlz+|1Dn4Z@Q1~ziPF325zJSr88ufJ;Mz7vc)DN_u~_N`uQV#L?|K_# zCh1c{uj5r|(rJRLxAmD^kF(&a!V)_B3CE9f-jk29vq9p>0M1PmV*8O=*gIYZ!S5DM zl=OmAojD{RBLVc%RB)>8IoKlW$z?qE(L>!4{N|%%5>k{7u|M?SQJX$~F&IMB-U6B1M zf;s7?aPU_#ykVs%@_i*;y1SW7=)KF<)D{67^+vEKVh@^?-y(UcQDkGHF<*FSm+++g z74(hW4knH|@ZcWBU2YSpnNuc<4oV~5i9NV>dlG)Aj(}@&XTa`(H`BLr#A3Hta_o#W zY`>(0X+v6lP22rOp3*tSq6OYxe|@43S}oF)bPfUmDse+0oQoV zpt%|AG0#X8JfegktG*keFLkm%W2-<_sDg1~=Ctdfy+CU7E!erO9!HDs$4A*_=q)jV zd&lKs#KUu75~511#G;@gP=#;*(LUsN`e3B$2@otQ!k^3HSTu36vAHls*!H|0&CZDM z!_PYKtGXl_9JZnla}7~p*hoJ0awOP~48!=)U+ntMM^N`%Ye+j+z-^l)sMkOQx;pw} zT<`=~V={%Fe6j`VUY5ghsam$;oh@q|n$w?!C7@(yHLM9TrQ(W?kZk)@_-5$e^p6UM z%!v*}_D~~emDPf5i3p!?ei0OF_=0Zc83>3!hSQ4k@aOgOtoYa=G_0}2lIcp&{AUr? z-|B(?8YW=Tjzp|bc#o69Lx%7-AJYae0y=k~U}`5MJ==yhhs)MH~&2usJ3TTi>9Ex2PVVf%PQahASo_^wfXh z;G$iF&qa)(>yjDuSz-YG9wLx%TO2-XhYPCywZXckyMn8q?Xc^{I<~j%KhnB)6cO9; z67B|M33HVvV2UOO#iId4wbq~Yx!GV`+I+m~J04f;KLIP(9<%vde-8F6{J}EE8sXQS z=TLr=1dpF+02d}j;NFh^KywJw)0f7RXSOChY2{9cnRJ?|x#Z)dNp>i^YZZvy6+ljz zJNxmg0E1h-(BJMbR9zj~BN6?MqZjnSyvqAzZ^t9Z6TgVtCrEMGKwTU^`3@+>UVtNNP38*ukaM__g zV4LWGD>Yhic}_Zv*l&mBrz_BMC~beD!JBJ5l;efz1t?izOV6ltsMoxV28}{o|0oaj zlvaS?>2}mxD8qN}^5rxC29sqWd-2BNQdpRs5BD|C!Ck}W*!!ZN%u1`mDOtN=O>+_s z7iPe?v1>s$BN|%O=dp$3Z{w=FlBhN=7viUepx@V{5cIPaRMp3$?Dl0a%X)#(@#R!% zwfP&)_IV6PW{G0hL`B;4b2R%o4_dOG(L%X#?F1%PiLy$ zfd0VQ{G4a6Ktpl~%{V^=pr;1<660{w(oImh-@t!$InltPSWsqhnN(i9#soK$Nh@c$V}=300`+OJAd z`S1b^yMBlS4?l}Kn-p=Pm5{90cmsD@caUY`pU5=(H{dbkOAT+^#0*Bekq1c%LaU7# z!Y%tb^gMTAEh;AjDax~nvV{tspFbNHm5k?JF?ARdV+;Ai9buZTC;$Gr8ePXA`7m6X zj#C;-N37~0fvJBnWxE`EUStR4l{b7#D8elw6*%sRC9Lnw1vAqo_Mv<@tC>FnGG`Qn z+AL&FGI6Z(p*v0#`a{0bZsr<&8z(FoK^>hF*tA*1!XU{>i3VQSiQ(2iL9Sj4rb{%TtMMV2|7bIU)Kez2R0@{< zZW+oad?dSbZsP7KF*th9FX+9MfZyZYsLZL0X#MUOejMVrZ;M1|*THH~dwv62N5+Hk z@Ub-6w~c(;ABM9pN3rV;$&h6-1Ag4tjGp=vX#Shi!q~e-d^`dn1WOBxKGhOVJ3wcAnes$MNGJm}f zbZd08hDiw!-|-OoXfgJ+=d!@_86fzO0%Jop>48VixK1G(GYogb42pF5tiXPwb75VaYrv+m<#T{o&B ziXJ-{~Xdv{!S6)Nt==|WL!Ni>G+JJ!fr#P?mwFq1}4HuPN`7B zyrJXGQ#hD52QC%eL6L|Dq`J|8e;>$!F0r$q)@#A$nA(%`zt=prD}~jqvh@8w z3E}%!mjp-nIjC4F3J+SZupKYY;>-$ld}?pZx~vLV^@Xo&{Shnb^=>NfTQY}uxLD%Z z^~NBbu^yJ??Es${&6qN8IwTwtg?5i(7Ibwro|_rRW$!iTQ_C%C{MgO8kXUdW zVlvzC{uV{l?vF(4@_%qGPMbwvP~%5G?LfYg2o6cirA@tckTY0_3K`?+?~$H(?rSBq zF0LUKTZS|~J6nOm4IQjivE|0?->_-xAtF`yl|}Zd;t97w=A_&T8@8|E4KvrEi`Fun zJ5LS&#vFyxhg{h8zyr3&4T6)k5{>9F=O>Z_$kLUX*b$!vwnfrB-bWJK%={p4NFP#Y zeZ_1vH=(7YJUToMVC7RBiKOB;FkD{6+->b}*5$WEGHw;Tx;P0w^!cEfQHpS`jWkr)WZfxUd7`r z_b@D(z-(nn7?nGojy_Wf`(zTp*KHm#yBvZKO#Xwlim#w$ObR<%@(q11lrc|_Vd(Ut zlPH`!hZnwk;D0Ig!rMdH%s{u%aJO$KZU2x4niH-wm(z7*#H3v?5A}J_ZXY7QAc{>K zc?ael>aX%B3MKoFE8~f4-dz2<1g@7p2Chp|$)nGfT+kkZZC_5}hh47(4u``)PVFjQ zzW)L=V=VZ=Qv%R>YXrf&{NcvVF_1q!lKpHR%WapIqCxB#T$OVQHf$83kIir5hJE#@ zR$WHEO`U`Rrv|XZ>l{q4Tgi=5RbcG=X!z?Jju}=4?DW#Zu=|1}Ki&Nuzm~ll+CjXE z1KX^*6qrESn<$u-wpKXu_YC2>Gty+u4O!S98o_dudYIMw7^o$i(W3H(AZbSpYOM+7 z>!Z^kDBu=Mx3qw!+POkY@oC_@CJvN>WwEBz41CuYV@v*~s`IuAV4zq%lo@`Ck1uYa zD~6AQo6agcxyFtT1emdJNB82SZ+``Si8Dy&-IXxv!!kT#Ys>mdhVm-gw5iK2RV-+Z zBONn>Va~qaq;yO!+-u7cCR;3oH-?I|Hb$IFxVj0PUcY8RE-@^lCK<<ZcrjI>DpW zZV;x~3VCk;YW7@UOS?6h>n?Ms+7|$aylu$na|V1waX4Oc?G`M0N{Hi7P5YKtVu0&# z`ahDcJgTPeeHTiS2Bk?fm(p~{v-dgomS{3%3K7W=mHHSmCDNo)X;LIb0|}|5?%DgC zdm}>LcdW+tP2D{ga zuU&s4KF=!2bnk^wob5(pZ?k++yEl~CEH#t4rwU~ewMDY=qq+0!oL8c}LaUlBucB)v zvS(!PEl)^?cFq&$RyK>@7m%FV4n0xr@l21i7q4}2<=A1MwlvXM!jrpms@hlx#% zwu)bG?vfRh|CW`jGqR(di^V}Fa>NH;ua>o$oR`fT)y8f7Y$WTQAr&WfNyQ;kBczsD z^F(_aF`G78i$4@%&Dzw@6LSt+6@{5dWe+>eYL>q|FCvDLq;BhCWY4*m>0$ z*^dNoS?8OhGO$I*(d^@68E?5#&F<=U=_S1oap|o`;t~CR>E4)$vN?eTVkR(M_Hn~~ zaqJ95*%HE8*32!EktHW&2BF2W;AZuj`86ssKgBz;6O#yOVO*c=rtuO=ymQ*kZ^t3=vs@qX z@h7H^jLtoAfJeD_zU6iC(a~BtLtc;Q((7%q9?fjo%m0RCy4jVZnn*0GPZdh5qbG>7 zt6N0jQ5H2jC+12Iuh}Xy@75K&?RIl4+0i2YbGTQ$HRq}Ls?JWCew&ljE_Og>J=)*k zU~8%PTyBxr^KzFoaK};Uf5$J#x*|sL!skTMOU+o(ELnp1!1yJyqJwGhH30N1u~g zPA=^gFicpX7O>nbh&GcY=mD!MdzxE`=#T>S${vujKL03((xnG zf-fs&Hx~fMS?N*Y!K|_3k6e=)ewMYkrBA% zW)Ztk>Y7{(cEEEk=9&QX*Xpds(5WK z5`58@Ue~Y?MdW5sNiCYvM|$?`w(Ll!2Zs(i(a$5DwBrsstu8+3lr=BPDQrQM)9Lw9 zPVYv|b=3cOIKt`0s2@6Nk}n6HIwB7`T?;?xM7l*ewT{~D(RawENGFZa=WF8+I$4h1 zZyEJ9OAb2ynj7WB8TD_Djy9uf<&Rq9frCyLqDSXhnu_jxHIeT6XC^gMG!?xXG7-gk znu$`NnMfhcOq$teCOTAPE*dJ{PsOTOh)SWU$alJ#RPLCus9aTz)VVxGZ*1gW}?nTX3~)5W+EFO6VaP1CL)_% zCz)0H)1}|Onu{!7Pmy-bF&7>BWiGn)^A@%2P64&wXLRmwV^Q2wQ&IaybCK3RU1`TX zbCK)&=~CB$Y0|O)3sFZ^5k7s#Tx2sw#O_Y)rdnT_i|!j+i0%n2L?+KCNL5Guu&3rC z(u85k0!9gi_2#0%k!Du@*f^=O-2Y$OCdZJK84UY;_Eomx_5)IR&OY{~e++fOH(vBF z@&fB>ZA0xIa%YNnJR)7}+n}^vg)HrcfAxNywf~XZY>?V(RabIe7f10#L@wCN-mu!3fLM;FZR1IR4iRrtnQOc}c9w z*qmR*46YqQQyz|CrYrk1)19nHOK)SwAe4nJU1O<#*Y=UCsSY9EQ%XiJ*-X~>pC*TH zK4+$F|3zx%zl078i%@K-ocMMBav2@IVpMCZOMM3iHhBK`3Bv}pQ( zyr}QF26c_2EAn1`Rw&~Pq>xZe7%^}d0)vk`EDWaKD+}DX^$tS?Ji{u-Q$>)OFx)zpN})bmg2rXEoMzqfT$#@Qo7Np zhqZQ35*<4;MwFIvL$q{Xlqk>bxm4@-2WnvBTc&c-H~9Km0ag08huUhOz!vUWMy+=| z&)jQNmZjZh*`q35Qumh=qz-=hBG;eEjE_Z6=mSqm?zTXbg z)CKoMR}RLoo_fjDhQ4U3>i`1-rvJu%&W>!u$~Ej&+id1YUjbz^<0=#Q_6JmCQkaSE zR#*dOk!OlNlHt4k*|kZV@Gp@j8{(19ZunS9R^$&b9bp#CgPS_c{Y-17HgABPX?vQ{ zd3zbzaje+T?~pN}CsE#cW2k@Uzmap42bi^T5157p)nu{lVkkVRP9D5GiM?_%ncQFU zo^^Q*8CNG0X6=)itWxu2rc7}xZr_pvl->2H5jL5P6+R+MHY-u$)jiC+z+dDcHiLN` zZ^!yu?;%r{YBGDKs7Nn4bukO%E<^2O*6dmP32f~06eetb5Czf?u&FM4aqYtrxV-f& zV>0g#1hJn`ri&xh98*CqG}_PjT6r)U6NJnKY6cXJEP?y}=7Ij#Q<$H)khwZdi!!~a z0Jo~>k+#=L$&0fmFdhAO8N72M>3hE!D;+*c)+*VK?(6Tc>tAxRy8D*+OBky9WT=q_4bvBfaD#iKh{Om+LBMHUtA)+lYO0`vyL%! zPtH-S8HcjHGJ`7qAt$Z4Q9u$a|53HcyQM!a{${tWJHg5|IddLS(Ch(fvG3uL9 z0yVpT9GlhS#5TmAV$O7N7)OH?q!ca>6Z_(sUY#VSdYLY@apP(7a9kwy=1eh}=`fwK zpr^BP>{;mdA+9?1=4&#x<2d#9@Bw%y>@=JnwVBPAWHakd0e1b?9EM-Nn7w^mpV~Xk ziZUx&A$aStNYHXFfS0-UK3_%sKHtZ$Nm#kcM$nmP#$Wd{jH@?g8}HqTJPyhl;oMu} zC}_rh0;h$Uyd{i~yTx*~P&eZbC*i|%q1&T5Jh7QP)Tvke6YMHZo5ljZ>s>?Mm_G`9dEZ!`vhh~V-P!`qlPgZVW#<_TW5Er*BhByg1^Z+U^8KR6+4iaFPPCUaEb zLVkk%Vcx`t;oP%=VqVw154_$JDf~67?{eS&cHlSU`*L?ovg2B^MS|OsE`CmJC1>G@ zWWl|?34$VvpZw+eg@Q|ut$CW8tocSToJTe0aO|wM@%z6V76=k*1l0kdJe4Pgf{-)! z1szldKljX69@YGeXCOa;d!|vFPir0){!01HiwJjj;MIQP=0CUOt)Dtyu*NuD@E4rs z|3o%|+6GqvxN(8|zBrkqQ#8c}bXxJ^-rnQBn0J{M>Uf^>&Q;zaCX~yWoVu8^+Pu^@ zc`a$LOeypF7Uc5om$}#Tg4ePTwpZ%}#e#LZh z%>R9LSajnncg*oCoJnsCxT&^r9Kq484j2CJbtri}hP!okfiU5%3n5)_OfaZ1D1@eu z1yAO#Bz7FB5q4)ECcb-~5IjBmf_Fbmz}@zrszA8-l;De`p10ZS3cvjBFP`zGF5*Y2 zqd>4el}IjZ6%3iH6YsaX^1c?f3y)Q62sZL1f|mCA0yTQIFuiL(zslN9_$qx1vE7#C zAMu?oDBXFVe=@3x<9non;~BD)KRlGoSJjl~ZT_w!i2pc;cX-ovPW!?xZc1i^1LyDn zx74$gcX0P{{vWR^4!&B7{A*Xf^Ov3!@Dg{Q<5e%y=Dztn#7SP}$^EvmSn%NP1l|i1 zOZ$T7hJ56>hC7(>iB}QTz`Liy@U)j-=Do@d=e=6Jf>*XxnX}gGC+Df*2VSGv4=%gn zl%O|!HdpJfI=8OVj33+nj6!fW4M%; z0q_3!f4uKoKH6K2jOUJPMnV)^!gbz-w>Aodw&^YoYb2`_)r z6ezSr3e$Xd@D^>F$cyaI7K};R#V`15z(0MA6+~9A;vG}05va7431}+)%e#xEaq72U$CF0oy9YSN&=7PTAXDMtT|88bNP!N z%JKSj=kn{PP2kP?I2vC~RQaug_WUe&8P`)&fYj?SDoe998yL^y!Gzj#&_}b2=6|t*I#Vtoi{~t8_){p^ z?PUllt9Qim&j!fusF1jlAP<$ex!B@53!V*z;XTw@;ssKLEoWZhdF`vwKQ}cZ!ny&R z>o^0y#f&9&{ll@aNg_~c=_fuk%fXf9g@kbOYE(Zf6GsS}p})%wdtjvN9u%K+NmZ4A44HstLUQ{l|SheX5#cQ|#!Tqt=V#HRhba8BW0+AHxRP?}`~ z8`}J_Tv)0wSDZ{+paQhne+$#ra2%zc+zo#;{zVz}_rMS@9ZPsN^!+bW>3bI>_}d$0 zXgBFJSl|+YA0Kii4fRve^$$&gTV9cLd}$0QT9Aq_C@VAT>(t=w^?6vgH5Ce1F2eO8 z*}^}fC)i@iOuW|H6?e5A!JD`L!O!G%aoqZR@Mwq!)I%F#!kPQ)qNOF zY8(JtHIE|m{6sYG1fCQMuB4X$Z-OU_Q1fH(Wa zqNTo}AgIZWP`n*RH$7O0P9!PO%bYDG$CTC)L-Z+u(~J{HWm!1kq+m;LnqDmImas&| zFc0`fzoIKtH_}r)#-i6lEAZ96+w>(`Kx?jCgueR$qWMJ>eWu5i=sX;S{vEwRHy(OM z=Zug<_%3hQxJg16ADRjew)~(wOCG<*qhSd zQ;lf%r;|b4lI23WIEUD;EkZlCID+`<6nb)21g*5n0BX|##7ARi7-_01_;0%$5KenX zDA!&AT?XMirD^HJfs8+3`(-Kc%L_%)2{9n>R5!i-)h4dXC8Ag+Kag_n@$$}yAKkwMsu~b-FQQG82;;Y73*^cpl$STsJ>wplM%tYnuL$rj??Q- zN1(FrTJVZRy0Aa%EExOvJaF0CPhVa#LcgE>kcgXhS8{7|I;fZ9!+QlU3Firq3B~WT zkazW8qBO4(?QBs+JAb~S)BnZ-HmD5zmo9)Ra{JNyU~80;_Z4h1;9_f^OF~aoN7&PC z1eAUVV9~31x^_~6hWMJy>Hh-a$ll6@JKAVE?DIzIeHP5w6Mf#ey|+$@hm zAD;sA|75_bt902Pjo3{o8-UE04B-0z=1R;;+Ee6IFQ;zOnlUb zs<+ppwo}T~=}Q9kQ-nJ+baMiAb@CPR z_06PtbQH5&&Y2|5LYcTN>sW5c3??R0gRR(8&(!$5CwJR+KrgIH4eM@U;tnmqi+zqU z;~h&#GQ*D)z3gBvXqZyQ1~H_IS|eHT(E<)!`-=>_4Uykb9W3{33szpT40`j9z>&8+ z@G(mbKCK#0Ywf!rX?tD)B5rO2-JCtb=BOK}{zoc~2)4vW1oEIf@FHlx)K088#zme3 zNfJ|a6&w_3BUCTd1Pij=pz~51+&owWy}P4P9^sZKedu*J88N{PfU_f+ zKC3{ZrpyAkP$?6HNb2YS2N8PQnT+lE&++|0H<;_ALEJS>mq@K!&}HR#I%~Ee5OYJJ z2Xzkj6paCH>c8oVn~(6qk2L6A$;FqZS0KseeMD@pGPbK4&+J?L5-lPXki!Fry^0;M zxxfy#J8Hm|_3OZMDUEu|)v#aM0vu7d1upfd27|O7zPiqeEc7fyt8TXA_X+=C$olu_ zR!tI2(8;IwTt9_su6_l%Q$C`=X?^Gi4ua z{zy#TntD&RY^ z+YH)$Yckq%aFe8U@=c^0Pz?p5UbrD(F1kO%9|kn0;G9Qi8E@~2a5=O^UekL~sfic% z{malVd1YwYmLqUi`wP5#j5++_@*hkJF=5`k$pUfZvl*SOhS*x=60*9aijTO=XAUY0 zv7B`kz06a{F~tN=VG_zTy~h@xG7!lbTT+2j#z{ju1K6f? zU1_Z6C3c|_hx#J7lg;oL?O_#s67TnxW1}8jVBbHBp;}Jx*lfo%N}bdc?T^%=x+~;G zoXi{4>59kf9?#26!q8h*<&_~-Xmb)iYCc8TX+*MJk0aUo$TsRkp_X)Y?RIt`uYp}2 z{|N3`dk1m?QqYO$RyuLQC9L;q1JkW{7LsxPcyq&XdKz1RykFm;H{o2#vkZ4=nQ{x0 zkIsPkyeg#KGm%-c?E$!A`5D{j%tvv13sB)@c}!2UXU2bV$4&C3AUf~?dRX}wd+DY^ zu>DH)t9BKj6qAPXoK3OjzrDCsEsaJN5yXEEiNJis93Edc0qJ}mKZ;GG2)_*&1)f|$ z4=fHA7^TVRC(n4oAAvg9;zZ#nj#q^TgC>C5WJOfgX^V?u^U!m(_24~sDOx?Ej0y2c zVQHuz8aLA#f7mt&b!^wfu8HR0Zt+7ToKr%4cf1T_e%G;tJWS|@Cll0M1<30^3$33e zL2gwvp81oezm>Ps1AVn%xnCCHbfXj<(apemXIRKGS0qjX_lOb6UH6B|JrT{V_ znwZiyjKT*4vFvp`{M1nfs;@tTqOl$D;2}k9JM9kIcYYj^u{@Xvo%aEJ`1~6xt&jj0 zqkejhZ7{yKDjZZy*J9>hg19>5A-0)d2!ALQz}Cg*am48W!nEBL6nci zk4t`N0b!mZJZ<_Ac0SS|w>^l%{_9lH)sPC|ZGkQfI)9xg(LaD@8(zj6{`SMjjryqb z4FJIprjdz3L+D!HE_i3bFTnlrt(pqW%1^DNtoanwV+bNLrM_hc?}=)M{`Sw_*K;&{^6>o|V) zHwWAK>M=JiSwm_0P7W!(1OLaL|}v`j>BPsnO=$M0Gg zGzBu(b#^1?Qa+xDwBd~9NGO{1|lqr(JUWT`rexgVO;e%C|9A6Va-vZ)awfB zsx89&W$D$i>(61|`R2qrkrkr76VU4eS8>^@SM&*=I(k6M7}po8W0LTM8k1EKS;vKK zggWwO>XDv@1qfYz57g)U6S_V=NIb5!Cz47MvC7w6`a^aT-Qs)<`HY`|w1*WWb@h74 zAb4!`SXnSi)0qu!YrX<2Pi4`!wa*i0rnI4L1w0hD@fLmgue~tVWjvm`dpRN#282E{ z=Hc`&kC0hr14;{$r#JMD#Zx|x1qz(SVBNOg%rliB%KLdR3D$bC58x1^-@b)?_V)p^ zP4PqCnKN_tlnT4%hCFq>DwS&VD88QYvan=C#27WbMPQ(^EXGjVSVb7%8(YKE3C``?de47m!j-p_YeDSdW? zDL@&GfBO-1Yd;`Vw#Q)En<`@9_XD&g(S}~ruZ1|J->Xe0%s@9fmkWQ4B+>T=y+D&m z7a(S8<3&4`(4gupc8Pu=RNa3VfmNl1o!}zUI3z}|AN(e&jKc817$fYd*iFYgrV)KN z4FS>>_`aS8{+s0pq0K_*n)Hu0pLQF?U7UfRg?>Q+YJK4K#u^l>vlLY@V-R1$uHRO2>GP?q)d_3MX;lIj?w{@jjsJQ|8AEqk>4R5p=d zb{B19(ok`UF2G7>(6LvR$j<5#q0(r9=C>q5PHilVSuh?=+|mn_*8Ae`X&#Jx!av}> z{5~DKEdlMlt%2>1$J2Mjny9{U5!ASK20xlx4^BV+4Ve2KINLo51S!44PRea)b(tD6 za%)AQ7uA_lZ~vj0Q}XHc{++nsMKNykISF!o2Z;?TdB~>PiLN|8AE~-{v9BK$P`>Z0 z$h&Gs+27L42=FJ+ElYuBVQ8;p(cNA(1zsKkMN%TcC6j>{Nv%~a}P z#c|5`<0-PtC4mYHnopfLvyRQ|HDeU}ce1*M=SknI&airtF=aG7j-_MAv36SmsaJna zvUHe5VBXgjoWoD0>!6Cv5~7j68*J7Yx8d+RQVeV{S@)*$s(Yi znuM6yiMai37~JEi2qIVR2j4#VBjs!3@WRvY;DYjGxV5_y*{Apb*HbmPZq+V)%B32< z3;T!WG`s=ZofDCJXR~C<@e`6My-C2UUk1{x<{^_LS3Jw25dU)eKxqAQ#8l`ou#*j= zfcIC4?gUFLNl-@%$(jKClJ^`ZvH>juzt)SqWah?Iuph z%$RB~5qun+ji04B;>^y8!1?8Jyl3@3#=B}gF3T9E|HLAEbms-UCuAW^Su(06*e0ZR z-B7~#=xm(PvyQo=)JFcSpFpyHPncLH!aL~$K*9XL;C>MlTYb%iWK7!7$9;wUn@R5zmxNo}xbX#$( zTKg+Pnb8n%E?1&M>!MJ=TW{n&YdV(xHbyJHt&oJB*+Ms7N+Cv@YT(Li6(Cl(6s+Lf7{{K*!>{IDF|$e59A5FD{Hme;(eW%hj5M9cxvv z$s1KH=`evZqb%^@%zT*Ua~^CfzkxnxG~ng6DqzmMTBP501}u9*!1u<}K((PYA)?E0 zX+H=5(ow{fAt~6eM;+@|o4|mj*6?{~I$Pu3N%?JlNB+*jZ28Y^?02~~c1-jNwnz0E zx#hMwOI`lQtjt)6Ppqh7M%)gwC9CICE=?bpf#Y1NOZ5ZgBE3Tj#$Kl~N)A(H2adA` zHl1Up>0M^O=k}4)d*bjsVmCGW<{Z{H$evx)kxF&1zR#L`h3uIZ*=%C-4m{H)7N%W4 zNnD=g1%GBl!O=DYIDT&dF5h(u{dr}Dt9L%3Ww$N}>NCgUhZ|CmTiATmm9GX}wHu&H zk_SCWz|Ktt*__uxR7o3-tecB3gGSsmFctfhG~)|T zAA-XvE1=DyV2lpGLu+il6IK@c1Qs1rq4)Y@;P>UJc&X7jAUd%T9r64nOny=-Fk(*% z>a05iEw^8y%t!j*zb{3AxAFpbr*(?n>9dM1D@j5(-W>oBoOO}ys|wn2`zpe1&O+p@ zwF>pOP6nTryhK+er$9>bdBnt|)6%qspw{m!emuz;eZIAh=n|!aNox;4kzphBTI!Cw z%8P{!Y^iXD`%UoVdNi^5#4>d2NCaMg@gn>aFbO$ZbP+Ft9>9^ZXDBOaEF3!80GO(G z^mUVQ@Rf}Uxp{B^B%eqox~%(wk-iyR(O`iS4wHE02?07bw*l-!lVDAA3?^EC!sqXb z!IxECSi!P@8N9E+Y~8R3F6GNJdweJ3lg_HlEXQJ6xAGnu($Z&wACDyy;+yJ~mpO_HFDk)3 zrnl(_F8{#KMg#C7I0r?aT!qH08V9ZAu3?pb`taxG9E@Yai4Id2_@T_4ari|eF6%&C z?b-{sn+?F_N{h*5O7&Rb3QKH@2?o5>`LJG25nS}EL&Dh4_&|jN>F$+<(wxePWM?tS z`w@UU+xm+_0B z@`)Ijd1@uBv0lVn-&%ry)Ey&3vsN&{Tjr2g|E(qKg>~rgTMn7|XEL#4c0KVPheE9! zEw~}!3C5zIu;5t+BN}4hsuL;rZM!?O)onRrWxSDWsSx2HS0i{mFqDpY;*Tr#YA{Vq z2`JtR$Xl+N%&RA7NH1w9^wtrR26tSUv?bbj{I6c{=+S<-=oN>IC{iK2m(68%Zr6wW z368jIqz*)fw-ULkIyk=f94_8^7mgh%9?fUf$Q$||q&jal9{0-@=0|KI)>mR+v0V!e z$ZsNJjGSTTnqg#E7>&DK4np1DWlZ&cXYBTEA`@`m6Z$M|h40i~qxu~#*kP1oFN>B3 z>f|_h@2MrUG5m!>%MpmYKLF*#l4HTYMlW5yh?z>nybP#+i2iXzZKOt z_k*`;=g>!~Ccb%s58E!UpcNZ0!xGo|AZ*JlFtqX-vG7_1ET0<*-mP(|8kbQDb&@y3 z+4G+fgVsBNbHZcLXulYNL@Tr`bRMPuy@2h`y2u=^Poj1|SEWuRou={w&8Zuj(Tvf{ zf25URIT>+N30hZ2ksdXWdRnE$dZ|lEVt5LB@^LOJ9_4pat|YQc_1xLNdK)S7=~8m? z%|Ob{=rjXQN`Yzz`_AjYH`*NZ4POw3JSD;NkIpE}ek_dpTZA&ken+kuBIq8O zOb>e(!j`=Aght3zpnmQxQF3Q7yt%UwRBdgg12?RLryuykUq$sq@17OF`Fkf=CjXwk zvqKs2&1Rqj?}B0Fv~g(Df_6d13`O*^#s{a}oeflWG$H=)F*sslF3K*HN?OA2)6bGK zK*qcXJZr-fVDL}^iXTqH=k1GyWi6-houoo^Bld=5`8f z26%Iz2I_vfh2F}RBHsBlg0nQ92s0VQsD&q>mi>Gh6rP5kpD4q`^JAEkPUd*zo&fDx zT8B)(PeTh2UJ+i}ppFCQ|3TwadO?r-T`=Gj36BU*ft_CyQ1YXPc(t0lP+7AIt$KY1 z*f^8}-yL?s^ugQ2#NBsrxzlRAX%7$2(+S0A%NF2w;npzMau)1)mB)6k`9n!m#!$oX zIqRYm!}@0qv!*+@vuoF0A(t!8W)}x*vCfUF@o#XK;hoQ7CzfxcHf>gBZ5mckRJxq# zV0I_zPqkB%?`2VjLFMeJKMeDQl(K3S!z4(kZ_H*)LB(y|@x+`tc3EU6_v-+%dwzYfSMy6+N0ecuBYfX=A0{ zJE$m35q#fj39->T_-a=sh`ySNHcjcrh4V$U=*9r8X}cJxTmA-V7V79s>O_L?)Q6cx z2XId7OMK(HJHD>^3|H?kg*Rp;z%8Z5c)0fz%KfcB`fpbmUM~6u+P23cqm!X%q%sD0 zXI_%{yuD4=B>7hdRvne(_a6hBRuBOGAtxd$vsh(lhAkLadsXFTuWN~HTZ4wTJz zLwmY5Ad$l=!lfnvSr~DHTBXM$vEg%Yh}uc{s6K1BZ6$fwuTv z^cuHx=n$?(PJa#X+Mk9J;h3+&Cow+(uQHwB()Bd?Iu-|QyaI`!WhgOLTN3+v1l|Uh z(1MWTaMqLt5SDaX=&Gy*cNpl9aA7Z~4gE)~8oLjkuiXiM2nqaR(PPxP!x#x0bHSw( z6Je)qDw!tDV?rJ3@Vcr9a$Kl9le{U7G;Pl#nGNR|Z_AG`!DTD%4_OO_+S8!zk0zKH zrH<_?KEZ~iyiv{Nc1B+JIkpOz$$(v3nJ_IUQsJ;Z?r;qvJFd0jzmy&m`TQx;ThIvW zM{^9P2U$#z8%>VUnGY9;-$ApZFYuAQp=k1~jR2gS4(Cs9g@*^vLbdU?ap)&!@Gkm1 zGH4fpw5DXjx26=KVh`Nr9|!*(C4s}!QrMRL2&Nx-j^wh|ReOitCMJ!kC2V~Lz|~pf zNQLFKV3h(NRq*%XEY1k9&-{u{Z|_G5v$OG{8bv6Y77s7%cm%iOEZ}(bKcs#=4p{In z;2iJONYaxJdrBrC^&4}c=*SoPu_hrIb2pDLpS%|QIjjc{hulZx&rtl~#0G2~r%71k zPeaEaOoOfK%F!Y#Lon`qBltRcnC-XrDA2h@7oPQ;jq=|Vz-y0Nkj+Q|6yLr>|5!Q| zms}Wx`Rfu1*?UgZIi;Am#cG zzm6jbz0#W~bn|E2Y(v3$yGnrLD_y)m>jY>D8`V}kaAgQT9`;r8qO3m5eP;k!dTE=f!disj-ob2%gNWnW4-EoH)m%PP#*C4C#^hz*Opq z<|am#-2;41r!&uo%E<8FMr87VQYL6nj&co-B)`jOQt&~C)V=ag65x@I&Ns$lpO{I^ z>K~>!0h*AN<|OJ%q2O`94Jd6~3|-))3%hUp2MhP_#<9yfQ3fl|9J{E%7;MpjneLuo zWp;wl^NlCUx%C3cf1Aq8ZsW24lX1i`QpM#s&rp2EytD+y6TZB)mVw zrl3ALIk}V8W$RFySsNYj;2XNenFOxxu>!|VFCmgGw!x?A6Tz)g6Uj`K<6!IBtF-g+ zm*Dt(HL$Ut50}2=(%Nr2BxiPHBSno$+>OOHVoSGrS;OoG8Td1D3K{=53H=SorHTY^*c(w_nR&0TQT_|W z)RVg}DYPMs8qQT`KZjaTXTt81=T)}A{Si`fRc;h@EnCd~H%E?=dAP7*<2UR?gBGUx zjf8F5naYlJJWfSR;>k5vtEfTMUyP>BBK)Dmk43DQs`MhM zVb_6EsO0A?l=E~A{Ol`-n%y#>o|O%F(f66SRUrf8axTL+_iusHcToc8Qgf&swGSTJ z-A4PTSONWrm*CZ;XTATCVlX-RxLwwPhpObyBbcDp_~gI>-rWNy;g;8)#~6$n|1_3p7 zbiCC@ikFplNmyikmvvNWG zrRmjnPC>%E!g6}Ojt(k6|4SGczmqt6sRevhF@Q^a82Wld1b99?gSOY`Cvv!M5_LBf zcs2JsF(=y(txB&$Y*Yw1Fm?tWsfvAQEwkAzO?wUcPX+rekl2F{-=79Xa@27++6T$V;1^#BZUx@%g_g&9o?|b zh$wv#gM^}+;O;no+VH0g?Dc3NcF8H?J!b3B0^8K;&3Bw>Vca*#(%6H7OdBI44Oc+p z6)nMRtzx1!C;;%IPt#lb$D#5I)9{~bABFSZ>Y%3!HV}Z)YJvmR3S@o2e+4Q4LieuUj3 z<=H4Bd$QCqkC4`9lb=QpIM}^>55se~MLNA!W%cE6z)Nio7Mx8~u7pgKZLFBYvg)q&?Tl&~6K2-ffdX*nv3Ug1^_0^j}v%>Gj- z^?EhDZLWippJcEc7Zonx`eK;uFoY9{yfrW=Vgd>-( z(&H2DaO3T7^uUf5@TaE}YWq!Nj#SSE6%*o+#$pcBI%OeTE*+11f@WhQ@ocbd^#Qz| zjbh$RTaW#n?NEX3YU~ykfv0YZf*<0mVeXs|FkfRXHr;v{OQvv0&jp(q^0NmPuQehq z?hli5eS*o#%&+JRvjWcU*MPN^fSl2KglITA1%i(U$O-eOFbO^h#3M}(>`5MG#(xY% zXPfpi4L))BV0S0X@ts5thMy!iL{;Ew!C2DotrfFM&XQzH;y}*4GMY@Of$L0E$vp;3 z$!I-ivMrPaZ!UjFr!H;8yPCxeY z14w5}2nkhVQnz^?4r%Vi)=yu+~d0;7faD*UQ44)=vBz}*G} zy1DWp&1Fm&IzsH)xXg_dM*IF-%rH~owo>K*?W|esth^W<8U{(k5EPGICSVU z?K>_N%orO6#&4)ZwUv4BQ{;49&5eeI~NhgL=1B;N5;J=AVH*4As1huQkoa zCM$!4vp>!IA4OLl2*vxxT~U&RM7cw;Gw(68YiD+mq>>~_MbW8r5UEs>gpxw!NQEvD zxsvSct|XyLR7yy6d?OvAOLsrNf9H?+=Y8Mj_#Dq8?Ka#li`H8y-4N0#X`fLeExJ}J zjaX>Ro^;sC+74@^_WFO}RHay&=G`X?RGA}I@$?ftba0TTJ$S=~Y^UY7GL_}YC2K`- zc@?twKkd@r6)oID0Ayk3WwMCZg`%OML$ZCT-$V^Ys^VoSi$td!jKpK#YKl!i>&i_& zEtO8_`zQ0SB}J&=EZgLBM|4F$PIe95mW@%c5G&<)%Ztt)k!7flk`K(*7S%2muzJnC zoKnvVF2{1TZ0DC>(($*AMBlT3^j!qao|MIN{YmE}S6_B8lj1kC$M%Oy^*sW)lvS^! zI)`jz>KBKY?T=PU$6JOA=Sv?9V`OKzLuytswo+577pcg0zuU@g2?&vHEvH4LpVOHU z>-yNo)k9puy8`Jp=hxCzbz7x>3hJ0?=c2h+M)z3a#6h+%;EvSxLpvj=9w9xUdx@E; zIahcy#7a`%q$UlIZxG_YCd^EF9vjiq%3k?9ATbY3VhxU5=H$z|ncO>V!cc`XEZ@$a z5%DH-yQd{EuJ(|S@6FHaglTyLhL6&@+!|5LvaS8znqVsJg(zh#D zvClnhSj~(plID+){hl#jv@vZC+u(Ldcui8kI*p8HgW8)o&mAGsy4HhC#qyfXV^fOty?K9cmUsyJ|car>*!aG^xR8_GDFN)n}+$t)qSCJ9>N$yaxb%AnE6akC@e?!k9Z2f7o1SZIQ`yby;zj zGF#*QmJ1O-W#EIWLVjinXQvV+4QV{dM*T|^*51u#SEx@BMgIN8MU5HX{%xa8{Pul3f~Bm8qL#jAx($7kVJO(lfb_ z)9%fXR@bXDQ8mk?rnw$Wkzpz4pIQm|A^NMrzm9n?HDRx4!mqbVRsD#aplve+=m3cG@*`CL5BrW-g9MLgWy5VE0 zWME2_?WE{^EVsT+x=y*0(bX+s+!wu;)_fjdmI|_@>WQPJj|g|$8JZ)cZ|w=0L)uiS zU9P!A>tLDmuS1ITk!2|}&SWC1xj&V23W(#vrf}ST^O5qAU2Da&=j{=Vndv4UyM3dK z$zCIGIj1Toqx?nl+;7P&+FWG02QG6fPuk1s{ol)EZyH6eOODE%?)?@?(i!nSpOvC# zKF;E&A}w*|O)L55Rh+c-ke+dnFed^ckhs z0@envl#ZShB>eD$FO4CFq$9q|naeMRcZ}S0msl7a;A#gVx#g=BWc@)(Qn{gwS%0eA z_V2bBDKNdo-V@AW&Zc;<;Kh9Isj`)HkGq{z9py?_#0D~t&PTD|y$-V$%s-~*6D<`_ zFl2-NQ)Z*AQW?v%$wIAP-j#lRYTUS|_UyX$)x#QFE9Q{dBsMC1DHrP_W858`nLrO2 z)3G3*2~#CR%dIalwW*ZQfY`>aZR=ugh4Z*&TVokTr#|NCxpTr>53VvuWrOr&p0;f0 z+AhYHlt^2FCmWesA~M)#$cZ``W=Vzy`$@Ez)eT7$dgo7ILo|UjX?-^9?qM&rOFAc= zJV`?oT$m-vsM#P|wC5Wmw;0KuU&1kO4JL~oZ_{Bb_7@9Ztnd?&FUN_ZYCa2llkHh5 zb~Vdi*UXg_`*Dp5qwLc5Ev&MrT`u3(%C|cm$B9eNx!D;Tb&A2HdGZ>Ib7G4mPqEUX zJ8WMYDc;&KAl_svmoHtuSsb#UTHazfw@TU8Sza~~RV71}s{PwE?ZW1%iqBm(vzt@A zL~j4zcxJ=x=W=OKi8yIWnfSQ=xT*=4X4`2@x+o5|)3ST~FI_6Rpd!oXB`{Cn6z=r= zqwM(kF(Q>{q4Z?c3|67EmeE@9i?PalBn*rUW}wV|SZDW%UEwuOx<0Fr%eGJzty`YS z8K+u{IK{)v!G~4MuI(o51wVo4d8ilrJVB9*Xc+duM*d^P7d~^YQ<9i^p&lD_*@FrD zR>lzIG}bLBkuB@6=YmV@rJsCPNrT4MbLDCJY{;|;(oMh4a=tp4TWHl#xne0%DeF93 z=`?Q*_h6`8IDd@|XZ_|8XR2etSe?2gvAp%XQh2bIGji&a42*s()eqUu-MgPCtP0hV zVvQV5A~t7|MqlGJZ8NxHyHeXf!!_etg;X|>>SjHcuH#I;#Im=#W2D;-9hIeqJZ7FQ zY?BlVIquNN>C)Q1lM?fwa?UVpE!(i^(l9ox%^ttJn9Y5pB~x7Qz=ZbL%3hg$=hzDs z+^u6n+!#|r_Hp|dY38<0$#xrSSx5XA=~=DxVQey;`?ldTXJ<<=pN5Jh)psl9O&+7{ zzP}wOPRi_&U!dm5bvKQuD!!O0uZXc0k4ak~zv0m*`?Keu^!C;lviDcg<<@e4@uR0k z@c9l3xp*So4G$sqN=%rgOM9($H6u{pcxUI>&ZPrz_3jv<=d@jxTD` zSCh?UUqgA)m7QY7Xz66B_m~E0_~khE;GYABu7^&Z&@u_p>vA)PqLhw zo;!)9cU|MwN!+CiZ;fPqnPZhX)wx3MLnU|TyP@P+U9iAW(V9I|d|1l6b%2?myGY{7 zFO~+YB}-mCe=q&YpTT(-`U`6`I=Q)x-O@c8HzjAj?B#-jw@QtLmQwS-6WNF3{s_Yt z=(1%^*4%HSFm7>woK!E_j(Jg}#&o=2z})zo%pP!i%-9BcNvC^Sa(C}%a{LSHnD(L7 z!n^Im+~d0t>Gh#~?97rvp`ob>8zQ>Kep8I%Rt>2!#>z@ee$FgzX8Rb{*r1wS@?a{X z!y}k=Ls8u5y*^B8Xohrbjfr$?#eAv2>Mvt3dG;`mqAPbxIU){TbyyUzG*%va%0*tH zlOq2rYLt<#%CaT!i_FmXtL*t9MQP=VjWV+-cJee+EPhNLlYKv^Bo3SwFJ5Z1RupA* zP~6=>iuGQ5$$$A|$U@*!xsh40$ouztSyi{U_`;!ovV&cJWPaaB@tU3(d5cg}Uc5M1 zUXk@cY9ljbvo4*nWlzPh<2T)54Yzt$n&_2Er_(81)QsaCwn!6Nr!qn|{uB4PLnsY5 z3uZg=Z!ilJ8rf0WZtTX}U$|H6s-?f9?+L@E>>uX)RM|J0Wo({5W@O)h&_%ULTGp_Q zGoA2M`g(1E^t?fV^wUUd)>9>!{jR-*)zN#;JlfCaPCOK``8;)L!qcVF^_v&4s^UZ3 zhS*-_Wy567G-pO-`K?)!mccYexh$6PviUA)^*P0{>cP@?TPiD`y_zRA+&}EYuwb}# zX>#FQ%PEXab)?`);-K*KV@pZ$qCPJ5x(jRY@g%o-ktZA5d|4>`=))BnMzO2WFD}z> zztF^;l3smpAk`cDQKI9jBULaPBN-J| z!S!#jkwh$yNn7xW%X;^To5(j5uR6a%u12qw?arSjj*pulex^2GeBxKV$mF4gEXO=YBq~V| znGAhmhAm1W*D@XPx7Uwlt;WkmeEWQvP_$kC*TzxyAfJ+JdTYviP1VK5A@fBIBj<}N z&7)+yaR%!kSCQ9owIZ`U*F|kfs&d=+ZsHHqnnlzxp7`nWFqW*f-~#xDT*)V)Q2Ik% z+B{N8Qc<16jz~Ji)xCblyee(9P42Cy?JqnUrzCY2gxcgG}b5Xf^Ao(n=TS6|(V7In0KDqwM0t3nevb zuei6fl9hw@($EPilCa}Z(!iq`m8%aWNZO}Q=X?ftN-K6HNe0(mm1HOA zN_FNw;f!r%%$l>$C6Qs-OmBWvvcggsd$|*sa8pvy()-XSlz*$Xu=%T@QSP1MKLEf#50vMTV-eL zr;7*sTts_YksPw^G6m!;cYj|XtBO9(RLsejEfXG=?QfQH4HG8I-l~82zlTyZdEH`} z=K_i7v$IfK{lrRip+F#xZ|@f!@zIit+{|QMF=OQOAy1UmBbS;MDv5Qd0@>Q$0@)VT zzoHQ$Rt`55$(E<<$+PVx+$Nq9%XKZ4Jo@l~$;)hGzRWU{_P3nl{&XdA9nGp-?W!pf z+dtWYlg3`0{}MBf{Aa{?r+YGg-6ycW=M_oR5>>e7u5Hq(t54ZpbG7E8XKOL7OH71FQ46@MCk>tu;lq zpMw*)-n@ABxO}2?O4m~1->k#Z=~c6(?|cxGP(Fhl8#_ywuX~M z^5?Kc&IhHsyjIC3G(vi*Fi85sdZ!tJ;mi;OJeGb{^O>4-IOZ*cb!|(8X%FN zk*sXhOQCk9D!Xx19IFy?iiy5!A-%7_leYXZ5MTVhQm(adwM^qY7B}4=D^|O{R6IJT zPW1dfCEF2`A?p2jKr|}l8FOd*6j9h14RLL4lgvWAT%=l`DVq=)El<1YAY1)ZEUz|E zkgGUoiM_|Uh{BZw;(`N_va3gCvBz)zl%*fLCwdJph#v5i<=;%5#h-^7M4cz}#rAKD zScTB zk`C@(^)2bc%_-7b$)}kSXY|<5O%Xy-$py*wJ{L|`Ayq0q`A0Id--R)Lbe0Vu9hpSm zA8a$D$)41{$6kT|xTv2IQiHk0OqJlP;9Q@Mq%5;k*cUj53s|8jnLK3{qj)y1@`=+N zX0K1a?Lg=`>a0$$;M|6eN~56jOwH*LT#$}Eci-o>1Z{{CoWIZ}T!T%R&@h?AL%(0> zvh*o)D!bcuN6#H$$?I6=Uf*TguAh6DNA{yQuR>qOb5*kK@NOK*nRIpb%*!?0bG?cy zW*=l+gGpcYu})uDCO9e_9_=r&-O^OKm1ddeNsY{&X^PU-4kF3Rafi9F z8`Py9!kJ7);aq0BvjTT)_9vn5`;DB>%Nf#*UyZq4;bszgn1)X{kzG09aFq#EdB&`m z7Q_^}ALi=s7jYV4NuvEvU&&Hmy_DMRs}ohVpB82B?htLQw-vq7sN$rowP>G%r6?;- z#Mmw=WbM^5L^ZWRvU@G>SQi(8%;xuZS>EZ((r*Rzvbd#jvadnMMGL;YU~_zjafR!z zq@xDcu?L@~$(CH5D~f$&KYRzGW#pH8A|+0ey9g{_X#Pi^{A)xgP& zk5f96?0ehROrgG#yVfpgjM0$xo{Ex`WoHVf9@k+vE-)AJXGgNj)DJPehn_s&XhQQ{jPk_=zb$s*BAv7b@6uT69!J?1FFnLNPI8!;AsCN$`R*l>b zH?+PZn=@AeiSIO2?3V8BG)(C(~Oef{1m_#OIgS z;lQbJ_)z#9*gh;nTeL?^zPa@S;;%<=d2AOz20wTw)C1t`>H7F|^8qCwD46&yHmAI{uz1mxtcA@458!23RT!q+aJhr6$G_*T&csINCd z16v{Wpk^eBow=Eek!?paUWKCQQ_hIJSb~N zm!40i#?`0cu%oN6cJ?F`b)y{l4D_In`)tYgpG;ua?l_!eP=J`liRk&2XdL`;DHSh! z20araaPQO6SRf;D!uT$v<1vI2#{Ph6ojN#5!v!0EI*9@bkE1OEW2ma%d01hd98sP= zIHafsYd9F8Ya8@oef2`HXjt7m&3!knGpxmjdJcCG*Vqk}>4IbL>0!!#x^i+Ey*}X&^Hi?UP)0CgU#s9qq`|8?2V8A_6E%Y+6bM#su zZ+*tI8+5mEnpQ$QG`>l!d^85OCG-}m>s6zb!957N>_odGv0&j z1!uvi@ge-!(lLS~>(uBki<|hR^HupeQGNWtbQS);pfYLMs3+#$e+JmnNIt-3F_BR z;m2qc(F>o~)89`7@{6o``Hdr6=)rd>{Mwm$z{Tl2YQ3FHNUphq>cq+LTc#0?pDH7h zR?Q)QdhP_Z(+`8cs)f9ry+x#`tN^U;7a;HI4Rr z9F+pFb7CfP3^@tvJiijW%1rbz)))?}f1b{KRzLvUyMZyWynINq_3Zg6DJ15kcDfk@SjCciHU zLbG2?CvyESA*yIR@pkAm?{jWES#o|IbjUF0J+og-TJAUH>^^fM(_tc`%rH|LX{$vfbi@n3=N^%>}lp8`pg2NNTmvx%l#Z%9MW!!~NG zbjb3GC17K~EKs+;i%_?{PBeZRiL}&@5?NiYDEglx&w4~TdE-VJDLuo3{6&T~6%U7q zJhliH+d9JCduu?JmlT9-9|ar>GYRK>19I`y$5fT-530Av6f-!Rs-&#(C&jze36s6l zqShl+V}uv}qP-ejm|X#k?><6op#kNwg~W~62){Aifupqy>5;2V@Xxo4DO9wJIyJ(W z%6b?I+wXm$Uf)c_PsjJ7v4`XE*Y_9k!~_m+x7<$^>{X?m16Sc6FDk??i4ucyBOXn{zm5ZdJ_li>tJcU5viP1KoX<6 zp!2qMwa@Q!&>V&7@(`jpJ}y;|MAo= z%4XeY{-!r8=!pgk_*Y{0&}*6uhOHA9selbyl<5KzPBd!A1Ftjb9Gd~k>GTl_uKhsu zpFPd5Nye1^Y7PDXh^Kp~3cC4sBi^x8jejI8hx*c;ihf?1Kp&cg>6(xN3Vc0B&y_%a zHEnhzm?#Q&H>({>0PkZIg*$7TM_Wo zcA~mvZm`PK5L-QvgCrXjY*al1evk@~@6P?mH2))bGjy5U;&}@=?Q%nnYxWZ6`TyXJ zOHE|X`VXLWbnLJhBhm)D|0FtgjUc9M3xQMKmV>mE`|wSz5&ZB9k)c6`&^YxGI34pD zq;1M0>!})2<^6afuoZv}mCK>S&bK_~hC1@q?FRY5VQ>YdN-EgQhQbL(nBO=EUfdWS ztjr4r*>B#W5`%qM+#ihGZbuP*-^UZQO)MNZ`ha+BW(j29FQH9`jIsaDdNQfzGr8+| z6u!S}HmtA)XmcfiGk=dK&rJOfTBJR~Pm^Nc%CzgG*;f$?-bbKen>U&*Z?N6*7Pw(_9|qYa<9DBXP-A!;CJoP! zQuBRqY^wrv+54O5Z{|Ux`Yq_ZNdv0xiU+ln1s-dihF!OW!-K|QKx3N^K~3WW-Qy{+ zwW|PM0v^ybr;EJ3^bM3bUWQYQB-pd;Ak?lKi=Rxn3A^@nqoiad{2|8{o2)1R1HA}2 z-G7I?j?F>`)k}~_?2gbhHK;s1-kF~0NN81c@V2xLk^e?I!iD#GNgcr*zz7`B17bfY zyU4)u!^LEaPY~>UGzS>Gyovch6Kd~GAuatZ(eKN07(LeoDqnDhCOxTOT4Xp;UOx|- zhPmO9)p>Y&_foK%yND{T^q}=4_31Y=N1&o43lOHA4E^73M;5lfxkSg} zf=K|C=#`@x)jE`Q#yHr2!jt+|KMEyvT0$Q87Dkt>r!GF{qr;YK$!%j9s&eWb+!Ou+ z_!|a*uHCvQFMKQMrSpay%MTI+434pla53ON-F{1;AAEvOj8n1|{<9F=bcKAj{0x8A zqDI3pi?Gyl-9Pl7hnFZ5TLTrhH}mmuKDF2N>yNVnZi z707?6^KZT_!T()X6?7b};M;HI`0f0If|t@EL85K}f9c^sLC4xSTqE9#TP>f0-erBH zLETim%_EUY+-iv*TO2_vJqw2){Wv&lQ!#mOA|@{r!3bYjjF<5RFw7jnJ+1B79>>5- z^hvy2IRKP5>%cvU4bW(X4b}8$3ED%ig?&L;(9dlf&VD$A6bxsBvxokI-<$pe|Gk`m zyeCW1Yo$@-rhngw&}V9JK*t_IwuS6I*aOFq5@Klhys|yxh+rGJ^_Q9j^n zV3WTwOep#d`!??d(dX+)(HBcJaiJzWZ(jz|^@2gkmHV(uXB#i-{86ye%pCcDY9k98 zR8Zp|5%{bc4twn5$&~@~VU6)}T%kUjEM2Wc{)e04Dc{NXqC^Qtp16&cU7b(fITTBF zTwMwIFMkm`parbi{1?rPU4Vn{co7rHICyn;3hwgCgDdK#&|=V0W@FL09nRumaHi#DrO;p>W3DASFPY44S_ zw0guO{^L6bsS)YH@NGjr)jZfqIm&+k-9x1~yy*a4zgn0gA=GU;Kaw*a`3>; zU$mVnkAAT+nZDHmsWXMwX_vQSDd%kY@Bq*R+IE~49b1}BO+T(l_eOr8@1OlZ`8$@; zNwS-Gqum^0(*jQneQ0@WT@fQt1B z)PG)%$b0TqJif*Vy{a_B*QZU!!f^u9b%`(V_?m*I2D;#j3wI;WiUxG&t`>2u+#JQRMsC~N4&C2{ z<8%HW@N|c6G-;wHc>O$%aJr~~BK+o&3#}VL_g+5!P<9rt=hVTKo{gXMLhUs8gSW43!I!pwK}FY;;3ebH_|LQ_xFz}o zE^A7}OD653M*k!z+pNu4cATR0;z!WiHkeYPwo@>4&10%AZX(`wMSvf)>rg5Wt?`$? zWz^0HbJ{a(3T;VtVehF~lsT}YW+g9#s~081sm1egmPo~Y1|DVAi7huBfX_5FsWn>O)G2pIG=9209GNl^>>kZRy^zbu{CXLs zB-BF3_a_qbO;#cw=ZPrlRV>xFb1ka7@f3?Qt$a_@+GINRmheH!VW3V2ym;T`m?>pjsyZi`l@f?8$YJL-03ou+~F#+f572A|w zHo#jgONqT#jln6<43^)g@cDpvvaxJ2nbbT6Db~M*a}V!ARmIQAe`98#nQbq~Q2mwg z@JpZN01NwL&;5t-+|63eQ*->*hF~zBgGwic>0@; zfsH#>0sVwjxLiMmoVJ%B{qM!VEf|2hfpBunLY_^_-+K7|@pjM`VnEor`Jq<55O^kJ zHQcsMfwVYM1vQ#)kU4*2!JU7_w4nVMpIVVgpMT;FZ`83>NVKnsbk(>D z6|R_phu$*uZQ@#Tif<6yqgqX5Rx5z%Zk0UYJ$*3#YZCA3!>8o=h{IrH=n1$x;4}Ol z6iQ6rsDu3#70JV{H$duyS>)y&5GM4v0H1_6!NTY%VA1zw-_mOjaq2~nJMCT`ps?_95)?cUF4_v3$tUbmzZc-Mg zH0RML>3w`_B_r&WUxW7E^94IjA+rC+U37fb5XM~`dcWxkT)a1h$hrC$ggh}OWzDCF zhtnpamG1A5lgAq1Ih^mFu3Cb1zYtKlx*g36X~-r&K5$18`&!u8SOPs) zOe0?m7^BDko|226PVK+SA_rswdSS4W`LT5XP zt3m6b>V!xf`y+-bm}`P#T?OQi38z70(gLKl=OX!fQWB`J+<}*F{)z!O2#xJ9eyfC2VwH2V;;{EV@&Lhxg(T?HH(Ny4y36zG8 zH6nt4!NmnjAfa497JZuuZ|r`Gi2hNy9NZ%xXsyH}dMN75;~(&8(>0V=IL>Bbk~zHh zYd8Eh3sFg*w9qA=Lb%N69L(7>h|GH+<@@p|bkK;#C;WE9EK_|vOV9>$4k)6fl`2HS zI1!9UlL7N~03B9KKr^!!RQRf+`w3&f{4WP>(xu77`uBOHlXeg~<2@aM1IMAq*{9?S zlXhbN!b32;Y9r)-e@rf2@C*D_^T*Xh54rY19&pMmLkR^}z|H=P(B`fQ(7nb(Nmd2K zucTFg<_Xa+^|xqKmoIr<#|n2R-GHe+zIbq<1!@YmBS$Wm0HsA+$)EZmyo`)i_(Jps z>XVizw{jHxa48(Q)`mgnQJyfPbS1u_OoQ%-IZ(gS2!6Y?64l!ogE0h2wEmic|9J3F zn86)*+~qks8u1bqOHGl$Nq{oj2l+479}(nEOs6AP%oK!{7O;{-GMqxdW3bLi)P zC-4;)RnVn+iRkxFL|@Li!_Vjs;Lq5UOMiOK@b&6W3Kl#1(cBq#LEhP0e6tyU_|CIU zXua`M1=^Q0X#EjS(Bi^;e)TCYzGjsT{|xTsAC_+rI2q01cfL{;yo$|3)7;)5v)E+v zvFnN9nzsXabWX>`xjNYMiVE!6WP;*{Qb?60<<>>N<<3c8R=~_5mW_JJze)YnV{1Eu%(Q(pv z(F3U4{|2`tHSRN5w`~+P&o2m` zn0yaknly$MZT*Pf4jJO-!biATVK(|u^9)bcDkDEx?|}=BsZzf;RAbcafG&vtLl2%E zp%gS==#etL5%esM4WulBN~B!B8*X~#AE-=!&((-ITxviTD7zM_mP*B`(I z9XWUvV@_2muf&s2sexMwaUk(*5Y!i2<4N)}sHjh~U+VI2r zdX%=^0Z!eOj&pD9hvrAdVVijlkhD6AWE~%I(~6TwPCCG84yr&e?7&)reR)a9Qq||fuRe$ z$%wvEI9j3tJmV(7v^B4Q#jfjcfz=YcV7nTsU#o>TWPBvFU;3kuPc5jX4riFNbt>WU zNs45y1$frvZIpKRW~@2rL<(PJfcBoXs9Wtn^40RS&}vE%{#(SSRJ2CJl>V8danO7! zWJfmoG36ktEJ;FTekp*;s7LOl{gkWP4K!k{7HIe~2Fv`%;#u;OXxxO2I7yUBt}2K| zwaLwh-#H!+EZIkOw(#)wbAyy$S`jt6VLzp1yq|IeGAOCwQ5&icAw!c)u*jolNjuh%S*?{WWPT{ti1?cAFCUkMvH@xB1 zD`ZzHq|E(4P}}Ef&|L}&*x16G%DlFoGK~5M7ymj5jsDf4R`qGr-k}a`8@QQro2`h1 z%a@_=ii;sP;tM%_vJ$#=-UJ73+JLt|wSbP|?|AXSja0*=#prUS1!7O0Bu>0?h0ppl zk@DGesyE6NwWYN}YxO?#?{q1W4d+!K7eADr(SvP1MhUNgz|9M)d>rPPQDLm)!t6udZ@2)nARjWdOOAR#C0P5dfajr)@VJQM zXp?aixqB!FIM!{0zo#Du#TMaUlXDC|Wr?-mObS5{p8Uz*VXMr~)te~zb8afXd)F;W zB&(wr=?Ul%-UnFhQcG!f`0~^6NWN#oX8Iee!|(K(AW&VUOb?pB=Z6h!;}@OE=BJKH zq2%u_@UyRI(UDh6;W<4QemL%+=f8hKKdMgS=dRKgl)vkt%P*hjDzlmtd^9aJ%6$*fP!% zXNfh@yN~L?>DvwRPM{TB;k=VfQAi>M3+iEcs0sdO5kpvyx575TgYfUdo4Dnz0rEK^ z0@b7+Y&RPP+-}X|#YXhOvJYkOpHePdx%&t_bgLcJw+Y~BVG}4$+=cy9-VootA=C-b zhaFbVaJAO~-k2pZe27ug?A3%`kITNWZw(0&3`AU_1Pc!*Jgs3 zp?AT&yMCbR!FTZUU>u?PBo8>uNrD~qZa{g3IYvWc(pLY^m@A zn#U^x51dc9&i;V{_x0dITfC6^z#QUwd;lTYUjVE8J`+2=3dnM6 zRMm!aM+(lhQOD!xY^=5bP)n=Isc#FS@YWz7>axpmDk^n3l}*~BrPZgX&TEPIz*8e? zTD237J$4L>M?A$*ieD(|EJL4}nT1-OS7L%YjAsv>1;pQU5I)5UWsOb6b*6e)V`>0y zF!Kg?5f3KscLD(kO5`|7g zt?NLNf)GzqECAl;4e`$4N95x64d~px8Zz)2;9V`4Og`v~2i-OQU`T8@R25gE(Q^xs z>N6#xr6vZLipQek4`;#=qsD=*tX)uZk1o$Nv;p`Ssi5}yxZKU_57faq^Ju)T)C7l&(Zg;B7TCP}`-BL7eiYsq??DzN6v5+K zV*F`U4L-6$iFCZ_3BAV`=sK9u zbpozUpN>BDEx-e-Qoy91VB~x9G_pv$M2D}^EVgWeC}!s^}#)V z7zdnA1^cd~xLgnD*mM=!`|qOzqn}VjSuI8Bviq>A$m- zsrMO5e7D63RIY9nR4^Up31?c;lafrR;*}QkgB@3BgOGA+-wIp$tG_yBwpW{6w@3%R zQ+$RTgMQ-l>#j(wIElKtRt$DK&p{dob3n*i8}f4KKPZnVKu5OUM4r!*V3qQ1TvPNM z-=C(0Vvc+o*6uKbMR_Thdb$`6dh1c>SQ7LUL-^yJDt;ZL2_xcdD89rM9oSuiU%#1; zY~XyX{pkiVwc|5HZVE)^mRRJ`y#q{JtOFZXzb2it9{_4bET6) zXNHlArZI5O2OZFvjFCbABckS-C8@RB26}v*fp<(@1mbmiQTXJ;F#M`3<-H}6%wIB+ zT%H>b3g0Hck4`l_|7$L=evd8Om|X*v7Dqw-4tpew8H>|I8swPOy*S~60|>UYg(vH# zfV|URP|b%mAVKjzV)Bo3c9(#t? znBpmLHJ{5be;gx#Eyeuf)^F+NY$N_<*)qD1DnlJUj?@l?u}98g7H5fGoc>;a6byDHWfI%tQ8pSOa-Tn;)wSy z1LVwQEL`@c7A^NRgc{S1fK^$sXzZs6;QPBCx&_gzL$z8nbZ;gn{WlG@M87tz)lK=poZGh}p9&f$Ea;UXzC%EJM zk$gI>gE!81GH}#Cz}wI;gKU4>O9%`1flw<7Mwi|P#n29Rw>A-YppHB;=L;Mk-%YBn znQXJ`aweFRWepQ{ok2%;{RP5}<-`*IV`TOl7w~CjCr@kUD= zCAH6)qmdoQz{Be|fOA|U7a)T|qwziFg5+wqRZ-VBk|f2WWjpKrZ^=1PvCd zfvZKj;NLbOIBA@R8yZun&D&a#o7YjwBvg*noKH|&1VcFQz8CV_nT{8@Z9vwRBJkwT zKC~xpA3k80hBrPRgEeUds=VBSnr3|#g`eC`@o#8Qn!n~zrVGPi&9Odg>KKf&b9cdz zihFqf!|8Z~=~2ACkD>1O*Hgt;$Kv;X8dRS46X@_$64QoP@%gRuKe{2 z{`flxY`TKMV+%;sI^EzsY!H!EdxuERbsfY`&zppgAQf&V)zGB~JJL;G3QlH+Ozrq1gze7vxP&H8$dYPs3y8dW@hO&+1pL3&#TkA_m z3uZP9IhD&(*trF2zsn#?mL(BnfHkrBburNL%;R;bo7m_#+K|2O5x{r7I%F0NP#F?^tK)$ z8x6|Un#qZBH=z4|Jo%?J6}+{GB;)*UkP!=1(Y}R&MB$8|WR>rAqAc(TxY6-|*zDwB zeYr9lc6XJ4?m=^qp&kTvPg?+;rLJJ^(up84F^n#X8lbnBD^fN0&(W<_uP8OAXY_oL z674bFiF)R{jjGmY#_MaAz(IKlu2i$6{W>pG|Ama8jx-&kJof&gZ@=omyT}y!&OHaZ zxKu>1nievw-@8KBEwiGc`y=3A69sywTRFAC<2?1c&x3y5cb}f|A%`08J(srl76^X7 z{f`Wp8NiF%v5TmhXG`*q>%cYbHDK3yIWg(T8L~XtnW#@GxAA!PhZyY^MGX9O0_nfU zk&!#P$)|mWATE7A!5iuZi|;HV7})_Loj-;gVZ0x}v6e)I)iJ_jLOdwgHAJp@p9a)B z>&cw~^}zE?2QPG79I;5}5^mdvkcOfeJkb$Oo|+tqeNzuX;eKD7Qne1Uf&jEhe>7O! zp-F`Gx{=SB0x&c~8!>JNU`Oe7^ife4C$AZfT`wCzX3j|DD0vFc?xw+l1s{o-*9Hh< zJqKv%{|NqEqXI3yO~-j*DyZhNI=<+&n^aiuivH@TQIFHyfw8EC;A>=~kbrZT$~L6p ztY!_f+L6R8OFsGigAF3XE|Fac@$iX$20l1Xi<)d03|>@sfLDm20_Sf-eV?15_FHe1 zM1%lh@G9zTxIyj6O+-CQEWobpXmq`=5>4^Ai>y96VHM3=V1&zdbZf8`S)KEz>J=VR zwi6$t8E3<&=P9-H$l7<5&C3<2&~+C&H{${#udSuV+%^JRMtP#+#*Zn50h0Q9!3r84 z4M16|AJcDaF2hUf`zdtq05<)mOf@PhQ9FF6(694?vE|q&xbV^fs>#Qd+W5~58O9w$ z+K2ja+&g{B>2L*gIm?gAKI#B%^2*_*mO^x7qc26C+k>z1f~e>UJ?PMU3@v};0W}|9 zCVREbP+Q<+oTZ8J9Jd_UwRAjHTMM&YLOvb!(6613zarH z;{F3Qu+E_qX5Ha~18Isx#H4y+NxK73*qj90j`x6!Us*`9;RDhXE+WV3ro$a``r(y~ z>(DbZjr92X0-F10k;?;>pxI@0xWI#i24Wq2`t0c8w(0>YsI(@I@Qsmw`%T>NXfe7a z7Le4nc`)?eLVPnij(TVO8TI$3lRx)YfVeF)k=^wZDe2O;Ag#{@6inY8t`cakGm)K5menm==RpzbK;*)mTuQ*RIB; znSQjAPd)X&+O7kvsioUeB}8eV2&jOFC`r!DoRZKHQBY6>0TC4lEg-!GdqEVHqNrfS z1|o75EC>h*MJ$Mly(1`g#on;%JAupn_3BmM{omvJNMOCLkzZhl?5q6K|?Z;E}m$NKJB{n@-GvLAE@YI>_y)9ok1gTQp$02w=HZ6z)v)v}!cC8^$KkVb)R;0JzE~{__u3@kpXZ)J z0rCm>)ut2}V%`jwtYbsft82NUHFLR{O&6g}+BmGIg%)@hM1N{p)B@aMcL5esc?+q} zbAv`MX|UwZO6;Pe8t$@vC}1DA0MzwX;I*8Mtds5WBRZ+D$I3fs;HYYBf`c8>N!*0r z*iZ@6tAaRXr>?@fvwHZ8TR3s0csEjce29x3eGd+PVxYoR0a)T2ZE(e%h0jvohNo&9 zz$x=Rz+3qm+*e*3WyfTo8MpeQoG}NmRT>uP)uSoIUiScGROyKAW*gwxtkY=wr6ur! z{2sL4=qzwb-w7YweFyhmhy#{i03Qb)AfZYAqq-^RUe7om0Il3_Oev2r@c zYVr3YAK}Huw{f49^y}1eyWkz;6j-)w6P9N34pgq%kBa%q`0(Y;T&HbXxc6ZrBKKlB zJfTkCN9VbiYjnmN8mldVEt3gi@lFNgH{J)XQpg4ymtRMp%oKE-L=8 zyh#&;WOZ;_+c&_(i;8%B!VCJI>`Nm#@Iq z`(Fp+?zF==4{hwIQ2^&?;91moG6)Z!8U>3A!lAeOI$Zv837kb};Q4_qFjRjJ_Y83v zHXZ~-hNmH_ept*MTWyYePkN1vcH6^HB`vh9@*!yVJ_p7=wu4>|UXi2v?xKA3YY4o2 z5!HC?2%+A-iRu;5MtT?J63I#luvpfOSnMDs#W5*lbEX!_q0W(YeaBG= z3)c{lZ84NVq%4)y#HJdG6Yw$ds?_^W@x(2g)o9CxB2wgNN4B*Elk8S|%KG+7YVbrJ zIkT-dmGCGT-xj?eeK;%w^Rx$WHG1~OYtLN9SBwq8SMd6y!f`i1-sN53M!6NH)Vn8G z(&UN^%vPiO!Ar0xrVTpvupGB@^?-pdKcPn(;<*6}8^Q7=9pC`>Al^?PhOf=1Lof3O z+!Kxs$U-X~kCj~p7U!m-#lZ`~gq>z6Nne2at{j2~+pGrng?mx!uJdT(1_|7~+70>C zD4>WB5_p%@4rbMuaEoz|3$2a*f;%QY=RR-}iDTnb-rGZ;gUcaU#&J;stY0 z^o8EI2{E$73^ET7<9>RYhjec&f_Hf-Ao-mv^xvEd^Dj@pL~Dn^0~*E1W(k8BR0 zxBBCsIPs{rKbt((mJRG%Ey32b3$VCtIx_NC!j4{h3%5;lKqk5j+-yl03K+=7U%tD9 zzv3>$8obsJr3VY(wSmXr+iGjLxSd7J^?nG)q`G2R%NG+F!4L7`lv+&p`cQPSM+O=k zkpqhZ72tZstGs~l`TX-&)2NH@?f5yvoOlZH(fn)eX}lW|QB?j#EgmagM2+5;ff zNO_eO@nm;8@s^pDP%Ev+^Ir9fGuN{-ci#Jd5>rc;l8_Bw35J_zkn>QY$0b zxZK*MywF=_yy{xOn|1aH&!{knuXPXO*^YR{d&pzrc@6D2GxP$m*=qw=Cbpvi-ZjMJ zVLs^L0ymVk#uXN?re8PRl#jVh$^-MJ`Jp;dK5nyD1I>-dgFOBrJW<&Mj(OjgIDfwt zs1FE&2L05K>=cR^BTht0rC#8G`dHkwu@oQGqYCo(;x`b2y`BmUG@jCUBM0GC}zP%>x>c7_#v z7QB_|%zaQ&4R*OP@Cl~t@dtKUAcMGxT*fz`2I@JHHgFs4AD4;gGR)zQg3-v{PzP)L z%z?A53Q>=Typ2yLQ;Iu}@S;=X zdBj^I%GEiMYB-inI`006PRpm0nhLVKr`ap02IJn;m4-5EWKlhjFgr*#reyML(oA{j zU+j3(`HH04`)D2^cY};~-Hr#CHBnlpGbr=xODLAH3@^d zzGYzij9UEoC|9^eya<0P;}5h9-vHtGtFZK*CJ{8;5NQ>z1*@I4kPKcssb^<2mU zxerbQ6G19S!oNVJ*}2??Z3woo7lZMOjM0>4Cm87$gGO>LLZv}NICu3-v9?G)+;7O_ z=H0$B_*yGOTF0-Vb5n1E6?==go4*Xg59GFS$K`Co!~1)J8Aq03+j=g9Tkd7z>SM=a z!|TOh$cjQ#f%+gtK~H3wCj;7D70?UizMw643piBVj53~kfub{av58M}z`9}?`n^{b zu-+;a$!{%CGgx?M|@Bj0~|MO0^26vhu0n5ag)|) zaB6cRS4R+yZ@y9p4$R8LJuc)Rm9+-=!7?72c~lX1JiHr)DQ<-+4@ZC-=SHAh8wN@_ zlZA{k=cCq|t01Xj4K!FZ0Ay{zIMm~DT&F4WFiXD{gs{u7)6WJ$jrn`g#?~Vo+x7F& zb$KS#Gm8Vy^IfrCnTg=jiOFo^je9T!Zadd<;zl_8YAjk%t_z!T=AvHyIF?(v8%%RO zgl;sI!HK==xtHnt4)0}aqA7|paQMkS=*u)Uw!uU_uDa?Q%)b}L%`wu1vre;-Z1e_{ zR>S1lY~(}ttv2W^YZ^?;)RVn6rf$oH_coS7v2hRTv{f2)++-<{Vd+fmd+~|v*gKihTkJ)iJfcb1C@7J` zV=ofQlefXPEI#3){FK}voJInZ#YFpVU2;N80X6W#2trmBqiWZ+kS4_^$;SFPV%wMq zD(&1JV&;~Y=>4-=a#^J-S+T{1bh~trOnDYeiCHn^x&&iNzpes}@EQnS#&SV(fBJR2 z{Tj$=Yz^q}c!a77G@-}ITsW)2i>vu{@hXE zRbU@Dv9%0Kf7OBoB~@ULPOgQoMxW$f-lGnECF#hKk%;jc2IC!z8Sn*of=)qiY-IZ$ z$m>`FCYz_hdEs)PZ36{s=QzPQ>!&axZvi9*pTV?lYH}mCE(8^dY)-@bQygo5S@_&= z1?+L;IOgLigUuT6gc=Xma9mbbBBjJ*oQa8jvDBk3fVr*+<@mL59kYgEXSpw+egAo| zp~ex0zR$q;9ev;honBmL(|Z`>T@7aI@ED}t#F0l0NL1arg| z0Hu@LxvL(zu*YO+fOBE)pek@OwnyO%w&JxXe0iw>Ecjz4`;wIwtjO2EymMz_WlEbd zhmLtz%BlX?4gYHHfygTOv3)&wj#6RKAq>Q2WWo@P1ap?QfxCmIlf=$Z)S|}|2$w}{ z$~G>UyjAZ`4Ugzcj@y?HZzcC83okB7ZHZaBmvvIb$}Vplv~#=2uTG-t|M2Bxk*k7RUx*gUM3f47!bGJhTwAkRixmpJ}EP#FWFkZlRU++qGrVy zlLJ@2A*U@K4lVQ=Va~$!+*L_c@a61cxKm3VS)c6-A1}HK*h7wBZukf8)6p;4&LR=; z8l?}WNvZ)*nGY7y?{`o=I0l4YK8}UeNI0cK4x9e+9JiojIV^fJ z7s%Jn16ZFOD00Uz%rR#Q(7C6CvU^SDDC^pxAWsBst%8vY+RRxq{w>%UcNGO3G)D3- z=5afCFVR8WXteCk1i(5s2_56dah38(%w+a+?9hf#?gMN-JQNg(K1{uYYDbh{&WU&6 z_0`*u*3fnEcz*#Jn5zTxJa2#l=hwmNA|bBu!5zkoSiv2-;2z}l$i<8gT!NFIokAsj z)L?_jAjIq7p{LDS6xNKa|da_N+lPpDI^9T>qqviJ4N-7^CRYGO`ys&PmqoaWT-cV$;A15XR7>MHSzSA zHwsAGPr6;1O{%1EN#FkGNqg03s;0+0a@lNMs=bth%fFRJ>vkoASL=OIOSKce%Sjc^ z-yVvW9`(cGJ}}WFhAI|6cP!V}ZZ#aex(z26T)QFx6FgSZKy7xY1xOXKn3tEX&{uxO4Xm=YU}; zn2>Z2d$MmKNN^`P$#aF6-jFs}n%xQ<2Rt<^DvE)RN9RFzP6(E4D1?5QM%-JYl(8pu zcG$yVuxncjvFZ;6AhXsF zEm+YDaF3~QP3o6}oP-vR#*7JY>+~#m`}j6s8~+YWOR|UA9%^K0awVyqUxE*CODC@! zV3Sv`Z6Ot+;)x@JH^HzV2clv>B4)PQgK5|7@ap(JWJzfqacS8ZJk~ymP|!b3mZ*f_ zS5Kvqats-=+TWObbbdR&XNizZaxTJaOLI|EkS1BcNhkOjB4Y1^Cq(0i<>dIhd}4@i zC}~*CN8u)qLGHmRSWvA798>Cylm{>3GUrZ2z8PNJkjT5h*s+qEI^C5MvpE@9ixpsl zdj{alN&}(Ek!T027usyB2|Dyt;2FI$9BYfY+?|8zVqI_-s`lQ?Z5g791((v#jbEwc zR$55VG2M&6&)ZImo8@F4ZZU$ne!{e_N+}-qk=hBt{-`l#fA^7WLK<`tWs7IH<%0& zkG^Uo4m+GL88zonm4lIiWlKFg1ejV#9Y+#cQV?w!CfWTUF8LT-CV` z!Pb{QkFA13-dk0A9Q^q$uQK@D3aj{xi3`{}l?F3+mD*fOrB$nlu1AW6A4WU6mXR?yy>M`Jft z9f}iK?@VNgw=SP)<#vC%Sg-Y@#1qp(!SiX(tKttVJoFqtu(5aa@(4U zc-GfQ1(^f*=Pjwc`WHJ1XV<0pHU164SOz0D)yV=zk`{T;-IyrTzJ%81sF2StV-$1X zA>!UV3>SLo@Zw!d@R8i3#LA1M=!kI}u}YGMZ&5QvgQCoMGX6X&(yo}?zy3A3xLtv| zwoQqwcs7>I(+#)e_2^;M@>#)xA1ATYl|Nx|NR?^Td`;6bY>S@-dw7<`398QAS3{Qf zIc|q}l#_wwdEG{f)oz0=9Ou+pP<{_Ad+?r_w>pU}{ThMg`3L@%7xPA$U#Tpzyh=SV zAAHW8Cw|jnp>v?j;^Uqa3(d(DWSn&kX*5NRGP5(lvg~zg7v&~p$^YMd5n2_ zM=2u}h`hPUi<|uG2`2pXSp}3`;0p6& z8m^RYx-+jLb|vqz%WFQ%x02ssex1)NV_VEl%rk#2%%?OQF7u4y3VDk5TGaGsODLaF z*X*3sS6JV+Ewz`CF&Zk<`C~M6zZvU~ydBJSa!g%@EK4gYaJn!-Fg08>Suj-?9yVE& zh#_WAdPb>ph92lLdf#Bs|7{FRMjuuW`r5__qv!|Tm!%LC9@v%aK}0M*|_g(Y-j_J%V zpH9R;mJ*$yxJb%!dRHehSq#Wh6+{UG6Jo++!svY<5;D~oNNN-lBAg}& z2@e+0QE)m+bZoRRMwAqtNE;Be!4zRsY%nbjw76(7NS3-FDK4aQmxzRMf@x8Kow$25 zH5n93Em4>-MH>H4*`Obs9?ib*nc3;{lPM{zO>#^ogU9O8xhE58Ch~vAMDwqhXz?`@ zEq}#CtM8fkU&~f6#=q4mQ-;R>ifq>1Y><-1rh6bIjcxZpN*cTFK{wfkbf1v=*>?}5 zenYzlQa^|8K{wfMNhNXEw~{#gpCr-opC!?W^-Yx;@r@*o{B23pW;p*y)>RoUe~CF= zp=cuY`%5{sXo*x$5C$hDB+xt;{H3RcU+Jlof2=O~lxiw1pLi!_e{@c2n=Mt!Q9tA7 z=&$(c`ZYht{EDA$-}CdoE=j!@|F#C1GW36&pK3IL+`9>c=rl=BjQ#qAFUy;$>OIuM zd8or^M}hay(c>Ihy+VXBi9*w`$D2j&CUjk`E(G9@#ueJ@MUxgL0CVUsx@NZ%N z&RqSk7QlbCfG5lKy9G4=wm@fUz6whGDrom+k?m?gFGz z4nG2~{8x?CM}PIR@TV zrJWrOR@&LoDF4&h(K(VD(#eq;(pixj(&>;I`gI%rJ^elDNln_ZGe>`){Mq;?_4_AN z(ukdM>Hjmi6nrI@0bk2y;IHH|=zF>Rugg_0#=oskri_ulEtfyb_21_#=qeoibrn`& zgnV65l^H^oq9|4r7{yd(gt8QaW0PV;omY4y?HTs1=j3la!~f$M@uw%<8b;EsVQ7pX za%x~gSl1sNVXVOKsNZ*JNMJPVlELq_5dUf?{6`jlsapTM(3H+Kes`f?Dp@=$@;gMP zKSuPO4F4D#`Kvwt&r5!X?KhXCezl~KCHM}d{*O_1)`A}+~ux%e)OHui63vw zUp6d%mnG>~+9M0Q3Q=fx+1LD=T>WD`?$RuM>aS<8;GfSA(9Kh)oc#*$kBWSk`u#X) ir!f91s1#Co7o>=!j+5ALv#eU zOut8S@{dzzSw^G(yUe}*mEz}P3#g9SW~%e{JDEBujobWgr}E{7%(X_8_Xqbv!TFQ8 z@x?vVU3DL;r0%s?%{hgdq7!&fr6!d!oy?D~y9pyMH=^N*Ca^y4#hZ7Gpz5<1;rNm%e8=xp z7&|r;g5N9f>*zuaHfy20_FF-^kvq(Z_u*}-GCW?-ivJ0{fng`x1rxrH;NL3C!DF#5 zic7R$>xQo+xc?J6#~;H}mR0b3+fGm!WUF9ySxcKo_r67CYtzy}gm+=w4QKgARr z`PzlMtHoo|fHn<2(1}0e{dl#>SpIOF0$u)aIJ*8@gKL8qL1M;!;(MY9<_K2c#X}EZ zVM!I#dCjLm)w)#Asu%MAQ>6FElW|PKy6kno0+YJ-O$@F_=|n&PS-)@~{6U@$SLt^w;SE+@HH2lJD1on#eFr z5ROE@+8fOAv;+0eyC+PN^1!QWK9a#aFVqJN?-ed-tt>-x{&%k#eQ_32YI z+Abh^;TOTna2~sI-G>`>nbE3TUG(b^=jHFM>7`NK&Tkj$bC0n}WPRT$h`%*}Cd*vt zP7^D>U$p_fpUdz&f9&b^5C+?3DDr;HfDI>&aO_4mzG>YVxcF}}ZxAa0ozH%B@p5Gv zKh**bg(`8(Q{;is7a=zG7`QFIPhyuYq2uQ4#k=P_@rhSFko1di;D#lCorTcNh2-Y$ zFNjOExX3qWtX%}qcR`){t-TB`2S-uW(P}(S`wV{6m7@FhyhM{rzfpvIhVi3g(D;cK zZogy7fA(0>67h{d&R63I`#kX7;sLEnY3#V(B(y?bT9>z!TC`A@5mJgildE8PavnA} zjpnr_xZ*m1xJLLFmnO-s}CXl|(%0#gV zqo}z4Sn|TpfmVLZUcF$mg;yRvjn}fXgsKmop!*0v zp53Ma5B@o${b~#P@#q~vxRQsFkWK+8;8dTJG1NIcyUB{J91s6Fm8zD;!!Xy{OB=Zz#w>NZXi+7w#lN&xAH- zt6%5I*+a4<#kv$K0;-67mJY3UHRbOt?dWmYF5Gyl73at|v4vVHV6f@JkbX3h_KD9> z5aEM8JEn2F;$0AVeJtO)!j8M>`SH=uvrtam8a;M6qoMQ&dfd4e3+AZPYZ8{+b-4@; z^^xX^C++B+?bhJiG!=5{j^cBtL9qIxNBe3N>GczO)HZtr?YR_0?=G)^H-@QD_o_s& z+h#H`UU3`GFG>YP#Vo+e1Ei|ioKwwgXU(7v(&yQQ6CMZhyptX@<&7*)oqR;ldCZ6h zeHq84B~-c9$lLhG<`SeWY!()ppCaE~-U~i$Q{+dd#o>Cl36xBkNW)$`Vn*`_Hgbg* zfBb2guvT1-^8ig&+j0U^FPflEUjiGqtORrFLvd;SW1KwNfq>3xemC<2lscQk$y=wv zOm9CrWM*UO>li3dIRU?!fLq&!35!xDqj7I5NT%%tx%3)Pl+%Yzy1z+h?N1g`dlU0N zD8Qm;?hsIt4C1k^VDd_dY_`n6P!TKKCEbfk6P`0Q_p>-9xDcmJwWEGrMfjjX0JjS% z*7VmAuc&N6>Job{6{1C_NDg4$qGF5)N2rW;!Og+R_*GensohORwA|2I77jPu$X)hc8TT zLyOT*p|vRGZal%;*f1{8k4syYijpY>7cLZ!gH0dR(wfCKw}4>p=|LFrt2` zpy+ES9Gx|sbUBR1PZ3W1R^f3$XFH0q^!>y=m%MHQfy(b~K`3(C1j>Pr7 zcHC|4RZ``mLlVBa;;zy{9Pw}}pL0$W)+_1Jtg#ZHk(@}PUwp&c^Ueyt8WuV4DJ{Wm zN=x9|m^H%H(Vpmkz8^l!p25G4OMz`!XUIJ77BDkw0p;@+cz+j#6T=E&rQ2LMHeM1w zukK<-A4~9*fjC;lW*`PLcyiE-{_!{>)KnXQib_%FB`evpI2mYMH4J7QOv4aK1DYl! z&$k8r6GRn!MTz61u%|bJ2(P=buXzixY~@8Pxw;;tx0nEYScKJ^C*W~sh z_!yd+u9F@|V~7}di_H(^p&_spPo_kIkMu{jQ$CswEUADJIZ-xi8(`V)By@_E$EG|^ zeiX%%u({`1q-i!RJ8A-UR}$gdl4fSNuN&{r$;YE&4LIwhKKuyKZ9DL1siaLr6ws2&BA##^DgmAmDqZ?W?jQzLlNBM(p8^ti>;$+YW*HYP4D!(A^Wxr5tE z6#qIO-r8BYR z&^$Cxfwvv=V2VgFIlb>0Cf{$vV(mEGnkNNU{~W=xfmTv}t%`KV2QmHG51GG&3QbyS zPd&fJK%;d9{`-1-=vySRNhe2R`N+FyKhupL(-*+^hW5(HLPx&pNeKp*Gz&jYJ40q& ziRA_LZP2?K*V+nSy?u zFG+111*@PxV7_Jqc8ePEk4jTv=Gw)OR%M9Pp!sL`Qo%YFb29CM#7GtO4Ks27RHLcX4#fAVE>S3J(A*u z)AbDK(P0mXXy#&?-Mog;qffx2sug4`r$d4#fv*c?ux~*pEU{RGmNW08lHp9YGg)Di-9=u0k2Z^%$WR4Qd~>`R4|TI~Nw9%Zc%rA@P=ci->Wq`Yc6j$Cu-=!Q;Z^ zJAT04o9D>l3#UQj(p&gkd6L{~Jd2J;UxB~J8)0*<8-%;|fXWId{O{ikSS6pyb`1HM zL-q|g_|Sp6S!{!n3K?p$egW~gX91OWx&#$zXK>U7D^xu73jKz+!|t>-EYL3<#_0fm zea{k2bio zo!$mpN7iFD&4GBEQ;>cCfS`P)g7B3<09ttm@fI_ppVqD9KUMyKO@TJ;&lzBj{#oR& z^ikm^sUj8;VgxZKL{Y*Z1an%h$J_H_%XAN=5a`*I)1CwIW! zlhMp{wlSEek79!s;h;&>X}W&2VEc@TT%a)uxbH`S@zfV2`JOx5Gf*wO8Y~6(Rq9bK zxQ4AHu9e^HCi2vjYos?b4Hd5G@a{1M2d3yu%J|x$J@r>T@PuDIC~r7a(>~x<#t%Qyn^jYKPh2 z{p7>LM7UqQ62&Ah6Qjvmyh~$mWy;(SuyMjEtm?f6%dW4aJGxWZmZ|RaMq&XWPaM&; zUIu)2SAnHZ66D3p^7p&-X{VbDI$xIHTe5C~Pw-_duX-x1DL28r&o{%;8Yh}FEQ1Ur z8$-Qe9A@dI;G5`hGWLf&owF?oBb;L^V-mJtp7ADh=`_GbiGi#6?{nBDmqB!HiHB!f zZNcuuG5Fy$6JwhjS**7bJbTwJup;SfQ%4&_Ec}Ga1FeOPdyYVw=0udVQb5)82XOS^ zF_)AjK5*<9R`7LQQ| z43Cdjq?)r2UR4#>0f!YqIKHAuxb}!X9GgF%57viJn+twuzbzS8xZD$jWUav34fE*e+cyW`F=j)@N+X>X!_i@geV;DEAnwW-k-kK6-=7nib@w;u*XVSI91(9Y85t zSC}{2gfBX60>wvvlUScB2-N%oB3re1@1|_nwf7&f*mW0XSSP{>?-1H4Iu{FE40xd0 zQ(~4L4*kX($+g-)FfAbzXBA`t8}^Z8IVM9Ssb-dCo?vq-5A%Jl;FdAb=(j5r+scRY zgKjo(+(HpPP2D9twc<1JvM+!O8)INx%`Nb?R79QI|QY+jkK4t(=V5^ znT2jc z&}y6miNzVz7l|lI<4_;+7ncX+= zr;AQlqWbSLJiA$pZTojfcqQaJ#%Jc@ZMB1Vu(+5Nzp4iFO%w3xW=-lkV3!X!Z2TWA6!Q)AeE_C zULXp3BOx$Yf{xK14nj#Iy6Z;)xDKv_!1Q`t_O6fp+k6ks?!1l3)zeVZt`XiO>4Eg7 zi|}{4BDXs9kmN_lLx-)UaN_Q09NB2em+P1d#HQJSZ(soF3wnlOQ*|M^K!HCi&SPyB zPb=H>|H1L?zuAM?*{tW#GTbm(0$i_Vu>ET8IO+0Q@<_9oO?_m_zua)c@aHY~IJ5~> zFc)wBTY=i0E}&^4!)s(Rpkwa`P@i!EY=Tpz;Esitah~mblnd8~Rl6AaZrBZMj})_8lfhC`b?8^0 zWPHAIwm@PiYclNcp;?I=!BO1}-CuTM=cFyn>!N_XTXz)~n?z9cAD`j=VoI|>eVZNp*^!OSCf+1Mi(vf> zQ+Ak_vG&vl!a0_oNX|4fCTnbkt2dhBsio1(sQnFGdNz{{8(q(i`fG8SkzG)$SC6xY zKVd7bq~ShUDF}rTaAVypm|V1zrUdCh>ZZN8^6@J+%BGr3PpBugHSGeElw9y#pM+aJ z^?_B$WZ=C&Q6ZpN*rL}AMaOHg+AvedLNqb(!7v;-t`P4R=<^L%5*YU6lG8TFG33b! z+%4x%(2`?O$RRw`+J?@@WKntfZ}Q|+Xr;`>I(MQtb;fMCFstQVZ}=X0%-+%Tr!IwUbu$C ztDeA>BMLaC;j_~o;S<5;8`&@=@dDfb%#b;OBwDA1z{pjX@!&EwOzmCB1O4WZ0rN27 z-+?K(`BD`7&p&}g9$myfYmI`9*J{ySYYBwYS&$dE9#7RM!nmOztaqm&-5427EMjM4 zfZcK&@7e`kd(7xA@e6DmHGyrd-gsVVbX+7HHLV(<_WHQ z|4}&qPlG@+bsuz%+=r)q$I&*UF8F%a4Ccs7@{k>`D?{tIU=}$fd|6kF4o0T*S;STL z>4OgX|2c%e;`d|K*7I;lFW&BL{)5=?3rUw@mv*zwm|B2e?p5slA&XaqoPKPSUB6_-hRJT)PN# z_szw3D>SIaxMWiNK8^|omXO7#5>ef1H>1YdSW~%ncrZl z-_>~DEFYefUB%1;gYkgRN@14g3*tMc0C##D;t{h%kbl}h&W@YLd_J53muG<;+{$YoP+)yyxPc zXdPNvGl|uH+Qd?Ri}2;U&kGZh(goord&gqOzLgkKAelk#q3a%9Xouxm)c@uo5O{Bk~+i@8Et z89`agCG^qLHgafQG=9+6Mf;}*n1_xTjX9G{>ZY1;CcB;_#dSllkq-CK*vr}{44u`a zesl=$g;)KLaA3kn@YT*nz4LeAoU$%euK0)IDU*19W&}8|aYV^4I|R|aHz2vjo!pdq zfI08q2|V_mV((__vjOd9L0ID|@Z7W$_72VSt0@V%B3uL09E$NrKp*BWav&0R8gQ+7L;AcY0n^1Lpyo+G z{0>TmLsM@OR@f<&39Q2Lt3-I!!P5}*Vya-@_Cj(Y@fduq{SV)T=*T=s>O(>I z?qnDU{DlVk>tX5(6PCYg6gNI|4K{^56ZBp6#g5?N)V3m=&AHQpYi#rIblwd9OJog; z{AW%VM@d2B?ikp3T@F_Vd!mbOI?4?a&?=NC15dM@qwoG?)ds_O-j`iu^tN>-JsxJ~CKNf453`5-?wqVPMwAdE(~apk&nkG`Rt}R~#zDM{P_U}h ziQj061$ucUR9<*Nyss_6u7tCoFJ;f325Es|SUc0To-w3fqP(~FDooFAVG8C_e1yL| z?$j$A^0mo0GjKoZcGtlVx4jUtsEMUKxCrOQxWKF{1GuM3jNWlxheUTjj!|~uZqbuL zWvv1j2Za#Zs8)!$D?#75L}Nwa74#Xmlzp7XaJk8QwoE39w0vrU!dd0GW!haZaGlD} z$xvRC+{ThbXX2NsQZz(;KEB+wA2Py+@v^U7=ysuq*-IR74)>ged6qjs*X0r`UNZu8 z|GHysbs#o{6=BK0>zIEc0P|P+gP4Z6@Wz4ReE8#EIHGkLPT3?uO`j(TpGg6AXzf9> z!x=a&UY`uQp2d~tq|kMPISnarMAP4+xtCu*d@C`>a}keNTjybZQ+*Mnj5Gxe|0ptf zy##$K7sVp(x1vRBB;0wB4^_J*>1d1j=(?@~R5jDc59djE$(dJ9cI+VC^K(crX~)f_ z$8ng|3gWilJyb7n0Ufx=EQa>g%zDK7+5kKxwF!lrexTR=By6i=OtM3ckNRx@om0-? zealZ^I4+ctmlCvJ!vvnUHVKm#q_ES*UeM!}2aUqv>=lk<6}^LG1#5vmktHx z*&{H8Y=aNqW%;=FGce3dtvDL0nCpZuLHeQ0H8P{;V)>!s3 z&ICQ=Re9+Q3E{Mv`{2P8MR>1Ti0=&wVbzZD^zs`EJnC?QOwxXY-oCcZPu?w~>&4=+ zO0rKlGSdMpZ1$n;&7+lXTa>8QZCem2u4S_lRj5VpVeq^;hJ9F>3)va-xY!R7{`dKH z>{j22kK+zQ-!5mlyaB9}Fe zLxsg5!Pp-i@Z?YvahfRuJO1r~TDlkQ1wHUo#sCuZhWd^N+3;5{PhfvWnsn9#;jeuq zIBmWro#&qG>?GY^x_SvE4fcLuH_`Q@iTi=`y3^L;@KFZBDlI^AKqV`36tl1 zgt1s5G*w}sCV+>7Tp)Zn-RWiG$g9%Rx>q2p6Dbh_sZbzxapo=k<4j$9^FJalPsQXL*# z`5brdG66Ts^vd$3uh`suSD9z84FugQz~R#_vmbtEVUWpD^{Qc5LcfFT?bptizn%fI z`UE?);RbBG>_iI-Y-sq1a*U~$q3x{^xN`Sj+>=)#G`l*MN(?6BLpeM6PdR}!F$)|s z{61zD1=E`52za(O0wQG*zXX3H6LOSkK!mkWB0!OM=c#eE(WdaNbSKVx^qHk4Ri`mj}(lqf{dsou*&u{lQkIwAWA7LZgN6FIJMBGES)MDoG}8(ZnhLcH)>* z*D+{q3;e72icCKv>gt;Cju5S6TJgb%6dHj!sNaT>{ z1#>|tW{iz(TXDs0ReB`17_*-JzdKOIyS6t_#byL7$@v8fURc0jY@N_MVHkaKUzv~8 zHK21eQ(@wDvYks?+$9y44{4XbKEBpMicSfJ{^ffwEeA zt~O^FuD_#B!-C4;XW=|J+3^!zy4?e9mqL&hjw1FIKgq}pH}2hUNG{19LiM6;SZdQP z7;g2M-TQHl>1vB|6{}fTUDgE;H0+#pV+C_0K`XYWg`>_>aS!*zf6+*7_ zJ~HoHy-@1sWTZ3H_=O>sBHm>Q8oi2vItLE%ZfRtAm^ClOR9N)g9;@zKLgzU-{%ZCn z5?6i(ekfkYXZkI8=MljZovtMF+##q*7{Hm12LxtXQ|T&P&W=tQ!R|f}!D~LD5ZK^@ z(Ho@bqwY7t_Fcu;ievDZ{4P|@a_8GOcncRi*o!YSJ6V}rC=CmW1|ed|*MQ*ZnS4taB-v@Z_gZU;KIKAp|~c^lH+%0?`_s0}VTdUX7_U6{RB6qouX zlWe;Ul|HLQsJq`vFbTVX^FK(_y2eJ5SW-z=rqz%GlTzTuQ)L^SLe@rFGn5;Q0 zf=x>r;8SuG`ES)G+#FUY@Uv`$NJB+VOm|lknY?fx9krJ z!$>+>v)wTHCgS(O2~2#`S!}o)#fEjyfD`T$KyIxp&rFe{84h!()TNC-fNlXX)}(tW=hRnK51@FkKS7w~3+p%*{|aGXj6_+>IAI z?fH`V8vLoBg`m`Y1d;O{Al=6v3qsLIQ6s5`teUROrH1<4^?gTJ zg{v=wu6m5m+U+1a_O>u{+j5*XY8YHyJ%*Ajb)x;lmS*d#;kN0vY}&$noN;&+Xt$ro z{n}Be7IY43k4nSt5#h)ltN|naOK?9nl!(8Uf-U2V(bYH+f~J`8oeLgg_?t&8LZg=j zxBVc8td2t3?yETJrAs1D>|AXO&wo*6C`DDT7kU3dn{HMA zjJiYF#5#$;cpx$k1AkwKd$RRV{l1@s8}Gtx6U(9e(_9oeIGRV<)WaON*=&3KEGShN zMYoLF#5}xDV%RV#{#!YiTg+>N2UlM}SzDgaqP?G-I6ImdEWOGkM7NL$yK?cCdly8{ zdBSX;y#Pf#3kMqJKxOq^?8;lj`fdlI_O8Ep?ByomB}D}qyYw|KGmFNR&#G{hs0Y2J z;fyug?jbuo#EPuliSG{TQybf2qPul6XuO;~l<8a>$^yJ-W5oiRpsJ0M3rw(Xc?`>( z-U{}EvzJP7JA}>>=3;7+Y+<2-=Xq>10-Pf63;P?wA*9qw zAiwS>3bQk~=$9zGspd|njc$fhU)$iQxf5S?cmrfU3?+YCdfD|?56Hpi#_*%d7<#X% zLG+Fz%%jH$)1y?m&yN@=`+5c);>`GyW7$IAM~fiBPl7)a{Q+pboD82iiV74`AYoM` zxcXdz2IW*xN;M{Y$2Ppc%SrHd5A=vEXLo~63eUc)g$bYH*wrm@STNK}zg80i=f?_k z^7bB(tec0%V^+ebxe_$__9ZA?y#>occH@6<>x7$w@8FhzL#Tzj*yx{|p|mp(WM}>) z7sx+!3{wGVK^*=|v!T%y#e!=NZ?NptB24|So`t=-0ypF)&;w<6Ah{s>Hcf*u<9EK zQ*+!+>_e^4Y&IH%4-+){09RarwzrbX71=Kx?C$X2-2wlF|LH}h*Uc0jZ zI_HOwv-x4LZRQPdU6qOkN+R%aeiQDzB?0i@B)ZJd5JKEp|oG zeRk<2c=~L3JNSaB?OF&nhV8gu`g52+jJlig_^>K{}} zKx2gD5NGJfM=z4Y@TE<{eS5U=gUUTYe_Hc79mG>%8BCJD1+BNOXpTl5 z>AhVE#Qy`i`o{$tG>YNY0!dmHAVuYOf5XylQ&N|?TWz9|!yYpYr zw|@d(uj+_itJ*NO(+Gw~t!EYwx=8;^H$MK~2qEkP%mp?VXYkX!aPx z%9r4>{a5kbAA6K^QRCO{>>wJ!t?a$#b}YMK1|iFwh)aW_~O1)>v)fOqCSo?DH@d*~pOP&sQ zu1L{M6Kq(sV>O!?;6tYW*bf~lJ=kr>*!cMunf1H|tezPG#hR}L;puPD^xAJ`b^8>! zo{1$+yC1`ezJq9d%#qr(1;B$jpV-KcPXxCf`tsQ#FVX4dOm60L8ryoO@R*Q7Hf)^* zw~O(|Ah#M;0R*`)1Ye3&V_2;tCR%Gy?eOzNx=Rria&O?L={r!{Ef6j?7C@xcNu1<= zjWqu&Kut+WYL~x?b{ucR25TicO7Si zjkx8|Ei_AgOww*f^B~FVf&*f=$iFdL;j!;a{F52L&-6aP03}VhUt)`ID}+qv?OWFF zSq;_acHlToXJ&KiHq-bNLf$^jB_Z>-z^o(Duyw+6u(J?YLqyoEq%KMb@Vyh*zb$>a#x7z*cG(fdO7r0 z*bD7DBgv9DMIL)bAE$p;rn$SL$*PPh^6ckADt)&X)-93c=YIC${FVXYbJ`lFDn#Jf zp?56#Cp5?`=-w>=CDXDE!lk0F6k2rsS!H~s;dy%S)=o5#%b(gEz%@MR}9%cH?Q zYi{m&8yn~r43!^`cc*;>i@afct?(PHaea=5CU=lQsK!^h7jf0DLqj}sD><-f4Ww;q zggFTgaQxv^D7^3sN1_z%kqrl#(HhL-Sw3zy2*tquUl=uZ5oC^fjn?<;u)Z@6)P2*~ z@K_;yp4toJR%r8@ce8QeODry$k%m3t*O-&j0NyWpMMi3##y|Pqba&Eo!M|27GMSA8 zuhTE!+wVj?Dl(g@f6HeIVF4KAc?kX;*oSYs2U+2;2>Nc{Q&6@%$_Dq@VVTZ(;fTTW z=;>a~7T;{$`)&b@p%esl*YR z#*W74KShMuc3%8No5WB*YzlHs$5_g>qmUvO2yaD;F)}I^{<wiQlRs#Bp zx#*pF9HYO-33YN#VpHp3vNk~p&emmv&XG#;bEgHYI@t_Z{17h}ok8;Wz z1<7OeAwB&~kEgw1!3VN25$=%*?F!Os2V*taShd}rPi_7D3*qHXFR zDRoHO3zFc8!`;f%F$*ELX9bE%EAv@bj>CctnK;`&5NZc&NLg72QT%I$wwaUBJvE=m zPgw^u=eFVB2^P4v$sFg|r$YA@Lz<^ljmjzqNXT+u(0zWIy%)Z~9q&fd*(M$MhaE)m zqW8kRm%7ktQzFrrox?9bv1FM)?&7f_=6BY(Ntjz%4rh-WvyFM%f&Ujfl(D80<5p9$ zAlwUl@*Rcp&ySEttKzXT%MnWa2)tkT9gW&Y!Hg;Hw7q&Ou9VC$Ce;g#Uv%K9ra?xm7K26q3Apjm3D-w2pxahm#SJ?ZAeCrvZ{HunK*2^V8q~rd zF*O+eGz#>6o3ZMy4(P~sNonEy;F1V>KPDtuK+M zj?wV?L@DT1$xu8xR(71Hva%43G!lmmZKVsco~Bu7OTRE5x22Zrxv!X%Rx7t z?HJ}2O;pb|V@GEi7+k#~ocr)O{L~-Cim50K9TKYO1BY48sSEhZ&5~A^r-5GL7f{+W zhseF01qWi~=<4vJxX9)g#{8WIi}zNMy39ndIwirzIgMd&W41v^Nid8H8^JWb=d;^o z?y&#AvqIuCg{JdJ2zAwi84309;Lq{Od#w}k+MFF+MAw#n?v%mlM33)IYQwQ>&0$q< zFzG1O#(8TVGwtq1l+#&*`zj|2MjSp0Q!STa(es;V<8up*iXzZG(i^Y6*o4)pL9k+v z1qSI>vrRsd+&v&0W|2~6y73k~$(;tj&pI%5^RxO)MByXHscD1*!GW=aokHd#}*i{lkJ=>~& zW>vS8N!mpV&&uCp)+!&Fzwk9)elCW$L#l}N$}4bq_aFwC?88~bDrDn{cMv}L7KTb{ z&~iyx2nx~Wu6jEK z8XNKD^Bi*LP7~3;8$um6G5q&OgnhlT9zI{%4Qclpp+T=o_&}@~sMH@;;8O#)X8mKo z@{T}|+$$)K^nf@o1Nxu!R5s7J5m|mDIM1#_F)-lMro2X#_1V~zsRE0JX13aD4O~7Z z0TNE$X90yTNbL!K-Wgbfi@R5niWR0@m}`pGOM_8%p`G)^BPzlTk<-yL!4$)e$k6+h z7PQ!JXs%Vgfu#cn`0!0)_%ukCS7-clQlA;ijIQ;tr*RbTS}h~f8h4=bb{Dw3CKnUF zx4?wzVVI$4%h&8qgg*09s9s4i;fMxKoY=*3oo7Sw$%SzG>`y^~-D&*VC`MQMufnPb zNjRmEM80=K;;mn4zqu$p#V>3@~9#Eeu~#f@hXCV&k+c&|SI@HqD;n zbVyzw`qIYGP`)4K&-s#^^P531&jA(h#V~d6tyr;KlTk5b_8YG` zcwUqST}wOI)z|>DMfZY-jx}yxe22(eiea)+KHD?819y$lg~QqxU{vR6mi zm)3-X+r-J3wj!A+nQQ``X%+Y~Gy&JkO$1xF0NVdmz;cqe!sTmHbfvQ#9quxRCQlCK z(mm$9zoHAKNH~z=6<#Rnc>wM%zAqRNrApH*GT_3ICdiquLI>U-f{b~L$c#0>{s-4U zqF5RhEOH{DL*91Ng9UJGpaN8Mu0z@5=@1reNgIF7h3ZP~=mH`~Vf5!z+*0arO{;xMP_LwkPixx1Hu#WM^SHDQjk z_Yh0p8rZ(57?!XHc(P+E4|rw98>7|_`LZu8-R2Nmn@3oqULM)I-WMKc=fpm!nO&ns>jh1Wh+-1s0ZY_YZR(D{{P;TAg>;_52gw9pIfCE`u z1-NA_J$ytT&B8~rGh0f8yUSwXwcH=_(6&l2Zi*(4EGmYk3ni>isS)dMM$@tN7jT&F zF`{pKlsFId-Rrif)1zs_>8*+w$Q^S6L#ZsDygkG*FLK3iej}O9sBbtvWB|na5>aUC zLOd=uuq6#=;Aw3>j1)PDlbUX^aw0~RmJ~yt*IitDp&CplJ|n-rOyobyj-cxIpLl6$ zCCgMBL1yHBAn7AZ@%zh1U~9CN>U+!`A{?Xzg&*2s>9b>SLV7#QS+p3YnmOaxtJmSJ zaU7P+{|1*VQikSF2AlBeH90xq0Hl@t!Ci%gU^TTL-p@OV-o}-v+#AfUuem^ylXLNd zXaQbZxEb#}F$Cq{cW~oEGK`It!LA1xn5(^vdfs(Lr(<96!@oSD?HUb9FUOK-g*|Bh z^&D=myM-mK4?)Sri0Yoy!e?pXj2@WEQrlPZo~@tI?@=snt?-0bt{LE_AB&O>l61ay zCOS1Qqt7~y!NR>i$n)XbaIbEIaMQ(jY>kVDl8v`n<-+^8u=z5Ujh{gb#2=GCMe@}A z)ji>ND;H>Kk7eI7J`jsh({cBp5z%<~mRWce!(r)EoDwTbw)PYsb zwxUvaK>8`JTor)>)?PFzAy#+=;%R8^QHfZH%d1glWPT!qJj9*d+g5&=B_tm9=(~ zKQ~XoxnfOPYTpo1skZZvg0}7eJ_J6FG94gZ=ntOyj^TnBH}o@weAVuw4L- z+WZ9Dcb$N-2h>SfMFi}tvElVSg=E%^TolbyBRhj^fZ9!?V$$Mtdh`R(aT4Y8gEH}M z^=PO{?1HCqx-hnX52{;w;z(ODI2nSRaY>**^E&tC9KO# zi?$34Ld*2!Xl<~6D7S5ZI@Q0#w5}gNRfG-Q8^7bM)?-4d=la81H&0Btl#Xgf=V4Zb zGk9g)!nIyMNt;i(P^-olOVsqZ;ssMact#0#lnvc?5>JM(m$9tRVLUzI>p-_@NTa^V zMs$iQBZK#SY5YAqzM?r5S6YtdaI%yv3<)Df!xnd~8~8o^?=^!J2Qr0y&NldM*m<~J^_PkG6p*9N2ZiDZ>U_J|5Rdvd z4E>9QV6bo|i@CFt&k!Gr*GG%u)ODe-{Q6PQFOvt6dS#SdDoKa!L&&(FgZILq~|9i8<5ObF;3X;fP7QNt2>&h`xOLZw*o)bGAmjc(XI#ZyUSa z!3hfrhu;bE2WlkIV+oR*t0_rJ{)qVe#Tv;k+xhKNw9+T-+A~Svw_?G-s&@X!3}?al z#KK9N+9dIHFP5~Wt+bWX4if};5`s*%5V0!!BmSghA=nYhm)KNr+TSYb3#2CTi$N#8 z!IS3!IHZ zB^TH~1=Fks#Lq?N1>4jpfq}esyYJUd@$7CO5&2GX;b-KEJu0UOYP$RR0qxU8_p;c6 z12dE))TpE6&dg51mL08<*+aRKioSYrjl)jKvKz0(^rwr0hkIjrgNCgV{{B``L|TKW zac_&nGa+$DB4@e;514dM*(4?$N@ zzG%pGiEY^+TT+y6D>3sg<(r28o%D4J1PzK)Y%gy~;72YzAg+3_c5)vz@r!<(kjyzh zAifxrA&|+q$uHNfpVZ-4(zPc+ylr8qV3ENI!SQb8c8mR z#rN3f1haD?-D-t>QJ zwx1_!bvDh@5+pV<+J8izgilCKT3+Bb`DOy(x1+x~8l6Dw#{ z^6gt@wSUU3l049gmpFJ-3dB^L;OV<#l0KTtUlXj;ej#XWd*x)lY@n5G`)ZdA|Alg) zXz*K|_>J5F@%;~zSofm>K2Kdm(%d|mzmKVta7&fiNp6M2tJ+2MdZb&>8sOP(H}i1Z?$~!zNTk3POI} z;>Sy`6^QdDu^2~<_FpO&1R?%elGrqL$)ab9?VpCO3uK7bj2#US-k)5=!^IG>4|Qt z8jH&}@M*bOru-<%NUS1bBsP6#D0VPY6Yo(q^^au0lq;`2eJ^fTKPQKX; z%HmwsgS> zGj5TOuj}%I^b;<4v%(A_m=$ok0fFV{*$IvDm}&7tf+w zk>7MhSzJZui|oel@Dv>_L{Dgc{>WpdND0b{&99a6RDTWf6xYq*Q$D*zvkoVV9)>6J zcdT|2mpgnBok>0*N;{cB$E0i)ou0CmpZwC9&ms=;>)P$bH)6^}r8g#cpH?XFwdN<2 zyT=U0URRCyOBV;B@L%Ed+e2PJEzy%)1^3W#CLam4PdjP#;$-+z-ken1s!Rh7JY;G#{CzRrTC8ODdspDi`_}Z3zVh)Z=`vEn z^OLM1%i$JL{F4k`wT>S+VqncT|Kd%r*1Ca**hZrFj19ae|K!9b9rogVGi`XuwkPS+ zYCgOJ!aC92?j@q&hI^u5jk)5;A|rmmhNHCqlOfTAylmbFaXMZR$`u7{99C+4QoBd^2gyun;LeeHU3my0i1<6KFW z_ZrZt)jee8VJ-=7+rW`y@^Ft{HTG>bByTWO$dX0_+9 z;hB%>OuIQO_53gv@Q2cSmF45te zaiTrfn&9=(yV$A3od4yz3xAfWs>s*Qo>%m7ov0)`g*U%YgTJN7kf;7+gm*AMmMk{A z3Y70;@|JFVPWGCclb?fEinibU!83fo;m(k3p|zqf!N_&#*rLsc{CQ&$yjk^x{Kl(7 zCQB;FyMe~!In6sXBkUhZRL*AVub4q859v>(Xg_uhTJi8Ob zdlVSrfZvz+Z@&fbb57QZ6{G(0V$45_#gUHuRT}pEq1s{b!^S;gJr7#^mHmZBRWSG! zhm<9eqBMS5vl9Q({{4K_Z{|D&xk*mUAGRbkSWVI;vX{8Ln<9AsRENL4sYiU(q*u)A z-zgG-GX8z5$Ksobc`&qXj!5;h5bw*0#rIY3kjUE@l(|jkz58Q;R*wkDgyMMkeZ_py z!7cyD$FhaIhQWK}zu#+M_ij(ZzQ~OpKHo%to4ua5(MK1?w-sTH8`J2Q8?EWbmD#vB zr3cQg*hbGeID=Hm?!fi$T<92fwD5Y77B5mri;}M&V_1$)21l2X&9#x+OwRl~hT&U% z>vEwb;~ikzY#T{o&q)7brK)h6wYTCCYwnek3^U$TmZW<&(?n&JO~sBytg^PhET?D3 zZH5{d)(p94Rv$HLncYQw%%s)&O!s0bmfMxxj888ut&ia-=61u!j6+R9R?4f+F&<}Y zvzC8|wSKHv%iL^X%1oZ$X7kBmiq&uLF&ka)+sxHBv#g&7cQE+}YMI*_nasyYwk)Z# zm(~{oPcd76L8jFyO{Tr79V^*?BWo-#%jQu&wrMH4z!(Z+FzSFxpy?Je_wom>8pyLNCBciPr^PTgtD z&VDz-M9*^Ag+~&&)QKJ3hqn;-d8-Ha??x@o!M*Jq8E4jW*XX<$l*$#p<15>Uuq~(YWNu{GtAk(Y9XN=Cwv0t8@X?>6+xW5yQGedVKG1?v)vGY3;*$YGOaHNtpa-X{J+17idSx+y= zv%ihAIWGM>*qmRZXXb@ua43ffc9QpyO?}l8_9>To#?rHDHYUM~ zSuphhW53ZUW+|`A#)>ipDA?Z<92uO*NBg!^>lCSz*dD z7*=Oq9*$)_Si6K-IAeuP`oD+P!&U~29f^9(zWd)85>tD&WW0^}IRdb|swNiH6`f>R z3@>J?&D(CZu}8wlcRp*i^LH~tv-&AVtf$PBb$Gxom}y5St}SJ1p3>*qr%z!oH(kKt zGtaR)bbfP{2TeE&KQyvy&&6|FaV6_0p}as<4A&#~IU8 z^|^m(>&=9B8kl#@n^+3cPK?!9iCxq_&Q>t0Vs&j)W#4}t%yK$*ih1d> zAM+h=D(6a27CS_t-DXX!A5&LFo%KVunzg@fI(v$2KKuH<6APaOE3=e+OqiZr4(CJW zK`U3~OqSh-sjP(-fsEUB)|@?S7qL&Td}TBBpB4Mdm=iO-lgS~EKDE-3mg6op&|yQ> zLmc+?Ib5)vVtaD0vIpw|7^%to7|Hg=+<6Yc+@`5g?3rJevio<&u%9=)WFMA%;{-eE zvR%DSay~@T%zlP~jSPIpKEO_81>7@Waubr-p27ysvBo7<^x~7u8*mq+cR-pU^(Bqj zk>zdkZq_l@rz=L*W9kMhZi5&2NFInO1NcGHbky7#ZgzNB_8zpaJgE9lZb8mr5IDLX|rQ0_c79AI&9*6_p;c(6Rg~>{9ye_ zH?msTL9i0MhHW0Lw_sn*Xl1!8$a5->9%a!ll5OU(R$J|5oMwho9M*?FlX`lyhs`a0 zIYwGwKVxy=X6CI>4My_jc*gR;i%jX_35G&Y5##5LeaxDJi>)ixOR>i|t2mc>9f*UH z9q{%?HVS&a9ew?LhH6@#PCeas1G#!Wr#!#dVDY+(RIrl)>MREBr9X+A-DkOPv_^@0?%ANd!-k58h(MlhtpJp* z1%}I{Vaw&S-2DOymF`SHomZEE3T-ny?YlgF>M^}7+}8(9`4a#%wa%lk=}vfrT!V|t z?xH2BO!&DfmTPv&9A{-7LyUnj;j#P0#FEt8sP*cV>ebWd>gQjW(TU`xmli= z(D1@{ov+~NYp2MO3IX{wO`4R-aH3yp&c!i7pW)0y-H?&p2XbHj!4`9i$WDe0RqGN9 zls|eCO4aeGw%C~1*LIbo%(4W1Z-*!|BPKFlT}WJH8G$i_wQWVBJn*c)4w=PWCe~KO zaW(ZcQPA~9!rtA1Njy|V--dp06^$Qq4ZE%)t;|L=1GIB1zYS4~KEx2ZH;$l_uga0o zxk_l;6AIQj6ru{R0BW`LVY^iCqey^;u0Th zbm0Es6y#EwN$9QpLiv!(K={QTV!E3lp8MrE$e3MBoL4ZVguCJp%Qzjlir0d%MM4zu z_%rBI^hVP~Gthy69QbAbL+;d_v#?d!6%cuQi|}-A0bUoHj*{ps^iJU2Q5vescgc8*a1{Rs1o^$s(cV_MfhZZ>l#kEmzAWjbXJ>G|_)}DZ? z+CG8}qub!w^;?C9J*E(md8z2htD`t%o-^!x*2s~Xww$t(ZA4w2BJS918`!x(4QF3y zM$AA|BQNIwf+UGNGy)2D>j&M2a; zTN9wj`!^LlDvMuj^@j6TW>e?atOWFzSR!=AQCz%rzVO2F(_l7TN0|&<;l7v(VU*`A zsI%n^*jPJEc@=B~Vc`>COZRbMR*4(-e{BhGna_Yz--g0?^#f>%jXs`#MhUOH@Cf`4 z3dbq|%W$YvD&@RxIl?j&z~)+bXQvC6m?5mk+JsXM-+%!Ink`-*PGA@B*@(GFnHu@d zKxu~~(WewgNQnNz<-|9X&HjMqo)gpCY(uc)#}lN_@};Q#`Y$T+51Y1LWl1kIs3SYt zrAR_5hg|S#CC*8W#bs1HJa90B9{hX|RKGrs--SnE<&7WlIpe!zl*dl8Z=Dt?rL~im z`WT0vrcNQBR(yiRw~|3gI-o6-I!U*l8tz?1Yp8Gfh9>*mz%G2R-ksXGe;i~f&%o=%g~Vdahu@c!qF;xU zvFVK*pl~;n5*=2dp2)`{xabZ_Gn)`j`EnWDj*7;kU$y}EURBJ$X#=7j86ywhyWrc; zG?<}!86CRd&xHjtDC)I5Z1I|f-p{^=7VhweHPI)~bTGM|J>C+oDw)`MwV!Y#?KMy? z3PGu>E8C{@?11-DOd!8l1g?sB$mGo?Q0=J*yA9_Ezh&lOjy?yz*v^C8LN|CIMu_4z z11#SpgSXC*hWqV;@cW-`cs#nEs=qFU8nW(yKcouQi2M&-(Yb=n{M~SY{x_I^NRK-G z%r00BVI_>5Lq1(1Ou&=9ta0G8r z$wUDNy7*XVMN{CU-ePnvVj5m?JOT8EWP{P;G%DIy08FA(F~za~EiV#@_yTt%oS06q zRz@JPu$*d|z72q@KL|Y7fl6iP;-7gf;7`m4LgAYeaH)1gI}UN*`!-VU_oG3{-$bry>n5;t`y-)4YbTg&6Gw2@7omhT#o!Y^1x%}1 z1_r&`ICTl@DE+M4$o&3x6mvr!>@uUNg(1&~FjP)4|B2AX_0iP#nI_2d`x3Ne&vIhK z@)NaZSutvv{X{rtQ3}|ywjT{H&g05`s0NqwTexkHVhAvB(GY4pSYycl8%up;$B7P>!q?#8WC9aqp z1<5-AI=3p36%tmpwQ+Z~CxIEyH=*2{P^jy8gv+%zC&qCdwd|eNBtsGrvq#SnDSau#J4p>? zyDtwp8+~sTBxMrG{Q<E_a;7v8v^By&xFs&6X2|# zNWjrfG<~K$ygT@g%O4*AAB}0sq0x)jD(OWE7x`H9E*JcAv7#7J--s_Rl}O==gc{zz z27J$)hM%(66Q(;ZQ``Mz(W!$I;Bk3Z+pc+^sV&FvQx1z=g@^Wt(Yya9_v0uB{dJh& znlAiGnYNUoNC$atQpYXe6Nb@&r%6b`dKzxH(M&`$vcbEs44{-0hU^|{z|0qNaA@sP zuC(rB;%2W8(%Vu8EKB-;a)K+|eR&j+yEbB%S$b{jv$Wxt@J2%RZnxTQ;~7pp?8J#Og5t#HiSJW3o(SHt0v zYZ0xZjuQLE;o0$6oUp_O&$MeqM7am0@l`-{e>{l{(x!qpcL$+=T@BihRzX#sa|h?H zeF4sbaPI4VA2i;ZhdVw`71ql3pqRwX)ag5`Kw66uN-HYooIa}{j`Q?q)CP-NKy%c;&A)L6H2rD{{`|0_^80g< zS;u~$_0|uotl5h@^DRNW*B7cw{sj2$E`_N#MmVj}4sX3_fKi?ooV%=#TWK>3FL)g_ z+5AP7+OeaQs8ySdQ~p|_huRb5jDJpa_3nPGbkv3nU@4QDnNM)z^c1-D-Vp7v-3)GM zVUWw>m(q5jE9ju#mvG&yTX17=GW^f=5u@=?L<_1y=NTZ!Uo_6Jnhu&ngJAs9n8s|V&2m|h6PXX@tVv*UQF;w_fEL0z8A__dOQ`_tMxr&T2 zFz`?hEnno0u3o!H44jIk_UM+PH$Rq9;pe$%C99RXE|US?op;1WDNm^hzX%kyBZ*+b zM(X)!8+esE54Mr#(c$AyIHlfA)a(|TC=3LVqW#A7xn}|?sChlFAp(=JRCft{;ygSYa?)Z}mPKPpRi(3sS z(LtSTC{)8onmnMw`xE%9!#Fn8`VJon7eI9}mo$46PklJ|E!-U8=pEZT;~gRghGOqyq{FOL!9K-7G>U)+`g=CtA_pi)*NE zVnF!1cQ&_EC`RRa43Cis3Q~k^ATyl_WZz zbcGb#?;u_L6X?JN&h!S8PI9N%fF854;vIh#MDsiE&}%G?kt-{9k-caZuXkbzU1hnF z4m#q(%X3>S8YNooYN2`Ddi}Jty8w3w6=fxBYlKubbR=50MJ1 zl3`wG1HIwdC-P230aY(hp#3byh*cjxfhS}Qa&6hfJ@h^fY}}p*W~jD7)t7Oonp=a5 zmp_9Wp1Q*G7xEx+K7`P&uV~di*@66PYVeiU`7q$RDdm$PjRuoTaRfHDTS%k3jn;4-N;Kqq}S!a6{n* zm_H>4$UpRh6_yUb#dj0lcmEe0ndeOW>WhGSyN{!u>gC|VwN7G znc!Wj7fzE6M*1gjfd}#wj`Xbs))ASg;KmI!@Ax_Bwu>*^o!o)E&w3zWY%d&J5QE=b zw#Un~UZI)!7m0iK)6uRmCbqq3L42#uh8fHN91%id&iouaZ+#+YzPSqi1`A=y6*c^( zq5}SGNQR}-^2GA5d%&%m6X^YOJE$D8114Yo0Vjgvp=*pX?RC(L8xh%CLKWh0VU zld}c5J-Y;sPwzp~r5A|SWgH;)bQ_SYIi-;7?!?{IG)i#D!JpQ@Cpqa)!QuaUah{<* z7Vmlk_1@kBW}9hjYVC!0?7dIsC?2GZjx40_={I1+?nCE3?82MfZ^PC1C*bmhcS+WH zNS8nH!!0+yL5pN-vaE9r2>Yi*ACP^CWGZXhOxhB0{q1@xXUY`F-E|JhUWnr!sJVkw zRgVFepj9xQ(T=`vlf{;HD$vd(75&omg~9H1)L>XSmHTQl$c|cv(s#cG_GLPtdfygw z{j3Jc-fRFJpU1(pKmQ@)&}j7UY8v{ttQ`!_DTctqlW;6o#4CR4!}pQhl;3w#s=&+= zs5pKDbJW&jWuGeUct94SuC4*+&)DM!^(0bGe+#zPUL~@13GDoJ9_k1`Mkzmv19#P} z;6zIa8d@Z3{`7S>xHGJe)?R6&%mPAT4ZH-#o5rXs2drTkvlMOJ{g?P~whJ<5Cli*c z2e9FXY`9~Y8u!yX8c#h@h34y>K-P61z@=anWN>Oep?h{Ku`RP7I+?ve9S6=~XF>!L zyn>dIjb$yoFNFALptZxQHyIRN%z29!}fPD3v@eP5RR$pqE;57G_l}C0&1paKBtXDLi^@ z3s(JWOd9X0N3Dbro@r}FJF0hLCrXu^)8~NHn*8ts%}wO0vQgZV53qBIB{u9X#yJru zaIL=zz4W;OCWci>gFSgj`qxxS7?cCMUM&N3LcWyI2RP7xT_upf=j<=!_WtDRLDUS+~vF;p58l~5{o$S&4Wd#FKw7`UC5vm z>VvR)-Yej4-9$9{lu#>T8j;ccI1qbS5iX)1qKPx|0F*32Dh4w_vh-1yeL0`2xco6v zYVja0RSCJz(kft6U=mVItU(GL@6h^>7pT7%6|uRJ25SFz4bOYA83h;?z=G$u@YHXU zr@SQOVAm@HL_T*rA<3$ONTnOLXlde3y+Dwg*^Z`7n86uGt)ZrMEvkKy3sZ_71EpO` zgtuA@SbvH@VK0^wm3ejOSG*UL@75s7`7M+QLkV8I;t5vdxq_TR0$e-9hHG!-aWiJK zz-qA~d@<&PN{=C^h#G-&Nh}d}sS)+~<$|X^71(DMg@3=u2OJQM^g8bWrew0Efm;q< zd|(^i@F4*+4OhebwH!iq=RUmsd;}$s*-3dE+5vXn+>OI~uY!Q^+wg7de*DBK3DvG+ z(e776xW^wYA^RlrCjZk*EH$V`7ys_W&oVT~2=5Ir?RhlmKCuL!`^ml~D>fV$+73Z)K9F z+++w-w{j{eZVj+m{+NpC4g~6^4xHFk&0x{BN~(849ep+_13Gi2phbIyfOZ-NMMkNp z<8=wuv3(Y}dHN%AF`kYM2FHl8K^{6YF&6-4Ec!i(iFw3~A=#CusrO%c3HgI^;C%X5 z6x?(P&C*y;6W_X+>~3_c1r_Q!T`+HzD3&6xK79gF%lL!R!^Tu(u%*YzT}%vreX?hK{NDu=ptm zqWY;ltMWJvk%~w-s)c_XUIl+@PC@2_`FP8NP$K4f8gL5y!maN5F1!;m177soiZ|zK zV9#n5k@2!sM3oj>Caw?Hz%UZOHqgc30&YuG}S`|z2BMx@(nD&e%EPu{?-=Z+gSA7I$GAq&NuTI3- z4nJVOtc|j)y+oZ_6Uo}zSzdP^I&7J+=n)XUGv*taaSQ`x99PU9IR5ugz zZd!w#Iyux0kqKIoDFsKGC7|WtC}pUA9f+$8&|8IMVExt==2|NdT&2J0f}avh>3&FP zSC_&W@1LWCkuLbuyb^H5Q5SvsiqMv0CsAAdT+H6P07X5pfj1Yu#hOcG!8xn#`10B# z!1>o|)L3T+1Kgj&lF`dKfAr0;uhSK^L7@5=<=l$>mMZG4-FjDF|Zp2`~3luTU;Dcx*osTDMM@8 zCSZSi4f3bQXW`S|G@f^2A6oXti28D_8lAhd5FIN|gVye9ykOmM`svV3ta3MyzQ5ZU zUVC>LR_)1!-&K0(QNN?Oa>^>W&QgmkOZY9^5fca&?KXk(EF-A=xtYG2v4{+dAAwba z$*@a98sY=Wu+DfT^gdgR;O;WoihBZXe(w$fp9W!((HSfnZXW< zAx&J8QA0K+Q(#H)JY4*F0s18(KxStokk>cBPtBvSzmpVfIyXcpyiEnh+D%A(&RN_O zv6!+vD+6c7l%tl8X3ASG1Ft;R09G#5g62t=(4D9n;)Ah0F#UcU6l_t0FH1{W#cOBd zS10S>%$$De$CAa6vXVwull8#md!Nzk2W{Bq;xuUggJ#WJAPeRWYvI+Z%R#2;4ESh$ z5>~0!1UB!}PQ-Y5{Q2q_moAdSpS!*iMdvMruUzu6XY+jAQ~3ZXy@~?0Tlb&< zLJlwMz+85M5C(SG;ccOjc- zdcB+u?bO!LE8L7oi(PD?ihU05Hakq6^_GG?7fz!g<|gj7Ma$8@z2V@(&QN&ZMG<=6 zFzIWs1(13WN`=}6!*umZ>aFM!5!>qLMVs^Q1XeR;IX63b4r2!w!JuKrUjCVr>MeS7c>$29OOD(5f*Fa5`Dq}q|AsF>X+0| zPv_l2ykax7_Tec?x^)&(mHvl_nMvFoUhlXYorjU%{K3|rE28o4YJF5wtO2g3l>vuk zVW>qf92VUOgO@JKqfFlrkW*HGY8B<-xqrK0!zMp?A=(apTg}6EikF1Wh3nwd5L2)+ z$eLK;D2MV43?bj)5wcJC5AH6N#?|s&$fLf7EKJ-6PhKmdAE+{*)AKScXmut#8UxAF z>I}M@Z`)_^*}{NU_f(^6jc4KC%?@PoqKCBq7C(+{nK7aKPeP%ehe4yr8l8@Oz|oq2 z98EhqKon{_6Uo0!k(JVAVs+*wRQ3G_GGWGp1bP;Abp*CU(bTUKN`V*YwZaOqhPRE^Ch)Ay#+wKhv0pJEir!jE}Hhq9u}CZazhtR z=0Etl$ft#-j?m|kF4L5{vAzX;T-QqN{G5f6^$Y4h$Mx`$Oe6+U1IX*y5*&2qeoL(E zX~J~x26!Jv!K!OwWc=A2EEt%KHEl0~XKlZM?Uy9fcO1YB@fW}!cW65o-$g{Zw{mf( zED^Tl9NHvj4aKqroDeN#49G=q8q4IUnD|TJZ2EW0t3RIFUgSY4HY|ytPDYLZ9&HeFcA<#P6iF6+s5TB>l zBL0+Aiqxyab*g5d;8_MFhWzoRTu-bBqVWf_WOQl{35-_`g1xz$;nqFM*h4Rn=qF1l zBSthZmVF3RpRL0GG_C=ekQ7*AtWG^wi$w;k2=wIKe_Ts@89Z%IILz8-3jFAF5Q_psYzeqmEVq+&>|S!a!OL(DLJju2`qEr#6+iP(5V zhz@QMuo@IH{o@+K5|^=n1UNS#bEj9bFLxB3l{H_M{(V&kse)wR?e=Vg2>-!uwg4z zdvybTFhw7Q-L`~%j>oXW)oS$HoWMVPGq?vjwsQ|u+Q2z?cEE~>Ce*K*Lfom}hb&Es zfl>PvptE=%N=T7~5>Gd<fsW(s{^Xy#`UY!Vw8{dO`PhBaZQjnW)G<27nkQ z;py2!RLkEYB=ZtL*svP4Nt^|I`!hhkcQ0jGZ3&66O<+og8d{_6jp{vwM3g|9Ug*f@wrbfV}vjtt&g_FTHFYc}uV;7$6_UmcOGmIbfqc@LQ* zw}JNNg}uGFuEu=mA0x4={AT)zy!O2Y_W4g2N!teTZoe_4-fH_HN1bXS?Ug@rf9}L>iC5y*-;e{YRmZ4c z9RZNru1y3tsGykdm%t9Wk0A8|2fg{?PJEIvr7r!hMw`#{P%C$D26u`MqR}m?)ZC7-}gWP zOCCntw*ZU8(|`kM0vAb#15aZSG|mYJSO2TRQoTx~jNuE=$vFjYZVW~`hxZYCS}x%6 z%Tn~em|d{(zCSuNr5#*&?T-Wh&ckl8ogmTl8crF~N5d3EkN&%l+{%_huC@s^FO5gM z-Rgy<_Wxl-^~>X5n$~#vHH3n<4N=wG!qKg&)wt~TW$y0ly09f=G5)8r7qLCgCFnX=VuU5X`J*MU6(sRQ;*jfXaa_wmZG zcDC!w4A>jt1h{MbiJI#woSbcwF~{B!V6{F%RJXE?R53h6j_8%pyZX1%k<fO^4uQJ7dh3Y(fewh=`poTJmyEz z-l5vOjUC^~-dq>nSgNvU?dS-;_q?36bMGSQ-#@|SfI(Wrq?IhpKFwW6n&WYyB3k`8 z4#c5Qkpd50Sri6RjGUl?vJ-L0s|jhI zP=lqyg+zVYaVQ&5hTf>ZM%_O9&=;ct>Iu<|R9>t?V;}>wU#>&478{{LJi;HRdLUX9 zhBfk-;PX!=Wv`t8rE;2J(gG#C)Jzj>uQ`jJNK7E{rVkYEt4D_xAA^g?W^TRD2!XtE zLC+B-T>R}HS2fU&y38E}1AW&yc5;(goRK-)b2kyZtG+;NxAO;)G7eDpTu9rCT?vRc zYzgBmcM?T&DCjuq0&GVEP!D?$Wer{jUDq42{kt}7ndJ>iPvoFr?oXg;wG>a;myLBc zoW*+o(y{IKQ*eB?7x&y)Htx`$gQT;&F zod=isl)~$U%O~TxBKoCPF5Y^?5_&94hTW$%aL4?UQ0h|*(Ehg*c!g}C_4)dwmh(xd z_Ma8Z`(8w;+NYqfdQIH5GJXW_G$G47OK?*9UgF`lU>Kt*Pd-n( zDhxAgCtU5!LE_(P)E;CG8oW;nZTr80)eh5;#C-u8Z3qR@m95mna%T`3@(}5i@OsCKVCPT~h4#jPCD+cP;%tPPIQgJx znGSAH$w0lES5l!9lQu@x9z2~YOKlkDfq|9*r157C_phcrC?(U-u!jxxJNhGKX7`Zd zO4pzn?pj2VWG^CY^U=@&IZ!xmU=kgWBZdKJ3}){nDb_FZkne{~#u zt=@>f`&|W3`V(PS=l>YG5=SW7I4t*7A&H78M|Ni3_nldGXO;*_hoqEFDoMH&T@q5{ zELY{uog+C)c4wCmNl8MKq>@fP-KFU8`4?v1>$!f9CUa&2=bl^_%C7wLURXB=XC4vJ z^rA76|NiJi-((kH))Z8-3)Gb@w7Zu}-Yb3-^Yo-Gwj0k7s6{JCaGAgEWOAqOvskF)+I|mpo8Mn>T3eM%vsgdpV#pB+UC=#b8m?p4y~q} z*GY>i!f|0qWitC^-C6$jx?QYA;Tnj zC(`Qu#cWaS7JB9Ll~wVAU;N-b`pgln$#lcvV0z@tUD|5!rqG3dh1T#Z=l6b><9F*` zWQQGbCUx0!(RI`!J~Ho(eEwy6hSFNTu-PavU)MF=ts=~9M3&biu7UC&3AK1HRR?%)wXW8{@ zXNt8oz4<>5u*}vBcd>@aDY0+)HnGB)NsJ#ajPc6iGfMt5#kKMmn0k&6^ZnQ&bJc}3Y z6)h4!|8Y$yQ}bzp=Q&H?ak66%RO5_GsR>gfb5fL8KaGC$cOAR<%maE$=S*hNZc~P@ zYQh*a&JjO2lOmix@r`;6*6=?%zviWxbPCNTJeIa%U9oPWt-wtjAy!x2EbiD6)~ftEiB49BXG3-bxBP(BC5pl?s)^Rdti^}`VeOAq754_0XyRG-)lk+&Nhx-A(`jK2wY3wmpvvCXE z5#+|(5UnKqX`ez@F4w0wZ=S=yjB{w?=LI60#xDK_4=sk>7*7i|;`siTA)W63TlAqd zla{&snhyS}&F9rzVb@trXI4$>6J2u17E|Y6(lymQW-wSoe17$Fe#2Tny56Ojt_YW6 z`imNQi8hAffWS^h(@MZb%ywi7FWR&ASGLgar%o1c|IsJ*ufv$1PWOc88Z~L=iS$JdKP54Pi8R%tkDdzVMU84P z7es07y%YuUk+G8M;RacY&#y8CX2rRZq>zbs{gT^)(@FLcaoqunxy2hK`(pn{ss@V$ zSzQH!w45mxLDX)^mfd?K|K$f;6wL{#whPaaytRp!G;bG3j9z9-c9@%4ynm}{VP6tr z(KeAEKR)tNY)`%uY$$v$Ftan_GnQJ|M=D4yuie8UP$*&`pUKs{D42cB3X$py-vYO1U}v6 z?>nQ&MyRUNB$nkSChTccbxpgQCu7uf=9=FX&aZE18R0mxOEB zd2~~)2J`G_9&Pj~j|shPN?(6>P7F7z2~-c6(m&O&GL0f0`$R`&BD!@(Y`;lf68OfG z85v$Cx&+GUtOpy#dq^dg=K8c{Ev-a|B-LgTFxFEUm+foUcndLIL>@KaY}q>p`~~%u}K`ByMoRd z-Or3K-^jd;(i8I(o-mCO70goN1g|XVF&*=Cs@TZfpNX5?${bHu6DX~?%B=O&kQ~%@ z6uP5tj9mFG`po0g{EaqO`I4LFjQ{9$al>J0!SwRQV#Awu;zkQQ3vVA?0rObDT6#ki zbHrF)f<3$;7#Qpl1oxX%*Zql=-0cSwJwVa|vt&$=Po9%_L);>F#WtpBgd@!+7WFB8uY?dM+NpW#e+!K*K{6hGpNei&O0djy!#pt-=9LyzCVXF6`jfPM^YbAALgy)SB=om277x1s@k)ot8^~Jy1*Mshs6~i|-Q2 z9^Oxne7(SaTmDP%XbmN~r7kN`rg{Vd*-j?>U%lk5>Rra6C00VJt`(HqlnS)|%ayo= zLka)%xFkcmkogdBqOJP}Us**TS6ncTzkx)ClMM^W^x^&$N2a1xWM>Bhg?3*fARK zcva>k{o>FP_S&$nNd31WoAvF2sGgg}ZgZ0G0&ySK zzSoWYiLIn7f9i^FEj`CBK8p!QRlo6(^?cwXKNS;+bZM35 zI<{)nTXwTa5P#i~1EQDmwxU_1hv~OZF3>0I4zqSoJNfatox-w|WWL}C*eRLmsSk#=c%4(;w=;+_Vk_)^lS13FZW{`>osI2ni090eHl7Ncjr&z zt^FaSuiE8`dd_f|8Hs)Tk4fLyD8C6$(uqmT#)uMH%Q=m2_EnnIcTg4=6>MkYziZO- zO6Azo3bW{_SXTIKP)A&J$5@1$2Z%CLzp?wyE6~SpycF5H8;JQ~R-$tqO3ab10&#}! zN#3zyW9DMmZT|U=DfE*k-s~c+DSXH5S3;S)dOYm0v$%Mt9PQpbnXxkSVrPuJWt|p@ z*-h@ctnCydTJF|ddi#hc{W!vxO?4RKJ8$eVf9$r2Klj9MS}-ucz7AT*lMS?I6z#i( zlXNGsuS+}l>yj_?+Sw*{lJ6Ae?`jiT`?{t@@Z*h=1}zy$W%^#hx2=yD{&0fiNn;c9 z*!cPc|3e7mpTr1?{#z?qOUX!-9`;JS*0eG8qtb$bNu82RdnpT#jxlCIm9)Ug`5jY{ zV<}kt=Z>VyCQ+gw+aWo#u~d+}=``b!=qs^vUm`JE$cSTC+6w;hFH25ZNAO#0beTOq zl<0yU%}x=oq=QXcultv@2^IEciyF1-*t+eF{Oi|qX@R$jc<6@*8wX?9S1P1v%QIhL z!IpLWxIb6uYI~ z@>Jo6i95ba2KY5zjk{9Hiy{aqsRC@G_5J#czeCBqMG?_=d&1v4cx zgJ|&=M*$mlKs@vI#3WJWdWLsnF@5?AlC1k~z-XTV;$uFe%zq1hFeS|q;?1?K!iJ9X zl97>7+RZYW;Ur2+%5E$WWEx2^2jVOlNkA53)c=T?q9-G1tW6PbRPPYOmQM@`8fdH3 zN8;Dlc8Krj%ZO*!-JxgwwvzaS81duOYUnyvnYDX-O*CtD8NYhpT;atR6Ikh-hVbSR zeYPwpPjt)lB>gi`L9E$b%3gkTO0@7{i#heHU9@JvlP z>B8}gbmr|l{Qu|!^y3`B^I1HHKWoH|9xx4LYp*8oE)+BhJ%?}6ryqFWvL(65sl^4< zwdI4{W*tJbPlouOa~P`IB@mMPHYj+HJCSf*6KPlvqTZ6*;9}!!v}IrgrV!YGJvpg> zj;1xCJtt$~y~IrXwqyf%W56P9yjhb3&A$Jo>X&+>PRnc5vV;nF z_&_{zHy$MsPEz&qnUo)KinOs&;AXFNL)%B%QPAP@z!^ zgEwb$UvzjjLwn&f*tU@0i9Pq`@8Kc*6AIGElxro*3%U1Ig$5OmpPrph*q~jssU>D8vQp`}~VI zR%1i>^olqG=5L9ml42~L4Z~8a_rYi3i{RVA7);acBC$_*95;I{0{u_5;k!dKF#XVL zAn~;#3`xC&fB6suMJb2Cg!ve@M|Q)h!X!MGBZXxP6+m6`OAxwVfKP26#(j25qZ#ww zf%>&?@Ll#<#9Pb{^mN@Nx(}+M-@ArEd%}F;y74qTXi_y$O#2Ls#zVklZ!^ffS_zw7 zB7ywckHj%Bik9Y{z&1%KpiJLG&@qj{?oE~utTELnMjWagP{ks;o>v*iA;$LF!DGH{h#Y`3iZq^dU4aX8*N+S`m9XOTiabBnu!QzZx@Kbph`slxz+lr}Cxi`vC zn^ZpeJYNIp6}Y22APKEksiZ89Penhrs*o3K`_Wy+K=?1Ag^VPUSzHMaKiCWs{e|fM%9})gcqytn@eeK8PymVb zX{g!!5|-R|374k2z@xn_2(9!$-}ZW8hO?vb9NGD>E;A9|d5MP`xaJOiF}wI|5K>NqpQ8bym$&_+ozs zvFlk9m@`EJ+bR?>r|2Wdb}k2HZ2AsE_m!f%xDZU)I~5t1zlPDLL*SCqa-^sej8uAi z@vY1D0G&BBvdoSFBrhht_7OX z7tnpz7i6-oHM%mf|G(eZ$2~Y%nmpQ}!PQx_4Md%YM+c17P;=&bQq0rET)!_;RCj?Q zcj!qzI{N7~x;rC`#HZ4N_7<+=4K%hud$qu-kRkQe+IK*=?Nw1#GyIKP32(wf=xhWWg}dD z>nm6lG9TWXtOmUT8bNIPTr_>6jUa36I)32M7cg5z2AuVif*Q77K&oLD_%wVPmq}QN zzN9&ze<6p!vDa?I+q_gT`gcDhWiBD^`@MKg*bT_$$RIjilQ1l>Lpu-Zz-X!u>XrQk zg!W{De*XxIcM}7L7f~>Cr4XCvy&iT2iqL?@Iq;l)2bIkbcxf~T#DwjG#mD87APCMJU|tb{5s1P=mWg*TAQb2}G>UFv=*d=S*9b1iiW{fZ&T6 zR{1>*>HXG4??SEM);pU3u9^tY)K2(T_c+I4Oqx7m%}4$%cIfNX6KIk9V%U@(V9rj)PDRlVy#J*u)L9g6ZudirE4)W3+-+S<%D*(^_v z94(=iozW%TobHgR!Nb&t9XrU#$;D)-poM$+YCiY-N}9T|^EnxH(U?p+`iMMfD#w-L zy`moMZQ$nBsBwG0{6>PlXv(fMkg}Mfg?JvTsqhO%l;iIR%;`S`RJLO>F%({=P~WUGj!YHW;9>SOEQsWx=nZEx<&54H#02LdTZR#Rq3a!snLP;Xmta zj!f}w;FNHcGdjndSlrSK3Nznx?0+VL9^3QS<`|bddFa73^w{fh+kwWPv4xhQd6-`Tj0+ zWY;3tyj&I;naqMq52T^vr)^-3+zd2j{UsDB--3L>Bop5Bkw%a6l>q+o51KrF0c9Ry z@X-?pzTv###9mxKarSN@F8;`!?=-uOIQIP(Hhc0V&^XB!PczK`vR~9;$@Fz##=p}* z*0BcvbJ7yiT9u1E)_(*t&!&Tl4eL0)_wR8&eBxc9-e_yUM?dBe`eq#{l}ocz!AN_C4%Ahr;V`& zTd?JFS10Zj4XB(Z$Jtrs1CI#pU>3aeVnY_FTPTH4`7`+B zo+l@?GJ?P=)a4L$Ypez00SiJfIep!Q^X&_WVf#^hs;dQ1T!M)(O@GKsmA`IFwaDNTR-h{L8ms#S}fHXTAyj)SrfyObJDw@H5zjqcX_8 zqYjUfdWvlw)FLcDYoJJD0$frn=PtRmj`C;>B@Hv@Q)$<^I$$oWPrH!a-+ zc^~FbVf(gH(nY%|wUL$NzcW50Z}<-SX6DSDH`AV*x4wZ~BuXMP!b9ME{xs5i8IQZQ z<_SeDiQsB<%5pzyZbhlF)2WK)IBGIF4?c8VqQ;dCDeo)(c%Ty(z4#YKxUQLxf0An= zjyQXBvPV^j^`pxP&NTr#@jxE7UQdN*jm)4&zX>RI9)p3NtH6`T3xM<6$wc>sYdAJ& z1;d!980T>z@q6`M+=*6(3oI9*CAVh7mOujKYc~?Ves_UhYY~)~uEpk2IH9XQKotHx zkIh^lg16>rg9z*-kXW1|mbP}{;r<$Mi+w1*T(Sw&KI9Rb&AtE)Z7(#SmrM*n7Q8sr zhO0%G5v%*G;r0urC}X`f1T(If4o@cujcHfFMo$|;Wf0)fxa4$XojZa332%`*CgR*7mtr zx_gI&=C~IGN@KZzcv#7{Hfs3cNWfP}ch# zUT`iJGq#(_(SNrVY8G4qf$r7#L4R$`w!xFw7Zd@u{&s_gYgd7#lWmB*8`csF*G&Ua zchlfe<*A%A(W8WvWET;4Zw~QS90k@keg#E4WRdD{ESeYa0MGZ=2J71sz_g($D86gU^UFRCUk>^?@iNXOyXX~k&MY%g?(8;0?9BS@R@zkMlL4@5u)!0{uf%$DhF(yJVd?5hH+~j%W~)FHB&t6sb(sT zQPe7vBh-A+elvNMKr=u8F3K_MJf&*Nr#i|txuv}_W|9L%+`{5LW{NSB%sw}jldCZw zZt!gMD;y;!XXWTY)PEqoF9bZW?LcLCA-T{z z94&adlsFXe5qznPK%5ijz_Xtr&_sCw`A%XD5%Ue?p56n;9+;A*`^q?~TbyBOe+YQN z*T$bIiqWAlcl3Q{9BdT(6K18KfL{e4k(CU-PvtLq(RL1DMfRxFY7*)8cK}RXbqXc& z`RH5dJ9x%ipWLnU8a)ct!TWa?fIZhLVDPpr$cI;i9xWD;tw-*lo~ON}M$vgrUM7!R z7WfSG5EE_}%kjt?xqdLCN#8JMjfDW^6uQ>pnQ-eg0B6Zwfg1k_U7GbmpnpDfBZrP!#$K)`P!`+Jg6e4Gic^!*-^jzh${{hiRIh=8Kr z6IgEEAsEuq3v9>;^y|)2xEKU@#S2Q>DhsQ zC~O9CHZ($ND$sS=TaJxtImoe50%f%n;T4;Y*1K(letS!Z<_b%6C22ZLzj~Xf9(E(| zB(~s&=GACP=Rd?0PsdJ}E+)gz8o;;CUw|qsg1w*miLD3Iaof^N+?y9miY6t&e;tm)c@zw>cK(t^cco!wJT!*MNCy)qTqa3_)1#nl@5&6;Gy>Uc_h`|DFekRY!(q61+WY20D@zjw*+z;bG4O$YJqHpu}_% z3vnJEl9`7745q*{ua*X!3*@0`1`g6wfOzz zBrZM!nO8S(V>4ImWzpC0%n{1Y+F&Y$e~kqESZ$dM9@ zU^1os2Gmggjgsrm6Gb1B@Bsq@vP8#|EC`TA>to}<_7%(Ea%V%dFBL~Sg3Mu~<8So+ zUl^Qn;4!w^yBZvw=Lp8syz!6ctl`655y+lzi4S%96K|iqgXt6NcpL5mDr8oJ44(x^ zSMr0i;g%oR=-vgVEk6w2$2yT<_X7CvQ5){Czydy9^%Y!kolV+!IFRxcs_<+;B@kRa z1O*aZ3#wo->x=fVq>p;U^rF%TU(OIGX2QU~Vek#kaYsM}5LX#KWZ)UTn_NS?b5 zy_r^zrsm5~8r|~R ztY{!k)y^VugR_K(;TaUGe+&kcpGEe~%IJ=-I%w1T4@f_aMn?+TL7xK`Ji3+-ia$(X zD!)K%NGlHweLa9Ve~cq88#JMvtqq`|=kNrd@E4_AdVp&sR3RU8O)|aB(e#?hYh)zq zAZA=S3|`Ygupu@d->&(RIA@WL89wKe6{iw_XN3s0UU~wzyX-=H3=g4WYu|#QIVE7+ zvKNrjPoZLM64W+~B4Rq#O{J)NM8#8Sv@t`Fv7 z8hezHtztSV+5QFM_J6r2vd>d0QAg9Fdu0gkOM2!6XEFSNwDJYLaY*A zo#1F@fWOD<;NmWOqu1%p(0C{TWqz*$PRc3J*km119oRus>#G3M`y0`*`M=@r{y>xP zb_&n^g26`iI+ipRy8 zg9dHja5Dy$1!$s>2gk9E4hMm%t36Pkk_8*92Z@lZZ@}uADbOk%h7QRLwtDLZ+N%Y_s8->V&U6>*@CM{S)_MJJS>xvAj)tC zv1^Ymvg}~dme?rB{4jtwa&tj+u`|k2fCbM9KzX+hfQ%{HrWrp&VcYh5uzW!yn)*5wee%|XTX+8kw^jyoe;Ij_i+>bT ztdBL!Z2LpLzId73p16@Jaa_y&^*n)^zXMUt!V%w|9BjF9^q=X3Yr z6`XNVhxGKk!S&cQPL=&FqspgCaq)yyXan;}Y?l!=GxR6n&pf5#-!I@=L_EM`W=6t& z`Q?~%xD;wQ)d3vIMaEQ2G58;?8!J;K6PW^zc(C7W1_LMi%Lkj>_kt zkGD2l@Y(_nG;ASfkB4v^V%|cmpGO3QD&T&rKcRlj{ite93aq^v4encwf=opt)YqYd zR~DO)*o$}+_b3s)>UxPjG~~jH?2G7M?{XyjpAD2GV71e5;|-SZ0~ZJONfX?)sgzm3qZ+C` zKb+h}#Zh;)($S5!wWMdxbn^J=MDBpaO$1i7AQ<$WRB*CFmbpFXWt3LwoULFYiTL3oyn}W(O+(LSRjc~t5BH?*? z5Ja_eV8os|q)>ewK33caCAF%s_Toyya@I?9#mWlGddQ#_^Q~~~m;<3#u^vQ5zXyH4 zWs#oTiurabO6ct%PHJB`MO-V}f#f(6;>bQNP&p+728>veCO#iwjQeE#o=Y?F&uJ3c z<9rTmnkS7;&AW%fr&|J%<{)_aZZ8qlH8KADf;<)`xd*!pBzbd3xqFqqBAtDuRPW_Eq;ujE z(x+NTnL3>#cYiA)p9g&7($`zK#jCBUuTC;l*!Jb*z2h>}>73Wpx@~W$XCBYF*#(B& zp1YS&QG6_lF^P(FkWXxD(2M1Aw!m zBepp?#%2BMKrW)-p(Ob ze@uaO7tdqnJHv_W6=AS}I}4nBG>!1Oq6VV9>mc)dfa5ZG2tV@Z0wj0;#Ev!9BgY@g zpy9U~FbtXl-@D|49Y?Pqy$4s&HtPpOYJW2Lcx4job9oGXdh$^7>$s}Feom~r)#UZ=^%MFp#g#H#j8445NP`_TNxhM# zR^4;wo=C{2sxK!{LWP&qqW72JmIftC|Kc06!r(WYHRwpyemur~IbeaU+C&kr*X~w}aAU$EyixlE zgfpE`$}$Cx$MR%!|L`2(ad#u(zn+1+?Uv#P#;qU>wj#=^Wyo{Bal|{#GpK264Z1Kb z9gcnUMbiIi&aW?+Lb#lcK{v0k6DFM#zL;J5Sgl_3wpH z;u2xH_{dtqDSsE9HoXZm9N|JO2MP(@)^aXizE9*YpAWa@E+r1TNr88#4iN@hi(%vR za?YX3pTzsqQm}Yw6j+v%iipRF@GAm?!OCk-O}gQjZLDY zP8pMtdu6G_LMJk1ogt~dDF!`Okl`jq*u$*&N^(!(Q*xm95}8zELK3v63Lzbjx{c*-abWNc!sV9_JyqXfnXrP$>LehhCiOhd_ns~8d zGNo90hRiRn;p9Ct0C{fjiLgIAuxIyWiO$R_Q~MYed{8-xjnQw2mDVO;O+qc9TmBa$ zMZE%-Cvbn@oGQ-8OAm-WiWMLyMhu4!nFI1)HeL+Q5K8sNKtvB2 zc2WqI)3FQ~2h7D+CSJh)Z97LOt?`FB+Mn@RW!WHrxh{snsXZ%r~Xr+hI*i?dBNTc{BjOm=uM^f0rYsnZvoMsYH&&p2J$k zFhCvmg5RI3;NSFcnEdNG9?j*G^cF)Px8e)(gBQ{IcV|(>x_UG;^*z`lzXZ%)69wh# z649o0!|-!eA|BPQOw@Y`LFMdFsNxcdCePjqw`KLCmaCtzou>DJt8pp7RB|wk?14){ z%8^U=5@hMFPkE`{M`Th05vgiMDc(1R`{P~U_y-60CNd8N*hr(I3H=_0pCdD8(PYy5 zMza0-DJZ2;1{tYHGSamidAH7iMI}7AVtWxvSgcBRP3|V2huD#Y{<382lR1ct&IiJ{ zRY>#87RU>H4C*S)$^K7C=#RSr=KM*4EGxDLd-nxGm9_tfvU@G2^L0N0zgHA^9b}HS z_|qJxL;I2K(kDneelzUvMd*+H5az0@0+P?a0rmQ_WOlYTx?=f*Q=CiS8PsOjH|;qF zwr4{3)?(-|_+f%$9VZUUWnodmM3h!`*EG3X2WT$Y3TxITVt&7)P^@1k5!!ncxQ$MR z{hMbKPbXEuQmON}Z^JpX*jx!5wx>{saoB%g} zUx$5|^_93f>j79>fpE83X@twc2*UeY2Oxc3V|BBBgACUp(CFez*v=%;l*6e&-}D{P zHYo}W6)zxOyILW#bTTplDa6^f5wJFK35aQ317AMvLuE(aK-KtKxV+g9d86iDxZO^WF|VjrvS~1jWN5jb_REC#YyUt$vU%J`zom3JY?3tSA()+mCcrf z@8X*I%9*Lj7@5siImVS@BF*514C>C&ddi;JWR|y~-R$C`e9Etlt%nlW4;^GC7VC?b+;?dnW zu=(gHwz5+LyUQIX^xZpfjmkw}+D`+#zi5E)_1FyQH#t!I8wFM@iN{wy)4+qqb>J=g z5aMEFEC~8?84OxI$HKYi2!$_3XzTtk;(b*YsF6#DS;;DR%M)w3n&~CFX2)SW?N#8x zJ1?Qjw{7@0g_ERGeKbsZ{SXF4+5k#c0c>oUjdtHE1gg|Ja`ve*5Ioy}{IRj0B@6aImP6+pEw2%S(1!{oG- z(8!Z5Xj54h2<0(k@~tbx*nD?rUVa-&Y_gzi7?0#V6p$J1M?hg?D)@V|5jMW_M_bY* zC?Lq4Jg3}@q=SRWUoB5e6HgAJ*{K^rv@iy@UGT^BweQ4PH!BBrC>EM2J22ewxU19= zCk5^r*$Y(bh7>CMLIHW=LFom(HWmRl572-m-i z=C0{S+#P>d>~4(?TAJ8JXde6k!n?D3N5Ef_0GhjW zpjEdtavD}AA|-slJy!y%L%TrY^$S3e^~tx0!+#O7iIVp9MIv^eZ~vVhBeMr^Aeq3ZQ>1hq!;z8uFhU2M_9{ z$TJyX;Nz=AG`w#Wc|5lm%yKC~Rv#9CD_4)>mHS&z^j1sq)2pvAMDYtzcWJ^$WNLs! zG7&Is#0GAP?L(*Z8SJoTAyKqe3g#-jhd-jLvBS4zi1{5UaBM>{CJHYFb4LIg9d8D> z`3hp6Y9`l+t^X6)EME+dE;x$jJ{v*p5>v7+@huY8YLVHWFL8ouexQSwmB5h|nfOi1NMiGk zv#6IyfN_DIT)p>p)ZgYI@5h-~Hmj zirW{Fe8yxlZk;NDYiuAxUOXV{Rc>G=#?8dZPzWx$QPAKx0G-K>*oJ#0FdrKvHnr!2 zw!kdl@o*(^XT1yfd{79r^p)VsscnSOvGqhPTn8V@mqT|Z68{&!0LTT}fuM*Jgm{Yv zLYD8L>(fyDGB=B;@cBZ#K5PgZ*4}^{p*!B9?gx{q*CM4+V>~u66Wz|4@Q@h`z^0qS zV3VON;gMMhc^)J|zbxz&m&MyC;~?HUD+Yn1R6PA2Hr zn+|Zwkiw5OJq0ce1S;583-3jBayPW@rSxo5$x6qY+^{Yw%7nhc&3_wC+PXS&ogaRp zZWkWlzR5`Bn(ewu&Gl9_JLfh;mLvlz>SP7i^MH+6S7A4K#m$2H^;L#aPHdpkLwIIG zRyo}FoJZWD?z`MsReI!R-^pCR|7*Jruqcvl3rHA7;t&OtAQGm#tE#&P7>1yzAR+=v zHjM)ef+Ph*L03T}tVl3|n81LTT~sod<07JB&LXD8ydo;%>w#r|)2eU(_xL_&s;lm; zIk!)pdwaTj=Gi-{?RF7*9%n_Lv{2w>A56l>#d4rGnMov6rGWapDR|bblia*Gb*Sxq z6Aw064)|$H@psB=frrK+`1=fH!vz5o4*s>^q&AP?(Kt9gNax>!4j)T~VLs~GR&mhz=XcaN?Y$Q4L z?SWF`S)%BT}cPi z)~-j~^fP45mOOOKsR3V|c@J>UrJ%KGX~gKNhcMH6D7kIuX)pv0fELv@Ft>IVnH0B* zJNs!R956r^Tum0>i*$|2=6gP<(_j^}AD;u}20aB{wTfs&c_}Wt$cxlIlZ`5>kD^h# z3ea<2Gd$C94rLwjLX$*t$TLosTvJkpvd&%vcZzdC=b&|PUm!plhZZ2shE}qqI1f#H z#iC}J`&%}@vn3~0=|Wq@Hq3ed7Ln$d|)X$P@S7 z(T#YVJV%_M2VE_Mn{G`bho{^}3JwP3k$Wm+@JT;%_px;d=B}VSwe{%N{C6aZOCj0u z9w_IC6}f>GLk`@kOj$fnr|(=JLMb2o07d-wBR6tn+AxPdwruqoT4e9gn$&`@! z?msR3w@5`q*pN`>h5@E?L<-zoqYM=yqVXrICNtY*^f;43FJd2i4VwkmpQ> z;TM)4f(o;hpmS^+ciFH!csKJI@!^#^ypU}{c-`HDd#!CCqS9gr7|;l-vuubMBX9Ud zKqGU%VyrQ9BcxK-64ApEC&7x?9G)i;gdsz?HM&_GTIkUN}UPD^AACH z`J;G2at;3Og%4p+>H^b;^syYgDF`2Kc>!-XK8o$nXeRo3KzQKyF2ZG=K6kZoIpLFP z4Vh2=vE;jNCw6uXhD+Nt34cv7YCA?@eGSWrHwWdgGLOs4czl|7q zFc}=;%z?WN)HQ-dD4QZvFgP+{8r>Eeb_l%<~~t+qXaysQvG zE*sKFotQ1AcW!@z`fki52QM=rjUNx7+MnmrhR6EQCWcz{OGk`O(J(|IeMA(NGM;*@ zTnZyM)l;&e3u*ml99%vml{gqXhFEBw4x&!Ff%v)%%czOkplP2raY_(Nblx%t8*8Kq zx3g+Qj_)?`de1VDqppE@=l12A2D}GYMPE>OvILjzFaWtZdW6onUkLe2Z(-mXIoP~S zmuOG{K(?)hsCvPI&H?$*wzLx)KY`0!Z+!(v2v-towyKc7u#2H$%P=&i@jW~?eiw1+ z)=p5`ZVV0QUxf4DFT(g@Eu@;AOj0|Xz}1-PNDveNcDAnq;p&&*xcn?~z$t+K&{%{U z4wM2T+ntfONjzA!@ILH_(jZwD0dPA$7)}o;0;XUwsS)9Y70qx!2hX^Ia_(--;AIp! zb;KC7G_?j^*gX(V*S-kW!$&Z)gNw7L?;w@67NUr94f1`ZIrCnfC#p?pN3m{BXmf}$ zI=!Lmvq4=$ryrRQMXo^)Y}Io~`7>=VQ@?Lm0Jvhz+h#tW5HD>3{>; zwcLwml+d$N(y(>oF&O#5)QVqwk{;Z|rk}l6qjM)7p?oa->2{|;Ds)5%uc>1m)#kp7 zwl3&T-%{#N`@gEh5HL+?>$AMB`fCk{QKkm8+Vt}7V_ zRPYL8A@6#ho#b_`W%Qi2%V<8Q5=D#F(+}^J(lU$2;rR#hNUX{PgEN&;Ez1|m^HuS3 zr+q}-pKO@rz#|9G1+aPdEp%+TK6ycLGvJxB$ODG%M0(8uqHD^2LU3{liW!-Q?!R4* z&(c{B9`5AB3R)dLx}-_U6wg69m?7G^nFCX`i$I3*Xi^8ii)o`&sBmxrG4!`gt}w|G zU2JpUUR*1SKWh#F8cKOs^Q>_Y?0sd?Hu?n=+-}8W4jzZMIHu^gOJ1-az7#Iqp+yKh zw&SfjaWLGw5sp!L3(gg&;Whr*1br%)5RHEfaw5|~;k68KABx)@x_q4LJsTyP@mXTdKOdg8i3#K+DYuqT7Xer>+xK|8kc=q0^M(V7>VeLMNye3wU~#Yb&<~Txr-q6u+s!bD#>i_~&Rm*EE6Hx8fRU z<@uv#7IJ9UwYoHlAxDu(kj z-0?v|U)-my0Nv$;NyhVtV_g#&_p7jPZM`{WB};w zyADsPk+t+z97XI~xC8{ScMy^J`Q&|ZA*@-Hgtoso!T;EA392{0hpZ?aI7`QpT)y1| z45^bMFEw?LDHgO0zfA=-9`cw-lJl;&+_Af@bbQrF58H2EP5t{7SL=3Nq zMyG2%$z7vTaVt#)`18OgcqkGBmT3CS&Sj3Q;58jba$5Z5=T5n0|5_N38^=2~G&<`G_j^S_5g);XpI-m`4 z6Uje5)Q~m0eyC~l{kA4&wWxZ@;%U>itv0%7Peld>TYNAM!21)b5y3$9{ujp_c>fr zYATn`Wj&;3V)-c8DUi;6HJw+ao{PWwScq;!j|SJU;i$c6I21ikv54Nf9-6$8gL@jS z5PQ%hXu+KKTfNx~F0@Od`2h!zl1>znIrSB;60{%sQyYnUcDgW2+8+8n^#|mkTI|vx zB{KilPSmUNPckG0%z^UP@!0w^@k~r(EFTWF=TUKnk}`wILSo ze*q6H`je}`odg_LZ^5Sx-GcWOW#FxZ5YOLM#GKJJ;r@Mp$7e?CfPP2@?8@DNt7^}{ z+S=q{%wtIvEvP}ajqheNvOkh%OIwIz?v|A@)oQ<>u0Ir z%EMM&Tn1dm&BtQWRw4Pl`-y#jE+pot-vni&&*SspVOXTO6t|dFhQ^JoCR)?xp^rQN zMCdT0^xKpvfPQbg$`-r!KOEm&~? zf$iJ0pk2N_Do+>>2dhtk&&wMjyuF)8>QWiXY*2*$2RQy%N=| z(t%tmv*1DvQ)*IBE*xf)NtGU1h)N;j2{fM$crh=6GL1$3>MGST+dj8|G%gc9DUT0!QNWPH7TBcV!4JE)SG6 zJ>e3+0_eB14xf#e;M;g!c(LX-aOz_h*mbHDBt%aq7VpWy#4S?93K0p*-*|#eZi9&X zA0`mdVn?h?l>~m1wqlWEyfNjze6Zkk5%f$ui9OVt1|JzN!Ly#Ff~xEF(9NgavTWoK zsPoi-u&6UfyQK1nyEa>3$~IegdHzwDe9aVAw#yL198VJRUbkW4&^(}K(N1h0*GMeL zl4Y*bxMGBnA{un>7hFqeCvLKEC$xFgN?dqdKww7ts90(^ez3g-TD7^N6E?Nnmw^LO zzH9`MLOmd|*X;r`yzgN7Pb#s=&k(m&Jqv|&o+BDwKY^XIH-N48OX2b42{19y7!2KH z0PY89iW<%}j5ea?|h<%Oh?c@#CEb^$#?z90Qk zD~HY$45t+rsgMIRDk#CcdDM*(0Xi19n7W$dNGq(qj8_S@(P@PQTz``<@IIXcbXH|> zrIiBk)Van4)wTehJ2R2+-);ssIR6Pv4a>o?rE+j&O$Pq%DF^%IZVH|}42Sn>6W~fW zS?v6EdBQutk|=rRg5%k~@Jc`zn1-bSd9N1)`_&A*vsnr0tZ%h++o2BX=on0a^BecZ z{(5AI1rf2{8Q8^_EVNK_B|d>}fd@x?;67FCM^>14gK*DX=o$A9q`A=!v7aWOlW{Dp zY4sjrx=|y_i77(iH%nlUe<;}Uq8(65T)4(!(`kH(c_eh0ugdKsuwt(58h{f` z&2a6hMQCpobDg$J0jez>hCe+ek2#Abph=IHqkdBe*r~r1xQpVz;SWVn?f~?E=J277(NCClH2p_ z;aSX*%2n1udTX8_i~E~N-l&b_(2`eZn1M7}BLAFdIH2xmoSgOiBVb>Sz8QGuk5LAF)R-A#hJ2Ch`>L7m1 zeVrwKIhF8G$^{+zH{gpyGVrbcY~oDW0&L=)RIv4|0^F6cflv!u4ZP=F#V?Om#M4)% zfseoL2W^(E+(MZ|Y_N_m+-1tL73w6`>|Ll>eKa|_I>FIg(4XjErCK;wa9!7}GHZFV zUD;aQD*0u;HCt{e*@03uJAF5KyT(AiZR>lRYBM>ln!NRQt2<7e73i7eSGNh>Y*zFS zt}d<^QWLt{RdD*ij_RXx0&6I{LR%4Ey86~{QPrZ1$g0d!`vvPSFRO8_n9pBpoKbDI zdSunR&gWHuX^X0C+!OhyJ6~5_ym7PgWx*o>s`n``TX(<8H1-`o@!Imr5Rb4*bYXhc zvt7@swmT;B=bQKl`bSKydR;i8x-_=F%IRYz|H^dznnkX+_%|ur>fQQpsvlMB+rr~5 z)ssC3S2sLeU%6$|K*5qw^{NSIu9gSw8gVGh35W)8^W&OK)rQT<$vDEa+E1K<-TKq=zPTH$}&5 z0}P3}cAUN?1iy+ahehQI2O zTg{3&ysA6d8P!vpkl?J!WPSsU39=H5s@A{w#ctJw)m70__BDaNq}@DSOFJX|6M~ne zJNVAyeyv^`(zixiTdMj;aANhoOSN#~Y zr=8~Q-8c$4hb8mE;@{HG%H?@)_dTN98(yH78*Hct`7h|y$}D=;?M(W}iw>$^^KzIY zT13Tny7NZwUqd~5Z$iJ)n$HWA!%5eu3X*CmptP>7KoM(Z(<6%B@bnL0a8AQfN>r49 z8jaVId0`JxnZ^dF7O;UlpqL3a1ni=_!X2SsNglalX)>BIMhE#97$E7O^{CQq6)C+- zhCH9Mk$PCWjJYQ@kV?Ngj69r=p}H7(o{7vuWcb9EqWWy6BXmZfIltx7!kaFXY=kD9 zZZ?XVgdIkQE9-c>TUOZ)I2_K;OE9uYyvVh+o^aJ>KPG2u<}%#IicYp|oTbO_XlC0s z78}^=oJ!%Fb~@PRJy~QkF@1-PLeo-POP?R&3i%cH$*8$)y9_M>8+Yvn%l#OnmLZ&M1z ztJg)8U@ld3K^4t1JjwGbR3qi*IP=CDI`bM^X`Z@UChayk1$B%$P5H{6r~UT6CR<`x z(reUR>9?C2$QOyW)EK7%-tgp^bkzEr^h2u$yzJ@=DCE8cZ8hGR@>gDjSgx1o>+6s5 zw!FDvS9q zE4yPq@yP8Ou}@H~SGi2>Y3_v@kIOS_Ei^aOI@~F?QY8A=G+vM3g^j#uJ$18_RjpzK zFMzk(3VobP7YmYYcBW)oC9iU~$-j2U`T{OvvtwX`bsxNy*SSR5COyv4x<6N!_hhcQ zjh*HYo4NI+R$eW~tRD71Y5n%zZL31-U~5pAVho7EqI+147Jl~xh*Gp(ZKj#+CR z*RVksPCLr)OSD_7er$-8l#!EE_m7d&0ZY!7;%)3g8MZD z1cTP+7_|QhgU&x=(ES`kzt1q}{WJzPOP?dp9NTDdB=f=zII=xfZbUylP=+hT-Yvt{XBlyHq8WO9i-v5LF-N|e;&BX* zO*jfs;`rF`;5fD+YamCzXH|TgrRpqGj&jd{UYq{|?tf19?-KZbvl7ghRb%qys`+}w zbjR143B(|d0u!xZu}~rcX1xu`QaA5yP)x(3w?Q+Zme^VL?vU7Vdm9ovthXVt!+RSN zJIP^6k!HwCaM+&+genVg`U=D2B`jxFbsU?+f*ciLq*xRi9UeV}SqJfAwkiur21SR6 zX9+{XgX0-5B;zG2CQ2L~KO-uR89*@u62y@)!HhUC;-bl-IcmZg(?YseNxXQPa8{(K z8@CQygT>>h#))GSB>w-B4d%u5W7zjKGrMiR(SMC8py$uOz9C{lP(m3`uddW7pcZbAo zNN+=8=hWMf*bVJ%^pdSbB8kI3m&D=!B#F-dEQv0h&$85r&m__HrzKH~HS$}suEKKr zTg;gZ#Sp3g-^!`UNTgbl*-+F5JiJnUM$LWzziKa60Np@5ATl*wqmr9g! z)c5%5{s})vf67mfAMw-kYkvOMDM^X-Z*!0tOYf)osmc(@tCv8C36o^Um``_1;P|ms z{G7&)bQr zs}A(o`t%euzC8tvUr#~f&lEJwkN}Rf6|)KCNLw?TaU5wIW;326&1W_fIMTLkdDcXZ z42inw@GbC)f9FU&^mjcAKgpaw0zZ!9^HoHZKg+Xv^e2u#>-mvyNuVSJ8xsME3Ng?n zl^ugwQrR(JC6yh6@;{Xw6C=q$CODFTOjINTnQ%x3PV61{W%&Ctof?c`cZ|O5{9gMz z`TILv5|7<-Iq7?HDf~n(Cx0rJq94g+(ARSLU#BZ2*1ye9W-QmAmdo$O`pZ5(lZAsn z&BDs8kWVwJB1_DXi;syHMY0uHp&Ys3m>JRW-ACA!v7GX`W!UGI;s3Fm`j;hB8b&as zVQ924Vx}l|O3z8h6prXC>KizPh@xPR41TSJfPKeG5k)%xd&61wO3*@=FrWYd`k zf4@veX5U8il?;Cy8~vj-{?Ai>h3zM&MrQTzr}+xmcc!ua z(KM4d0bilf_zTTvmBhDU*!+9{3GgTU^MB<3TP2YCXRrS|(XsuJ_lX?;ucFiAz4z63 zd#Ap=Fn?IEd?`zkwxmWD_avgw-n6grgK8%V8OrMA7F~7ZaMoA;BO`Q n9`*aS({5q>ky8nz@E%B0KZW$Ej{XZsUHZyCmn0{B`ejzF&X#^D3;+o5-(ZFMRyCCz#LlbGo9xG8%QA)e?1{s97U`4w~EOK8%$PYlA9dB z$i_EiGqVPZ>FF@}E>RgtBcQMr*IY)81|HMApOTj+?`JaQaChBwTU18M1SqF8S&o^X zC6RmWtjUJV1}y*AZn-%*mD!a=Y{iYL$?KSnIg+?Kn?z+KEWpNJVCLrA#H`W4XnNzM z)+$jMVM~~;Tob^y>e|*mpX|qC%X0hJM2^Y*EG8UlUS`bXk`V>DLvHd*7CzCubMKfK UfFPBbfdR}oI6+KmG7oDY0H}ONlK=n! From 7b26d6699a6bd444ce53cfc86493465b0112a4e6 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 13 Jul 2023 12:59:44 -0500 Subject: [PATCH 08/98] Adjust location of KerasTensor. --- .../Keras/Engine/KerasTensor.cs | 64 ++++++++ src/TensorFlowNET.Core/Tensors/KerasTensor.cs | 53 ------- .../Tensors/Tensor.Conversions.cs | 17 +- .../Tensors/Tensor.Keras.cs | 27 ++++ src/TensorFlowNET.Core/Tensors/Tensor.cs | 5 - src/TensorFlowNET.Keras/Models/ModelsApi.cs | 25 ++- .../Saving/SavedModel/load.cs | 146 +++++++++--------- 7 files changed, 173 insertions(+), 164 deletions(-) create mode 100644 src/TensorFlowNET.Core/Keras/Engine/KerasTensor.cs delete mode 100644 src/TensorFlowNET.Core/Tensors/KerasTensor.cs create mode 100644 src/TensorFlowNET.Core/Tensors/Tensor.Keras.cs diff --git a/src/TensorFlowNET.Core/Keras/Engine/KerasTensor.cs b/src/TensorFlowNET.Core/Keras/Engine/KerasTensor.cs new file mode 100644 index 000000000..9287284f7 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Engine/KerasTensor.cs @@ -0,0 +1,64 @@ +namespace Tensorflow.Keras.Engine; + +/// +/// A representation of a Keras in/output during Functional API construction. +/// +public class KerasTensor +{ + private Tensors _original_tensors; + public Tensors original_tensors + { + get => _original_tensors; + set => _original_tensors = value; + } + + private Shape _inferred_value; + public Shape inferred_value => _inferred_value; + + private string _name; + private TensorSpec _type_spec; + public Shape shape => _type_spec.shape; + public TF_DataType dtype => _type_spec.dtype; + + public KerasTensor(TensorSpec type_spec, Shape inferred_value = null, string name = null) + { + _type_spec = type_spec; + _inferred_value = inferred_value; + _name = name; + } + + public static KerasTensor from_tensor(Tensor tensor) + { + var type_spec = tensor.ToTensorSpec(); + var kt = new KerasTensor(type_spec, name: tensor.name); + kt.original_tensors = tensor; + return kt; + } + + public override string ToString() + => _original_tensors.Length switch + { + > 1 => "[" + string.Join(", ", _original_tensors.Select(x => $"KerasTensor: shape={x.shape} dtype={x.dtype}")) + "]", + 1 => $"KerasTensor: shape={_original_tensors.shape} {GetInferredValueString()} dtype={_original_tensors.dtype}", + _ => _original_tensors.ToString(), + }; + + private string GetInferredValueString() + => _inferred_value == null ? "" : ""; + + public static implicit operator Tensors(KerasTensor kt) + => kt._original_tensors; + + public static implicit operator Tensor(KerasTensor kt) + { + Tensor tensor = kt._original_tensors; + tensor.IsFromKerasTensor = true; + return tensor; + } + + public static implicit operator KerasTensor(Tensor tensor) + => from_tensor(tensor); + + public static implicit operator KerasTensor(Tensors tensors) + => from_tensor(tensors.First()); +} diff --git a/src/TensorFlowNET.Core/Tensors/KerasTensor.cs b/src/TensorFlowNET.Core/Tensors/KerasTensor.cs deleted file mode 100644 index 3204b4ac0..000000000 --- a/src/TensorFlowNET.Core/Tensors/KerasTensor.cs +++ /dev/null @@ -1,53 +0,0 @@ -namespace Tensorflow.Keras.Engine; - -/// -/// A representation of a Keras in/output during Functional API construction. -/// -public class KerasTensor -{ - private Tensors _inferred_value; - public Tensors inferred_value - { - get => _inferred_value; - set => _inferred_value = value; - } - - private string _name; - private TensorSpec _type_spec; - public Shape shape => _type_spec.shape; - public TF_DataType dtype => _type_spec.dtype; - - public KerasTensor(TensorSpec type_spec, string name = null) - { - _type_spec = type_spec; - _name = name; - } - - public static KerasTensor from_tensor(Tensor tensor) - { - var type_spec = tensor.ToTensorSpec(); - var kt = new KerasTensor(type_spec, name: tensor.name); - kt.inferred_value = tensor; - return kt; - } - - public override string ToString() - => _inferred_value.Length switch - { - > 1 => "[" + string.Join(", ", _inferred_value.Select(x => $"")) + "]", - 1 => $"", - _ => _inferred_value.ToString(), - }; - - public static implicit operator Tensors(KerasTensor kt) - => kt._inferred_value; - - public static implicit operator Tensor(KerasTensor kt) - => kt._inferred_value; - - public static implicit operator KerasTensor(Tensor tensor) - => from_tensor(tensor); - - public static implicit operator KerasTensor(Tensors tensors) - => from_tensor(tensors.First()); -} diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.Conversions.cs b/src/TensorFlowNET.Core/Tensors/Tensor.Conversions.cs index 18bdc1aaf..fdd62aeed 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensor.Conversions.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensor.Conversions.cs @@ -14,19 +14,10 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ -using Tensorflow.NumPy; -using System; -using System.Diagnostics.CodeAnalysis; -using System.Text; -using Tensorflow.Framework.Models; -using static Tensorflow.Binding; +namespace Tensorflow; -namespace Tensorflow +public partial class Tensor { - [SuppressMessage("ReSharper", "InvokeAsExtensionMethod")] - public partial class Tensor - { - public TensorSpec ToTensorSpec() - => new TensorSpec(shape, dtype, name); - } + public TensorSpec ToTensorSpec() + => new TensorSpec(shape, dtype, name); } \ No newline at end of file diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.Keras.cs b/src/TensorFlowNET.Core/Tensors/Tensor.Keras.cs new file mode 100644 index 000000000..ca946ca48 --- /dev/null +++ b/src/TensorFlowNET.Core/Tensors/Tensor.Keras.cs @@ -0,0 +1,27 @@ +/***************************************************************************** + Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +namespace Tensorflow; + +public partial class Tensor +{ + public bool IsFromKerasTensor { get; set; } + + /// + /// Keras History: (Layer, (node_index, tensor_index)) + /// + public KerasHistory KerasHistory { get; set; } +} \ No newline at end of file diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.cs b/src/TensorFlowNET.Core/Tensors/Tensor.cs index c0e5d4357..65e1c8576 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensor.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensor.cs @@ -146,11 +146,6 @@ public int[] _shape_tuple() return rank < 0 ? null : shape.dims.Select(x => (int)x).ToArray(); } - /// - /// Keras History: (Layer, (node_index, tensor_index)) - /// - public KerasHistory KerasHistory { get; set; } - /// /// Updates the shape of this tensor. /// diff --git a/src/TensorFlowNET.Keras/Models/ModelsApi.cs b/src/TensorFlowNET.Keras/Models/ModelsApi.cs index 44dca58d0..2605c41e3 100644 --- a/src/TensorFlowNET.Keras/Models/ModelsApi.cs +++ b/src/TensorFlowNET.Keras/Models/ModelsApi.cs @@ -1,22 +1,15 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; -using Tensorflow.Keras.Engine; -using Tensorflow.Keras.Saving; +using Tensorflow.Keras.Saving; using Tensorflow.Keras.Saving.SavedModel; -using ThirdParty.Tensorflow.Python.Keras.Protobuf; -namespace Tensorflow.Keras.Models +namespace Tensorflow.Keras.Models; + +public class ModelsApi: IModelsApi { - public class ModelsApi: IModelsApi - { - public Functional from_config(FunctionalConfig config) - => Functional.from_config(config); + public Functional from_config(FunctionalConfig config) + => Functional.from_config(config); - public IModel load_model(string filepath, bool compile = true, LoadOptions? options = null) - { - return KerasLoadModelUtils.load_model(filepath, compile: compile, options: options) as Model; - } + public IModel load_model(string filepath, bool compile = true, LoadOptions? options = null) + { + return KerasLoadModelUtils.load_model(filepath, compile: compile, options: options) as Model; } } diff --git a/src/TensorFlowNET.Keras/Saving/SavedModel/load.cs b/src/TensorFlowNET.Keras/Saving/SavedModel/load.cs index aa763fc2e..091dbb810 100644 --- a/src/TensorFlowNET.Keras/Saving/SavedModel/load.cs +++ b/src/TensorFlowNET.Keras/Saving/SavedModel/load.cs @@ -1,97 +1,89 @@ -using Google.Protobuf; -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; -using Tensorflow.Keras.Engine; +using System.IO; using Tensorflow.Train; using ThirdParty.Tensorflow.Python.Keras.Protobuf; -using static Tensorflow.Binding; -using static Tensorflow.KerasApi; -namespace Tensorflow.Keras.Saving.SavedModel +namespace Tensorflow.Keras.Saving.SavedModel; + +public class KerasLoadModelUtils { - public class KerasLoadModelUtils + /// + /// Corresponding to keras/saving/save.py/load_model + /// + /// + /// + /// + /// + /// + public static Trackable load_model(string filepath, IDictionary? custom_objects = null, + bool compile = true, LoadOptions? options = null) { - /// - /// Corresponding to keras/saving/save.py/load_model - /// - /// - /// - /// - /// - /// - public static Trackable load_model(string filepath, IDictionary? custom_objects = null, - bool compile = true, LoadOptions? options = null) + using var savingScope = SharedObjectSavingScope.Enter(); + + using var ctx = LoadContext.load_context(options); + + if (!File.Exists(filepath) && !Directory.Exists(filepath)) { - using (SharedObjectSavingScope.Enter()) - { - using (LoadContext.load_context(options)) - { - if (!File.Exists(filepath) && !Directory.Exists(filepath)) - { - throw new IOException($"No file or directory found at {filepath}."); - } - if (Directory.Exists(filepath)) - { - return load(filepath, compile, options); - } - else - { - throw new NotImplementedException("Model load of h5 format has not been supported. Please submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues if it's needed."); - } - } - } + throw new IOException($"No file or directory found at {filepath}."); } - private static Trackable load(string path, bool compile = true, LoadOptions? options = null) + if (Directory.Exists(filepath)) + { + return load(filepath, compile, options); + } + else { - SavedMetadata metadata = new SavedMetadata(); - var meta_graph_def = Loader.parse_saved_model(path).MetaGraphs[0]; - var object_graph_def = meta_graph_def.ObjectGraphDef; - string path_to_metadata_pb = Path.Combine(path, Constants.SAVED_METADATA_PATH); - if (File.Exists(path_to_metadata_pb)) - { - metadata.MergeFrom(new FileStream(path_to_metadata_pb, FileMode.Open, FileAccess.Read)); - } - else - { - throw new NotImplementedException("SavedModel saved prior to TF 2.5 detected when loading Keras model, please" + - " use higher version or submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues. to let us know you need it."); - } + throw new NotImplementedException("Model load of h5 format has not been supported. Please submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues if it's needed."); + } + } - if (metadata.Nodes is null || metadata.Nodes.Count == 0) - { - return Loader.load(path, options: options) as Model; - } + private static Trackable load(string path, bool compile = true, LoadOptions? options = null) + { + SavedMetadata metadata; + var meta_graph_def = Loader.parse_saved_model(path).MetaGraphs[0]; + var object_graph_def = meta_graph_def.ObjectGraphDef; + string path_to_metadata_pb = Path.Combine(path, Constants.SAVED_METADATA_PATH); + if (File.Exists(path_to_metadata_pb)) + { + using var stream = new FileStream(path_to_metadata_pb, FileMode.Open, FileAccess.Read); + metadata = SavedMetadata.Parser.ParseFrom(stream); + } + else + { + throw new NotImplementedException("SavedModel saved prior to TF 2.5 detected when loading Keras model, please" + + " use higher version or submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues. to let us know you need it."); + } - var keras_loader = new KerasObjectLoader(metadata, object_graph_def); - keras_loader.load_layers(compile: compile); + if (metadata.Nodes is null || metadata.Nodes.Count == 0) + { + return Loader.load(path, options: options) as Model; + } - Dictionary)> nodes_to_load = new(); - nodes_to_load["root"] = (null, null); - foreach(var item in keras_loader.LoadedNodes) - { - nodes_to_load[keras_loader.get_path(item.Key)] = item.Value; - } - var loaded = Loader.load_partial(path, nodes_to_load, options); + var keras_loader = new KerasObjectLoader(metadata, object_graph_def); + keras_loader.load_layers(compile: compile); - keras_loader.finalize_objects(); - keras_loader.del_tracking(); + Dictionary)> nodes_to_load = new(); + nodes_to_load["root"] = (null, null); + foreach(var item in keras_loader.LoadedNodes) + { + nodes_to_load[keras_loader.get_path(item.Key)] = item.Value; + } + var loaded = Loader.load_partial(path, nodes_to_load, options); - var model = loaded["root"]; + keras_loader.finalize_objects(); + keras_loader.del_tracking(); - if(model is Model && compile) - { - // TODO(Rinne): implement it. - } + var model = loaded["root"]; - if (!tf.Context.executing_eagerly()) - { - // TODO(Rinne): implement it. - } + if (model is Model && compile) + { + // TODO(Rinne): implement it. + } - return model; + if (!tf.Context.executing_eagerly()) + { + // TODO(Rinne): implement it. } + + return model; } } From 03b44c3b502f38509eff6453a0b40c70d114be76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Fri, 14 Jul 2023 18:39:58 +0800 Subject: [PATCH 09/98] ignore the LSTMLoad test --- test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs b/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs index 299337cde..cb570fc0c 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs @@ -81,6 +81,7 @@ public void ModelWithSelfDefinedModule() model.fit(dataset.Train.Data, dataset.Train.Labels, batch_size, num_epochs); } + [Ignore] [TestMethod] public void LSTMLoad() { From 3bef87aefcb84379af5e838ed2dcb8cdc897b4a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Fri, 14 Jul 2023 23:36:12 +0800 Subject: [PATCH 10/98] fix: make the initialization of the layer's name correct --- .../Utils/generic_utils.cs | 14 +++++--- .../InitLayerNameTest.cs | 33 +++++++++++++++++++ 2 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 test/TensorFlowNET.Keras.UnitTest/InitLayerNameTest.cs diff --git a/src/TensorFlowNET.Keras/Utils/generic_utils.cs b/src/TensorFlowNET.Keras/Utils/generic_utils.cs index 6a59fb880..5402f4995 100644 --- a/src/TensorFlowNET.Keras/Utils/generic_utils.cs +++ b/src/TensorFlowNET.Keras/Utils/generic_utils.cs @@ -29,6 +29,7 @@ limitations under the License. using Tensorflow.Keras.Layers; using Tensorflow.Keras.Saving; using Tensorflow.Train; +using System.Text.RegularExpressions; namespace Tensorflow.Keras.Utils { @@ -126,12 +127,15 @@ public static FunctionalConfig deserialize_model_config(JToken json) public static string to_snake_case(string name) { - return string.Concat(name.Select((x, i) => + string intermediate = Regex.Replace(name, "(.)([A-Z][a-z0-9]+)", "$1_$2"); + string insecure = Regex.Replace(intermediate, "([a-z])([A-Z])", "$1_$2").ToLower(); + + if (insecure[0] != '_') { - return i > 0 && char.IsUpper(x) && !Char.IsDigit(name[i - 1]) ? - "_" + x.ToString() : - x.ToString(); - })).ToLower(); + return insecure; + } + + return "private" + insecure; } /// diff --git a/test/TensorFlowNET.Keras.UnitTest/InitLayerNameTest.cs b/test/TensorFlowNET.Keras.UnitTest/InitLayerNameTest.cs new file mode 100644 index 000000000..256eb69c1 --- /dev/null +++ b/test/TensorFlowNET.Keras.UnitTest/InitLayerNameTest.cs @@ -0,0 +1,33 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Tensorflow.Keras.Layers; +using static Tensorflow.Binding; +using static Tensorflow.KerasApi; + +namespace Tensorflow.Keras.UnitTest +{ + [TestClass] + public class InitLayerNameTest + { + [TestMethod] + public void RNNLayerNameTest() + { + var simpleRnnCell = keras.layers.SimpleRNNCell(1); + Assert.AreEqual("simple_rnn_cell", simpleRnnCell.Name); + var simpleRnn = keras.layers.SimpleRNN(2); + Assert.AreEqual("simple_rnn", simpleRnn.Name); + var lstmCell = keras.layers.LSTMCell(2); + Assert.AreEqual("lstm_cell", lstmCell.Name); + var lstm = keras.layers.LSTM(3); + Assert.AreEqual("lstm", lstm.Name); + } + + [TestMethod] + public void ConvLayerNameTest() + { + var conv2d = keras.layers.Conv2D(8, activation: "linear"); + Assert.AreEqual("conv2d", conv2d.Name); + var conv2dTranspose = keras.layers.Conv2DTranspose(8); + Assert.AreEqual("conv2d_transpose", conv2dTranspose.Name); + } + } +} From 6ec39ba3cbfacb26096903a628db88ece042bf16 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 16 Jul 2023 21:17:40 -0500 Subject: [PATCH 11/98] Fix inferred_value of KerasTensor. #1142 --- src/TensorFlowNET.Core/APIs/tf.reshape.cs | 2 +- src/TensorFlowNET.Core/APIs/tf.tile.cs | 2 +- src/TensorFlowNET.Core/GlobalUsing.cs | 3 +- .../Keras/Engine/KerasTensor.cs | 19 +++++++++--- .../Operations/array_ops.cs | 29 +++++++++++++++++-- src/TensorFlowNET.Core/Tensors/shape_utils.cs | 27 +++++++++++++++++ src/TensorFlowNET.Core/Tensors/tf.constant.cs | 3 ++ src/TensorFlowNET.Core/ops.cs | 11 +++++-- .../Tensorflow.Keras.csproj | 2 +- .../Tensorflow.Binding.UnitTest.csproj | 4 +-- 10 files changed, 88 insertions(+), 14 deletions(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.reshape.cs b/src/TensorFlowNET.Core/APIs/tf.reshape.cs index 5da7b795f..102a81323 100644 --- a/src/TensorFlowNET.Core/APIs/tf.reshape.cs +++ b/src/TensorFlowNET.Core/APIs/tf.reshape.cs @@ -31,6 +31,6 @@ public Tensor reshape(Tensor tensor, public Tensor reshape(Tensor tensor, object[] shape, string name = null) - => gen_array_ops.reshape(tensor, ops.convert_to_tensor(shape), name); + => array_ops.reshape(tensor, shape, name); } } diff --git a/src/TensorFlowNET.Core/APIs/tf.tile.cs b/src/TensorFlowNET.Core/APIs/tf.tile.cs index 65975ac83..1220230d6 100644 --- a/src/TensorFlowNET.Core/APIs/tf.tile.cs +++ b/src/TensorFlowNET.Core/APIs/tf.tile.cs @@ -23,7 +23,7 @@ public Tensor tile(Tensor input, Tensor multiples, string name = null) => gen_array_ops.tile(input, multiples, name); public Tensor tile(Tensor input, object[] multiples, string name = null) - => gen_array_ops.tile(input, ops.convert_to_tensor(multiples), name); + => array_ops.tile(input, multiples, name); public Tensor tile(Tensor input, Shape multiples, string name = null) { diff --git a/src/TensorFlowNET.Core/GlobalUsing.cs b/src/TensorFlowNET.Core/GlobalUsing.cs index 209bc291f..7e02c9083 100644 --- a/src/TensorFlowNET.Core/GlobalUsing.cs +++ b/src/TensorFlowNET.Core/GlobalUsing.cs @@ -5,4 +5,5 @@ global using System.Data; global using System.Linq; global using Tensorflow.Keras.Engine; -global using Tensorflow.Framework.Models; \ No newline at end of file +global using Tensorflow.Framework.Models; +global using static Tensorflow.Binding; \ No newline at end of file diff --git a/src/TensorFlowNET.Core/Keras/Engine/KerasTensor.cs b/src/TensorFlowNET.Core/Keras/Engine/KerasTensor.cs index 9287284f7..5a264b631 100644 --- a/src/TensorFlowNET.Core/Keras/Engine/KerasTensor.cs +++ b/src/TensorFlowNET.Core/Keras/Engine/KerasTensor.cs @@ -30,21 +30,32 @@ public KerasTensor(TensorSpec type_spec, Shape inferred_value = null, string nam public static KerasTensor from_tensor(Tensor tensor) { var type_spec = tensor.ToTensorSpec(); - var kt = new KerasTensor(type_spec, name: tensor.name); + Shape? inferred_value = default; + if (tensor.dtype == TF_DataType.TF_INT32 && tensor.rank < 2) + { + inferred_value = tf.ones(tensor).shape; + } + var kt = new KerasTensor(type_spec, inferred_value: inferred_value, name: tensor.name); kt.original_tensors = tensor; return kt; } + public KerasTensor this[int idx] + => _original_tensors.First()[idx]; + + public KerasTensor this[params Slice[] slices] + => _original_tensors.First()[slices]; + public override string ToString() => _original_tensors.Length switch { - > 1 => "[" + string.Join(", ", _original_tensors.Select(x => $"KerasTensor: shape={x.shape} dtype={x.dtype}")) + "]", - 1 => $"KerasTensor: shape={_original_tensors.shape} {GetInferredValueString()} dtype={_original_tensors.dtype}", + > 1 => "[" + string.Join(", ", _original_tensors.Select(x => $"KerasTensor: shape={x.shape} dtype={x.dtype.as_numpy_name()}{GetInferredValueString()}")) + "]", + 1 => $"KerasTensor: shape={_original_tensors.shape} dtype={_original_tensors.dtype.as_numpy_name()}{GetInferredValueString()}", _ => _original_tensors.ToString(), }; private string GetInferredValueString() - => _inferred_value == null ? "" : ""; + => _inferred_value == null ? "" : $" inferred_value={_inferred_value}"; public static implicit operator Tensors(KerasTensor kt) => kt._original_tensors; diff --git a/src/TensorFlowNET.Core/Operations/array_ops.cs b/src/TensorFlowNET.Core/Operations/array_ops.cs index 02bf0e868..9d4647fac 100644 --- a/src/TensorFlowNET.Core/Operations/array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/array_ops.cs @@ -137,7 +137,7 @@ public static Tensor zeros(Tensors shape, TF_DataType dtype = TF_DataType.TF_FLO if(shape.Length > 1) { shapeTensor = ops.convert_to_tensor(shape, dtypes.int32); - if(shapeTensor.ndim > 1) + if (shapeTensor.ndim > 1) { shapeTensor = array_ops.reshape(shapeTensor, new Shape(-1)); } @@ -304,6 +304,10 @@ public static Tensor _autopacking_helper(IEnumerable list_or_tuple, TF_D { elems_as_tensors.Add(tensor); } + else if (elem is KerasTensor kt) + { + elems_as_tensors.Add(kt); + } else { var elem_tensor = constant_op.constant(elem, dtype: dtype, name: i.ToString()); @@ -404,7 +408,10 @@ public static Tensor reshape(Tensor tensor, Shape shape, string name = null) => gen_array_ops.reshape(tensor, shape, name: name); public static Tensor reshape(Tensor tensor, object[] shape, string name = null) - => gen_array_ops.reshape(tensor, ops.convert_to_tensor(shape), name: name); + { + var dims = shape_utils.from_object_array(shape); + return gen_array_ops.reshape(tensor, dims, name: name); + } private static Tensor ones_like_impl(T tensor, TF_DataType dtype, string name, bool optimize = true) { @@ -425,6 +432,10 @@ public static Tensor ones(Tensor shape, TF_DataType dtype = TF_DataType.TF_FLOAT return tf_with(ops.name_scope(name, "ones", new { shape }), scope => { name = scope; + if (shape._shape_tuple().Length == 0) + { + shape = reshape(shape, new Shape(-1)); + } var output = gen_array_ops.fill(shape, constant_op.constant(1.0f, dtype: dtype), name: name); return output; }); @@ -647,6 +658,20 @@ public static Tensor tile(Tensor input, Tensor multiples, string name = null) } }); + public static Tensor tile(Tensor input, object[] multiples, string name = null) + { + Shape dims = shape_utils.from_object_array(multiples); + + return tf.Context.ExecuteOp("Tile", name, new ExecuteOpArgs(input, dims) + { + GetGradientAttrs = (op) => new + { + T = op.get_attr("T"), + Tmultiples = op.get_attr("Tmultiples") + } + }); + } + public static Tensor zeros_like(Tensor tensor, TF_DataType dtype = TF_DataType.DtInvalid, string name = null, bool optimize = true) { return tf_with(ops.name_scope(name, "zeros_like", new Tensor[] { tensor }), scope => diff --git a/src/TensorFlowNET.Core/Tensors/shape_utils.cs b/src/TensorFlowNET.Core/Tensors/shape_utils.cs index 254cdad89..a77dd34ce 100644 --- a/src/TensorFlowNET.Core/Tensors/shape_utils.cs +++ b/src/TensorFlowNET.Core/Tensors/shape_utils.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using Tensorflow.Eager; using static Tensorflow.Binding; namespace Tensorflow @@ -13,5 +14,31 @@ public static Tensor static_or_dynamic_map_fn(Func fn, Tensor el throw new NotImplementedException(""); } + + public static Shape from_object_array(object[] shape) + { + var dims = shape.Select(x => + { + if (x is KerasTensor kt && kt.inferred_value != null) + { + return kt.inferred_value.as_int_list()[0]; + } + else if (x is EagerTensor et && et.dtype == TF_DataType.TF_INT32) + { + return et.ToArray()[0]; + } + else if (x is int i) + { + return i; + } + else if (x is long l) + { + return l; + } + throw new NotImplementedException(); + }).ToArray(); + + return new Shape(dims); + } } } diff --git a/src/TensorFlowNET.Core/Tensors/tf.constant.cs b/src/TensorFlowNET.Core/Tensors/tf.constant.cs index 6a62d34a5..ac26b3da3 100644 --- a/src/TensorFlowNET.Core/Tensors/tf.constant.cs +++ b/src/TensorFlowNET.Core/Tensors/tf.constant.cs @@ -46,6 +46,9 @@ public Tensor zeros(Tensor shape, TF_DataType dtype = TF_DataType.TF_FLOAT, stri public Tensor ones(Shape shape, TF_DataType dtype = TF_DataType.TF_FLOAT, string name = null) => array_ops.ones(shape, dtype, name); + public Tensor ones(Tensor shape, TF_DataType dtype = TF_DataType.TF_FLOAT, string name = null) + => array_ops.ones(shape, dtype, name); + public Tensor size(Tensor input, string name = null, TF_DataType out_type = TF_DataType.TF_INT32) => array_ops.size(input, diff --git a/src/TensorFlowNET.Core/ops.cs b/src/TensorFlowNET.Core/ops.cs index c624c9901..351fd18ff 100644 --- a/src/TensorFlowNET.Core/ops.cs +++ b/src/TensorFlowNET.Core/ops.cs @@ -144,11 +144,18 @@ public static Tensor convert_to_tensor(object value, } if (!graph.building_function) { - throw new RuntimeError("Attempting to capture an EagerTensor without building a function."); - // return eager_tensor.AsPlaceholder(name: name); + // throw new RuntimeError("Attempting to capture an EagerTensor without building a function."); + return eager_tensor.AsPlaceholder(name: name); } } } + else if (value is KerasTensor kt) + { + if (kt.inferred_value != null) + { + return convert_to_tensor(kt.inferred_value, dtype: kt.dtype, name: name); + } + } // graph mode Tensor ret = value switch diff --git a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj index c7fa7711c..eeb7c559f 100644 --- a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj +++ b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj @@ -141,7 +141,7 @@ Keras is an API designed for human beings, not machines. Keras follows best prac - + diff --git a/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj b/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj index 240960c91..7a6a7f92c 100644 --- a/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj +++ b/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj @@ -41,8 +41,8 @@ - - + + From 03472997e43ab36d447ca520907ee8dffcc03edc Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Tue, 18 Jul 2023 07:01:51 -0500 Subject: [PATCH 12/98] Fix tf.reverse. --- src/TensorFlowNET.Core/APIs/tf.array.cs | 15 +++++++++------ src/TensorFlowNET.Core/Operations/array_ops.cs | 18 +++++++++++++----- .../ManagedAPI/ArrayOpsTest.cs | 13 +++++++++++++ 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.array.cs b/src/TensorFlowNET.Core/APIs/tf.array.cs index ecac37eb1..4d9c3da58 100644 --- a/src/TensorFlowNET.Core/APIs/tf.array.cs +++ b/src/TensorFlowNET.Core/APIs/tf.array.cs @@ -162,14 +162,17 @@ public Tensor transpose(T1 a, Axis perm = null, string name = "transpose", b /// Reverses specific dimensions of a tensor. /// /// - /// + /// The indices of the dimensions to reverse. Must be in the range [-rank(tensor), rank(tensor)). /// /// - public Tensor reverse(Tensor tensor, int[] axis, string name = null) - => gen_array_ops.reverse(tensor, ops.convert_to_tensor(axis), name: name); - - public Tensor reverse(Tensor tensor, Tensor axis, string name = null) - => gen_array_ops.reverse(tensor, axis, name: name); + public Tensor reverse(Tensor tensor, Axis axis, string name = null) + { + if (axis.IsScalar) + { + axis = new Axis(axis.axis); + } + return array_ops.reverse(tensor, axis, name: name); + } /// /// Returns the rank of a tensor. diff --git a/src/TensorFlowNET.Core/Operations/array_ops.cs b/src/TensorFlowNET.Core/Operations/array_ops.cs index 9d4647fac..f80dcd2c4 100644 --- a/src/TensorFlowNET.Core/Operations/array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/array_ops.cs @@ -413,6 +413,16 @@ public static Tensor reshape(Tensor tensor, object[] shape, string name = null) return gen_array_ops.reshape(tensor, dims, name: name); } + public static Tensor reverse(Tensor tensor, Tensor axis, string name = null) + => tf.Context.ExecuteOp("ReverseV2", name, new ExecuteOpArgs(tensor, axis) + { + GetGradientAttrs = (op) => new + { + T = op.get_attr("T"), + Tidx = op.get_attr("Tidx") + } + }); + private static Tensor ones_like_impl(T tensor, TF_DataType dtype, string name, bool optimize = true) { return tf_with(ops.name_scope(name, "ones_like", new { tensor }), scope => @@ -658,11 +668,9 @@ public static Tensor tile(Tensor input, Tensor multiples, string name = null) } }); - public static Tensor tile(Tensor input, object[] multiples, string name = null) + /*public static Tensor tile(Tensor input, Shape multiples, string name = null) { - Shape dims = shape_utils.from_object_array(multiples); - - return tf.Context.ExecuteOp("Tile", name, new ExecuteOpArgs(input, dims) + return tf.Context.ExecuteOp("Tile", name, new ExecuteOpArgs(input, multiples) { GetGradientAttrs = (op) => new { @@ -670,7 +678,7 @@ public static Tensor tile(Tensor input, object[] multiples, string name = null) Tmultiples = op.get_attr("Tmultiples") } }); - } + }*/ public static Tensor zeros_like(Tensor tensor, TF_DataType dtype = TF_DataType.DtInvalid, string name = null, bool optimize = true) { diff --git a/test/TensorFlowNET.UnitTest/ManagedAPI/ArrayOpsTest.cs b/test/TensorFlowNET.UnitTest/ManagedAPI/ArrayOpsTest.cs index 72f598e46..675689bb1 100644 --- a/test/TensorFlowNET.UnitTest/ManagedAPI/ArrayOpsTest.cs +++ b/test/TensorFlowNET.UnitTest/ManagedAPI/ArrayOpsTest.cs @@ -2,6 +2,7 @@ using Tensorflow.NumPy; using Tensorflow; using static Tensorflow.Binding; +using System.Linq; namespace TensorFlowNET.UnitTest.ManagedAPI { @@ -92,5 +93,17 @@ public void TensorArray() Assert.AreEqual(ta.read(1).numpy(), 20f); Assert.AreEqual(ta.read(2).numpy(), 30f); } + + /// + /// https://www.tensorflow.org/api_docs/python/tf/reverse + /// + [TestMethod] + public void ReverseArray() + { + var a = tf.random.normal((2, 3)); + var b = tf.reverse(a, -1); + Assert.IsTrue(Equal(a[0].ToArray().Reverse().ToArray(), b[0].ToArray())); + Assert.IsTrue(Equal(a[1].ToArray().Reverse().ToArray(), b[1].ToArray())); + } } } From fa5d19dcdab55d7b81afc614f9929bc85c52cb20 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Tue, 18 Jul 2023 07:08:39 -0500 Subject: [PATCH 13/98] fix unit test. --- src/TensorFlowNET.Core/APIs/tf.tile.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.tile.cs b/src/TensorFlowNET.Core/APIs/tf.tile.cs index 1220230d6..a3b497e8a 100644 --- a/src/TensorFlowNET.Core/APIs/tf.tile.cs +++ b/src/TensorFlowNET.Core/APIs/tf.tile.cs @@ -23,7 +23,7 @@ public Tensor tile(Tensor input, Tensor multiples, string name = null) => gen_array_ops.tile(input, multiples, name); public Tensor tile(Tensor input, object[] multiples, string name = null) - => array_ops.tile(input, multiples, name); + => array_ops.tile(input, constant_op.constant(shape_utils.from_object_array(multiples).dims), name); public Tensor tile(Tensor input, Shape multiples, string name = null) { From 0c9437afcb9cc5852abcbd31bcb85c08afef0ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Tue, 18 Jul 2023 23:31:45 +0800 Subject: [PATCH 14/98] feat: add Bidirectional layer --- .../ArgsDefinition/Rnn/BidirectionalArgs.cs | 20 ++ .../Keras/ArgsDefinition/Rnn/LSTMArgs.cs | 5 + .../Keras/ArgsDefinition/Rnn/RNNArgs.cs | 5 + .../Keras/ArgsDefinition/Rnn/WrapperArgs.cs | 24 ++ .../Keras/Layers/ILayersApi.cs | 14 +- src/TensorFlowNET.Keras/Layers/LayersApi.cs | 14 + .../Layers/Rnn/BaseWrapper.cs | 33 +++ .../Layers/Rnn/Bidirectional.cs | 276 ++++++++++++++++++ src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs | 31 +- src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs | 11 +- .../Layers/Rnn.Test.cs | 13 +- 11 files changed, 428 insertions(+), 18 deletions(-) create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/BidirectionalArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/WrapperArgs.cs create mode 100644 src/TensorFlowNET.Keras/Layers/Rnn/BaseWrapper.cs create mode 100644 src/TensorFlowNET.Keras/Layers/Rnn/Bidirectional.cs diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/BidirectionalArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/BidirectionalArgs.cs new file mode 100644 index 000000000..d658a82e9 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/BidirectionalArgs.cs @@ -0,0 +1,20 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.NumPy; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class BidirectionalArgs : AutoSerializeLayerArgs + { + [JsonProperty("layer")] + public ILayer Layer { get; set; } + [JsonProperty("merge_mode")] + public string? MergeMode { get; set; } + [JsonProperty("backward_layer")] + public ILayer BackwardLayer { get; set; } + public NDArray Weights { get; set; } + } + +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs index d816b0ff7..a6beb77e8 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs @@ -5,5 +5,10 @@ public class LSTMArgs : RNNArgs // TODO: maybe change the `RNNArgs` and implement this class. public bool UnitForgetBias { get; set; } public int Implementation { get; set; } + + public LSTMArgs Clone() + { + return (LSTMArgs)MemberwiseClone(); + } } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs index b84d30d3d..d0b73ba44 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs @@ -40,5 +40,10 @@ public class RNNArgs : AutoSerializeLayerArgs public bool ZeroOutputForMask { get; set; } = false; [JsonProperty("recurrent_dropout")] public float RecurrentDropout { get; set; } = .0f; + + public RNNArgs Clone() + { + return (RNNArgs)MemberwiseClone(); + } } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/WrapperArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/WrapperArgs.cs new file mode 100644 index 000000000..ec8e16d59 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/WrapperArgs.cs @@ -0,0 +1,24 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Text; + + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class WrapperArgs : AutoSerializeLayerArgs + { + [JsonProperty("layer")] + public ILayer Layer { get; set; } + + public WrapperArgs(ILayer layer) + { + Layer = layer; + } + + public static implicit operator WrapperArgs(BidirectionalArgs args) + => new WrapperArgs(args.Layer); + } + +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index 1670f9d1d..b8aff5fb6 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -258,7 +258,19 @@ public IRnnCell GRUCell( float dropout = 0f, float recurrent_dropout = 0f, bool reset_after = true); - + + /// + /// Bidirectional wrapper for RNNs. + /// + /// `keras.layers.RNN` instance, such as `keras.layers.LSTM` or `keras.layers.GRU` + /// automatically. + /// + public ILayer Bidirectional( + ILayer layer, + string merge_mode = "concat", + NDArray weights = null, + ILayer backward_layer = null); + public ILayer Subtract(); } } diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index cb85bbba1..a04a9c051 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -908,6 +908,20 @@ public IRnnCell GRUCell( ResetAfter = reset_after }); + public ILayer Bidirectional( + ILayer layer, + string merge_mode = "concat", + NDArray weights = null, + ILayer backward_layer = null) + => new Bidirectional(new BidirectionalArgs + { + Layer = layer, + MergeMode = merge_mode, + Weights = weights, + BackwardLayer = backward_layer + }); + + /// /// /// diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/BaseWrapper.cs b/src/TensorFlowNET.Keras/Layers/Rnn/BaseWrapper.cs new file mode 100644 index 000000000..737f88cd4 --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Rnn/BaseWrapper.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.Saving; + +namespace Tensorflow.Keras.Layers +{ + /// + /// Abstract wrapper base class. Wrappers take another layer and augment it in various ways. + /// Do not use this class as a layer, it is only an abstract base class. + /// Two usable wrappers are the `TimeDistributed` and `Bidirectional` wrappers. + /// + public abstract class Wrapper: Layer + { + public ILayer _layer; + public Wrapper(WrapperArgs args):base(args) + { + _layer = args.Layer; + } + + public virtual void Build(KerasShapesWrapper input_shape) + { + if (!_layer.Built) + { + _layer.build(input_shape); + } + built = true; + } + + } +} diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/Bidirectional.cs b/src/TensorFlowNET.Keras/Layers/Rnn/Bidirectional.cs new file mode 100644 index 000000000..6114d9c7c --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Rnn/Bidirectional.cs @@ -0,0 +1,276 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tensorflow.Common.Types; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.Saving; + +namespace Tensorflow.Keras.Layers +{ + /// + /// Bidirectional wrapper for RNNs. + /// + public class Bidirectional: Wrapper + { + BidirectionalArgs _args; + RNN _forward_layer; + RNN _backward_layer; + RNN _layer; + bool _support_masking = true; + int _num_constants = 0; + bool _return_state; + bool _stateful; + bool _return_sequences; + InputSpec _input_spec; + RNNArgs _layer_args_copy; + public Bidirectional(BidirectionalArgs args):base(args) + { + _args = args; + if (_args.Layer is not ILayer) + throw new ValueError( + "Please initialize `Bidirectional` layer with a " + + $"`tf.keras.layers.Layer` instance. Received: {_args.Layer}"); + + if (_args.BackwardLayer is not null && _args.BackwardLayer is not ILayer) + throw new ValueError( + "`backward_layer` need to be a `tf.keras.layers.Layer` " + + $"instance. Received: {_args.BackwardLayer}"); + if (!new List { "sum", "mul", "ave", "concat", null }.Contains(_args.MergeMode)) + { + throw new ValueError( + $"Invalid merge mode. Received: {_args.MergeMode}. " + + "Merge mode should be one of " + + "{\"sum\", \"mul\", \"ave\", \"concat\", null}" + ); + } + if (_args.Layer is RNN) + { + _layer = _args.Layer as RNN; + } + else + { + throw new ValueError( + "Bidirectional only support RNN instance such as LSTM or GRU"); + } + _return_state = _layer.Args.ReturnState; + _return_sequences = _layer.Args.ReturnSequences; + _stateful = _layer.Args.Stateful; + _layer_args_copy = _layer.Args.Clone(); + // We don't want to track `layer` since we're already tracking the two + // copies of it we actually run. + // TODO(Wanglongzhi2001), since the feature of setattr_tracking has not been implemented. + // _setattr_tracking = false; + // super().__init__(layer, **kwargs) + // _setattr_tracking = true; + + // Recreate the forward layer from the original layer config, so that it + // will not carry over any state from the layer. + var actualType = _layer.GetType(); + if (actualType == typeof(LSTM)) + { + var arg = _layer_args_copy as LSTMArgs; + _forward_layer = new LSTM(arg); + } + // TODO(Wanglongzhi2001), add GRU if case. + else + { + _forward_layer = new RNN(_layer.Cell, _layer_args_copy); + } + //_forward_layer = _recreate_layer_from_config(_layer); + if (_args.BackwardLayer is null) + { + _backward_layer = _recreate_layer_from_config(_layer, go_backwards:true); + } + else + { + _backward_layer = _args.BackwardLayer as RNN; + } + _forward_layer.Name = "forward_" + _forward_layer.Name; + _backward_layer.Name = "backward_" + _backward_layer.Name; + _verify_layer_config(); + + void force_zero_output_for_mask(RNN layer) + { + layer.Args.ZeroOutputForMask = layer.Args.ReturnSequences; + } + + force_zero_output_for_mask(_forward_layer); + force_zero_output_for_mask(_backward_layer); + + if (_args.Weights is not null) + { + var nw = len(_args.Weights); + _forward_layer.set_weights(_args.Weights[$":,{nw / 2}"]); + _backward_layer.set_weights(_args.Weights[$"{nw / 2},:"]); + } + + _input_spec = _layer.InputSpec; + } + + private void _verify_layer_config() + { + if (_forward_layer.Args.GoBackwards == _backward_layer.Args.GoBackwards) + { + throw new ValueError( + "Forward layer and backward layer should have different " + + "`go_backwards` value." + + "forward_layer.go_backwards = " + + $"{_forward_layer.Args.GoBackwards}," + + "backward_layer.go_backwards = " + + $"{_backward_layer.Args.GoBackwards}"); + } + if (_forward_layer.Args.Stateful != _backward_layer.Args.Stateful) + { + throw new ValueError( + "Forward layer and backward layer are expected to have "+ + $"the same value for attribute stateful, got "+ + $"{_forward_layer.Args.Stateful} for forward layer and "+ + $"{_backward_layer.Args.Stateful} for backward layer"); + } + if (_forward_layer.Args.ReturnState != _backward_layer.Args.ReturnState) + { + throw new ValueError( + "Forward layer and backward layer are expected to have " + + $"the same value for attribute return_state, got " + + $"{_forward_layer.Args.ReturnState} for forward layer and " + + $"{_backward_layer.Args.ReturnState} for backward layer"); + } + if (_forward_layer.Args.ReturnSequences != _backward_layer.Args.ReturnSequences) + { + throw new ValueError( + "Forward layer and backward layer are expected to have " + + $"the same value for attribute return_sequences, got " + + $"{_forward_layer.Args.ReturnSequences} for forward layer and " + + $"{_backward_layer.Args.ReturnSequences} for backward layer"); + } + } + + private RNN _recreate_layer_from_config(RNN layer, bool go_backwards = false) + { + var config = layer.get_config() as RNNArgs; + var cell = layer.Cell; + if (go_backwards) + { + config.GoBackwards = !config.GoBackwards; + } + var actualType = layer.GetType(); + if (actualType == typeof(LSTM)) + { + var arg = config as LSTMArgs; + return new LSTM(arg); + } + else + { + return new RNN(cell, config); + } + } + + public override void build(KerasShapesWrapper input_shape) + { + _buildInputShape = input_shape; + tf_with(ops.name_scope(_forward_layer.Name), scope=> + { + _forward_layer.build(input_shape); + }); + tf_with(ops.name_scope(_backward_layer.Name), scope => + { + _backward_layer.build(input_shape); + }); + built = true; + } + + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) + { + // `Bidirectional.call` implements the same API as the wrapped `RNN`. + + Tensors forward_inputs; + Tensors backward_inputs; + Tensors forward_state; + Tensors backward_state; + // if isinstance(inputs, list) and len(inputs) > 1: + if (inputs.Length > 1) + { + // initial_states are keras tensors, which means they are passed + // in together with inputs as list. The initial_states need to be + // split into forward and backward section, and be feed to layers + // accordingly. + forward_inputs = new Tensors { inputs[0] }; + backward_inputs = new Tensors { inputs[0] }; + var pivot = (len(inputs) - _num_constants) / 2 + 1; + // add forward initial state + forward_inputs.Concat(new Tensors { inputs[$"1:{pivot}"] }); + if (_num_constants != 0) + // add backward initial state + backward_inputs.Concat(new Tensors { inputs[$"{pivot}:"] }); + else + { + // add backward initial state + backward_inputs.Concat(new Tensors { inputs[$"{pivot}:{-_num_constants}"] }); + // add constants for forward and backward layers + forward_inputs.Concat(new Tensors { inputs[$"{-_num_constants}:"] }); + backward_inputs.Concat(new Tensors { inputs[$"{-_num_constants}:"] }); + } + forward_state = null; + backward_state = null; + } + else if (state is not null) + { + // initial_states are not keras tensors, eg eager tensor from np + // array. They are only passed in from kwarg initial_state, and + // should be passed to forward/backward layer via kwarg + // initial_state as well. + forward_inputs = inputs; + backward_inputs = inputs; + var half = len(state) / 2; + forward_state = state[$":{half}"]; + backward_state = state[$"{half}:"]; + } + else + { + forward_inputs = inputs; + backward_inputs = inputs; + forward_state = null; + backward_state = null; + } + var y = _forward_layer.Apply(forward_inputs, forward_state); + var y_rev = _backward_layer.Apply(backward_inputs, backward_state); + + Tensors states = new(); + if (_return_state) + { + states = y["1:"] + y_rev["1:"]; + y = y[0]; + y_rev = y_rev[0]; + } + + if (_return_sequences) + { + int time_dim = _forward_layer.Args.TimeMajor ? 0 : 1; + y_rev = keras.backend.reverse(y_rev, time_dim); + } + Tensors output; + if (_args.MergeMode == "concat") + output = keras.backend.concatenate(new Tensors { y.Single(), y_rev.Single() }); + else if (_args.MergeMode == "sum") + output = y.Single() + y_rev.Single(); + else if (_args.MergeMode == "ave") + output = (y.Single() + y_rev.Single()) / 2; + else if (_args.MergeMode == "mul") + output = y.Single() * y_rev.Single(); + else if (_args.MergeMode is null) + output = new Tensors { y.Single(), y_rev.Single() }; + else + throw new ValueError( + "Unrecognized value for `merge_mode`. " + + $"Received: {_args.MergeMode}" + + "Expected values are [\"concat\", \"sum\", \"ave\", \"mul\"]"); + if (_return_state) + { + if (_args.MergeMode is not null) + return new Tensors { output.Single(), states.Single()}; + } + return output; + } + } +} diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs b/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs index b5d583248..c766e8d69 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs @@ -3,6 +3,7 @@ using Tensorflow.Keras.Engine; using Tensorflow.Common.Types; using Tensorflow.Common.Extensions; +using Tensorflow.Keras.Saving; namespace Tensorflow.Keras.Layers { @@ -14,15 +15,15 @@ namespace Tensorflow.Keras.Layers /// public class LSTM : RNN { - LSTMArgs args; + LSTMArgs _args; InputSpec[] _state_spec; InputSpec _input_spec; bool _could_use_gpu_kernel; - + public LSTMArgs Args { get => _args; } public LSTM(LSTMArgs args) : base(CreateCell(args), args) { - this.args = args; + _args = args; _input_spec = new InputSpec(ndim: 3); _state_spec = new[] { args.Units, args.Units }.Select(dim => new InputSpec(shape: (-1, dim))).ToArray(); _could_use_gpu_kernel = args.Activation == keras.activations.Tanh @@ -71,7 +72,7 @@ protected override Tensors Call(Tensors inputs, Tensors initial_state = null, bo var single_input = inputs.Single; var input_shape = single_input.shape; - var timesteps = args.TimeMajor ? input_shape[0] : input_shape[1]; + var timesteps = _args.TimeMajor ? input_shape[0] : input_shape[1]; _maybe_reset_cell_dropout_mask(Cell); @@ -87,26 +88,26 @@ protected override Tensors Call(Tensors inputs, Tensors initial_state = null, bo inputs, initial_state, constants: null, - go_backwards: args.GoBackwards, + go_backwards: _args.GoBackwards, mask: mask, - unroll: args.Unroll, + unroll: _args.Unroll, input_length: ops.convert_to_tensor(timesteps), - time_major: args.TimeMajor, - zero_output_for_mask: args.ZeroOutputForMask, - return_all_outputs: args.ReturnSequences + time_major: _args.TimeMajor, + zero_output_for_mask: _args.ZeroOutputForMask, + return_all_outputs: _args.ReturnSequences ); Tensor output; - if (args.ReturnSequences) + if (_args.ReturnSequences) { - output = keras.backend.maybe_convert_to_ragged(false, outputs, (int)timesteps, args.GoBackwards); + output = keras.backend.maybe_convert_to_ragged(false, outputs, (int)timesteps, _args.GoBackwards); } else { output = last_output; } - if (args.ReturnState) + if (_args.ReturnState) { return new Tensor[] { output }.Concat(states).ToArray().ToTensors(); } @@ -115,5 +116,11 @@ protected override Tensors Call(Tensors inputs, Tensors initial_state = null, bo return output; } } + + public override IKerasConfig get_config() + { + return _args; + } + } } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index 0e81d20e3..c19222614 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -31,7 +31,9 @@ public class RNN : RnnBase protected IVariableV1 _kernel; protected IVariableV1 _bias; private IRnnCell _cell; - protected IRnnCell Cell + + public RNNArgs Args { get => _args; } + public IRnnCell Cell { get { @@ -570,10 +572,13 @@ protected Tensors get_initial_state(Tensors inputs) var input_shape = array_ops.shape(inputs); var batch_size = _args.TimeMajor ? input_shape[1] : input_shape[0]; var dtype = input.dtype; - Tensors init_state = Cell.GetInitialState(null, batch_size, dtype); - return init_state; } + + public override IKerasConfig get_config() + { + return _args; + } } } diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs index 5f7bd574e..03159346a 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs @@ -5,6 +5,7 @@ using System.Text; using System.Threading.Tasks; using Tensorflow.Common.Types; +using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Layers; using Tensorflow.Keras.Saving; @@ -38,8 +39,6 @@ public void StackedRNNCell() var cells = new IRnnCell[] { tf.keras.layers.SimpleRNNCell(4), tf.keras.layers.SimpleRNNCell(5) }; var stackedRNNCell = tf.keras.layers.StackedRNNCells(cells); var (output, state) = stackedRNNCell.Apply(inputs, states); - Console.WriteLine(output); - Console.WriteLine(state.shape); Assert.AreEqual((32, 5), output.shape); Assert.AreEqual((32, 4), state[0].shape); } @@ -108,6 +107,7 @@ public void RNNForSimpleRNNCell() var inputs = tf.random.normal((32, 10, 8)); var cell = tf.keras.layers.SimpleRNNCell(10, dropout: 0.5f, recurrent_dropout: 0.5f); var rnn = tf.keras.layers.RNN(cell: cell); + var cgf = rnn.get_config(); var output = rnn.Apply(inputs); Assert.AreEqual((32, 10), output.shape); @@ -145,5 +145,14 @@ public void GRUCell() Assert.AreEqual((32, 4), output.shape); } + + [TestMethod] + public void Bidirectional() + { + var bi = tf.keras.layers.Bidirectional(keras.layers.LSTM(10, return_sequences:true)); + var inputs = tf.random.normal((32, 10, 8)); + var outputs = bi.Apply(inputs); + Assert.AreEqual((32, 10, 20), outputs.shape); + } } } From 737910df9e3eca18e094a2bffefa5516efc9ebf3 Mon Sep 17 00:00:00 2001 From: Beacontownfc <89081023+Beacontownfc@users.noreply.github.com> Date: Sat, 22 Jul 2023 14:23:08 +0800 Subject: [PATCH 15/98] Fix: model.load_weights --- src/TensorFlowNET.Keras/Saving/hdf5_format.cs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs index 8ac9fddf6..dd6609bc7 100644 --- a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs +++ b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs @@ -133,10 +133,8 @@ public static void load_weights_from_hdf5_group(long f, List layers) long g = H5G.open(f, name); var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); foreach (var i_ in weight_names) - { - var vm = Regex.Replace(i_, "/", "$"); - vm = i_.Split('/')[0] + "/$" + vm.Substring(i_.Split('/')[0].Length + 1, i_.Length - i_.Split('/')[0].Length - 1); - (success, Array result) = Hdf5.ReadDataset(g, vm); + { + (success, Array result) = Hdf5.ReadDataset(g, i_); if (success) weight_values.Add(np.array(result)); } @@ -196,9 +194,14 @@ public static void save_weights_to_hdf5_group(long f, List layers) var tensor = val.AsTensor(); if (name.IndexOf("/") > 1) { - var crDataGroup = Hdf5.CreateOrOpenGroup(g, Hdf5Utils.NormalizedName(name.Split('/')[0])); - var _name = Regex.Replace(name.Substring(name.Split('/')[0].Length, name.Length - name.Split('/')[0].Length), "/", "$"); - WriteDataset(crDataGroup, _name, tensor); + var crDataGroup = g; + string[] name_split = name.Split('/'); + for(int i = 0; i < name_split.Length; i++) + { + if (i == name_split.Length - 1) break; + crDataGroup = Hdf5.CreateOrOpenGroup(crDataGroup, Hdf5Utils.NormalizedName(name_split[i])); + } + WriteDataset(crDataGroup, name_split[name_split.Length - 1], tensor); Hdf5.CloseGroup(crDataGroup); } else From 05dbe134f8f00fa62aa9cda2337891f4ce66c453 Mon Sep 17 00:00:00 2001 From: Beacontownfc <89081023+Beacontownfc@users.noreply.github.com> Date: Sat, 22 Jul 2023 14:32:33 +0800 Subject: [PATCH 16/98] Update hdf5_format.cs --- src/TensorFlowNET.Keras/Saving/hdf5_format.cs | 707 +++++++++--------- 1 file changed, 353 insertions(+), 354 deletions(-) diff --git a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs index dd6609bc7..c80f653f8 100644 --- a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs +++ b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs @@ -1,355 +1,354 @@ -using System; -using System.Collections.Generic; -using System.Text; -using HDF.PInvoke; -using Tensorflow.NumPy; -using HDF5CSharp; -using static Tensorflow.Binding; -using static Tensorflow.KerasApi; -using System.Linq; -using System.Text.RegularExpressions; - -namespace Tensorflow.Keras.Saving -{ - public class hdf5_format - { - private static int HDF5_OBJECT_HEADER_LIMIT = 64512; - public static void load_model_from_hdf5(string filepath = "", Dictionary custom_objects = null, bool compile = false) - { - long root = Hdf5.OpenFile(filepath,true); - load_model_from_hdf5(root, custom_objects, compile); - } - public static void load_model_from_hdf5(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - //long fileId = filepath; - //try - //{ - // groupId = H5G.open(fileId, "/"); - // (bool success, string[] attrId) = Hdf5.ReadStringAttributes(groupId, "model_config", ""); - // H5G.close(groupId); - // if (success == true) { - // Console.WriteLine(attrId[0]); - // } - //} - //catch (Exception ex) - //{ - // if (filepath != -1) { - // Hdf5.CloseFile(filepath); - // } - // if (groupId != -1) { - // H5G.close(groupId); - // } - // throw new Exception(ex.ToString()); - //} - - } - public static void save_model_to_hdf5(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - /// - /// Preprocess layer weights between different Keras formats. - /// - /// - /// - /// - /// - public static List preprocess_weights_for_loading(ILayer layer, List weights, string original_keras_version = null, string original_backend = null) - { - // convert CuDNN layers - return _convert_rnn_weights(layer, weights); - } - - /// - /// Converts weights for RNN layers between native and CuDNN format. - /// - /// - /// - static List _convert_rnn_weights(ILayer layer, List weights) - { - var target_class = layer.GetType().Name; - return weights; - } - - public static void save_optimizer_weights_to_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static void load_optimizer_weights_from_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static void load_weights_from_hdf5_group(long f, List layers) - { - string original_keras_version = "2.5.0"; - string original_backend = null; - var (success, attr) = Hdf5.ReadStringAttributes(f, "keras_version", "", true); - if (success) - original_keras_version = attr.First(); - // keras version should be 2.5.0+ - var ver_major = int.Parse(original_keras_version.Split('.')[0]); - var ver_minor = int.Parse(original_keras_version.Split('.')[1]); - if (ver_major < 2 || (ver_major == 2 && ver_minor < 5)) - throw new ValueError("keras version should be 2.5.0 or later."); - - (success, attr) = Hdf5.ReadStringAttributes(f, "backend", "", true); - if (success) - original_backend = attr.First(); - - var filtered_layers = new List(); - foreach (var layer in layers) - { - var weights = _legacy_weights(layer); - if (weights.Count > 0) - filtered_layers.append(layer); - } - - string[] layer_names = load_attributes_from_hdf5_group(f, "layer_names"); - var filtered_layer_names = new List(); - foreach(var name in layer_names) - { - if (!filtered_layers.Select(x => x.Name).Contains(name)) - continue; - long g = H5G.open(f, name); - var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); - if (weight_names.Count() > 0) - filtered_layer_names.Add(name); - H5G.close(g); - } - - layer_names = filtered_layer_names.ToArray(); - if (layer_names.Length != filtered_layers.Count()) - throw new ValueError("You are trying to load a weight file " + - $"containing {layer_names}" + - $" layers into a model with {filtered_layers.Count} layers."); - - var weight_value_tuples = new List<(IVariableV1, NDArray)>(); - foreach (var (k, name) in enumerate(layer_names)) - { - var weight_values = new List(); - long g = H5G.open(f, name); - var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); - foreach (var i_ in weight_names) - { - (success, Array result) = Hdf5.ReadDataset(g, i_); - if (success) - weight_values.Add(np.array(result)); - } - H5G.close(g); - var layer = filtered_layers[k]; - var symbolic_weights = _legacy_weights(layer); - preprocess_weights_for_loading(layer, weight_values, original_keras_version, original_backend); - if (weight_values.Count() != symbolic_weights.Count()) - throw new ValueError($"Layer #{k} (named {layer.Name}" + - "in the current model) was found to " + - $"correspond to layer {name} in the save file." + - $"However the new layer {layer.Name} expects " + - $"{symbolic_weights.Count()} weights, but the saved weights have " + - $"{weight_values.Count()} elements."); - weight_value_tuples.AddRange(zip(symbolic_weights, weight_values)); - } - - keras.backend.batch_set_value(weight_value_tuples); - } - - public static void toarrayf4(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static void load_weights_from_hdf5_group_by_name(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static void save_weights_to_hdf5_group(long f, List layers) - { - List layerName=new List(); - foreach (var layer in layers) - { - layerName.Add(layer.Name); - } - save_attributes_to_hdf5_group(f, "layer_names", layerName.ToArray()); - Hdf5.WriteAttribute(f, "backend", "tensorflow"); - Hdf5.WriteAttribute(f, "keras_version", "2.5.0"); - - foreach (var layer in layers) - { - var weights = _legacy_weights(layer); - if (weights.Count == 0) - continue; - - var weight_names = new List(); - // weight_values= keras.backend.batch_get_value(weights); - foreach (var weight in weights) - weight_names.Add(weight.Name); - - var g = Hdf5.CreateOrOpenGroup(f, Hdf5Utils.NormalizedName(layer.Name)); - save_attributes_to_hdf5_group(g, "weight_names", weight_names.ToArray()); - foreach (var (name, val) in zip(weight_names, weights)) - { - var tensor = val.AsTensor(); - if (name.IndexOf("/") > 1) - { - var crDataGroup = g; - string[] name_split = name.Split('/'); - for(int i = 0; i < name_split.Length; i++) - { - if (i == name_split.Length - 1) break; +using System; +using System.Collections.Generic; +using System.Text; +using HDF.PInvoke; +using Tensorflow.NumPy; +using HDF5CSharp; +using static Tensorflow.Binding; +using static Tensorflow.KerasApi; +using System.Linq; +using System.Text.RegularExpressions; + +namespace Tensorflow.Keras.Saving +{ + public class hdf5_format + { + private static int HDF5_OBJECT_HEADER_LIMIT = 64512; + public static void load_model_from_hdf5(string filepath = "", Dictionary custom_objects = null, bool compile = false) + { + long root = Hdf5.OpenFile(filepath,true); + load_model_from_hdf5(root, custom_objects, compile); + } + public static void load_model_from_hdf5(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + //long fileId = filepath; + //try + //{ + // groupId = H5G.open(fileId, "/"); + // (bool success, string[] attrId) = Hdf5.ReadStringAttributes(groupId, "model_config", ""); + // H5G.close(groupId); + // if (success == true) { + // Console.WriteLine(attrId[0]); + // } + //} + //catch (Exception ex) + //{ + // if (filepath != -1) { + // Hdf5.CloseFile(filepath); + // } + // if (groupId != -1) { + // H5G.close(groupId); + // } + // throw new Exception(ex.ToString()); + //} + + } + public static void save_model_to_hdf5(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + /// + /// Preprocess layer weights between different Keras formats. + /// + /// + /// + /// + /// + public static List preprocess_weights_for_loading(ILayer layer, List weights, string original_keras_version = null, string original_backend = null) + { + // convert CuDNN layers + return _convert_rnn_weights(layer, weights); + } + + /// + /// Converts weights for RNN layers between native and CuDNN format. + /// + /// + /// + static List _convert_rnn_weights(ILayer layer, List weights) + { + var target_class = layer.GetType().Name; + return weights; + } + + public static void save_optimizer_weights_to_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static void load_optimizer_weights_from_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static void load_weights_from_hdf5_group(long f, List layers) + { + string original_keras_version = "2.5.0"; + string original_backend = null; + var (success, attr) = Hdf5.ReadStringAttributes(f, "keras_version", "", true); + if (success) + original_keras_version = attr.First(); + // keras version should be 2.5.0+ + var ver_major = int.Parse(original_keras_version.Split('.')[0]); + var ver_minor = int.Parse(original_keras_version.Split('.')[1]); + if (ver_major < 2 || (ver_major == 2 && ver_minor < 5)) + throw new ValueError("keras version should be 2.5.0 or later."); + + (success, attr) = Hdf5.ReadStringAttributes(f, "backend", "", true); + if (success) + original_backend = attr.First(); + + var filtered_layers = new List(); + foreach (var layer in layers) + { + var weights = _legacy_weights(layer); + if (weights.Count > 0) + filtered_layers.append(layer); + } + + string[] layer_names = load_attributes_from_hdf5_group(f, "layer_names"); + var filtered_layer_names = new List(); + foreach(var name in layer_names) + { + if (!filtered_layers.Select(x => x.Name).Contains(name)) + continue; + long g = H5G.open(f, name); + var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); + if (weight_names.Count() > 0) + filtered_layer_names.Add(name); + H5G.close(g); + } + + layer_names = filtered_layer_names.ToArray(); + if (layer_names.Length != filtered_layers.Count()) + throw new ValueError("You are trying to load a weight file " + + $"containing {layer_names}" + + $" layers into a model with {filtered_layers.Count} layers."); + + var weight_value_tuples = new List<(IVariableV1, NDArray)>(); + foreach (var (k, name) in enumerate(layer_names)) + { + var weight_values = new List(); + long g = H5G.open(f, name); + var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); + foreach (var i_ in weight_names) + { + (success, Array result) = Hdf5.ReadDataset(g, i_); + if (success) + weight_values.Add(np.array(result)); + } + H5G.close(g); + var layer = filtered_layers[k]; + var symbolic_weights = _legacy_weights(layer); + preprocess_weights_for_loading(layer, weight_values, original_keras_version, original_backend); + if (weight_values.Count() != symbolic_weights.Count()) + throw new ValueError($"Layer #{k} (named {layer.Name}" + + "in the current model) was found to " + + $"correspond to layer {name} in the save file." + + $"However the new layer {layer.Name} expects " + + $"{symbolic_weights.Count()} weights, but the saved weights have " + + $"{weight_values.Count()} elements."); + weight_value_tuples.AddRange(zip(symbolic_weights, weight_values)); + } + + keras.backend.batch_set_value(weight_value_tuples); + } + + public static void toarrayf4(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static void load_weights_from_hdf5_group_by_name(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static void save_weights_to_hdf5_group(long f, List layers) + { + List layerName=new List(); + foreach (var layer in layers) + { + layerName.Add(layer.Name); + } + save_attributes_to_hdf5_group(f, "layer_names", layerName.ToArray()); + Hdf5.WriteAttribute(f, "backend", "tensorflow"); + Hdf5.WriteAttribute(f, "keras_version", "2.5.0"); + + foreach (var layer in layers) + { + var weights = _legacy_weights(layer); + if (weights.Count == 0) + continue; + + var weight_names = new List(); + // weight_values= keras.backend.batch_get_value(weights); + foreach (var weight in weights) + weight_names.Add(weight.Name); + + var g = Hdf5.CreateOrOpenGroup(f, Hdf5Utils.NormalizedName(layer.Name)); + save_attributes_to_hdf5_group(g, "weight_names", weight_names.ToArray()); + foreach (var (name, val) in zip(weight_names, weights)) + { + var tensor = val.AsTensor(); + if (name.IndexOf("/") > 1) + { + var crDataGroup = g; + string[] name_split = name.Split('/'); + for(int i = 0; i < name_split.Length - 1; i++) + { crDataGroup = Hdf5.CreateOrOpenGroup(crDataGroup, Hdf5Utils.NormalizedName(name_split[i])); - } - WriteDataset(crDataGroup, name_split[name_split.Length - 1], tensor); - Hdf5.CloseGroup(crDataGroup); - } - else - { - WriteDataset(g, name, tensor); - } - } - Hdf5.CloseGroup(g); - } - } - - private static void save_attributes_to_hdf5_group(long f, string name, Array data) - { - int num_chunks = 1; - - var chunked_data = Split(data, num_chunks); - int getSize = 0; - - string getType = data.Length > 0 ? data.GetValue(0).GetType().Name.ToLower() : "string"; - - switch (getType) - { - case "single": - getSize = sizeof(float); - break; - case "double": - getSize = sizeof(double); - break; - case "string": - getSize = -1; - break; - case "int32": - getSize = sizeof(int); - break; - case "int64": - getSize = sizeof(long); - break; - default: - getSize = -1; - break; - } - int getCount = chunked_data.Count; - - if (getSize != -1) - { - num_chunks = (int)Math.Ceiling((double)(getCount * getSize) / HDF5_OBJECT_HEADER_LIMIT); - if (num_chunks > 1) chunked_data = Split(data, num_chunks); - } - - if (num_chunks > 1) - { - foreach (var (chunk_id, chunk_data) in enumerate(chunked_data)) - WriteAttrs(f, getType, $"{name}{chunk_id}", chunk_data.ToArray()); - } - else - { - WriteAttrs(f, getType, name, data); - } - } - - private static void WriteDataset(long f, string name, Tensor data) - { - switch (data.dtype) - { - case TF_DataType.TF_FLOAT: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - case TF_DataType.TF_DOUBLE: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - case TF_DataType.TF_INT32: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - case TF_DataType.TF_INT64: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - default: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - } - } - - private static void WriteAttrs(long f,string typename, string name, Array data) - { - switch (typename) - { - case "single": - Hdf5.WriteAttributes(f, name, data); - break; - case "double": - Hdf5.WriteAttributes(f, name, data); - break; - case "string": - Hdf5.WriteAttributes(f, name, data); - break; - case "int32": - Hdf5.WriteAttributes(f, name, data); - break; - case "int64": - Hdf5.WriteAttributes(f, name, data); - break; - default: - Hdf5.WriteAttributes(f, name,data); - break; - } - } - - private static List> Split(Array list, int chunkSize) - { - var splitList = new List>(); - var chunkCount = (int)Math.Ceiling((double)list.Length / (double)chunkSize); - - for (int c = 0; c < chunkCount; c++) - { - var skip = c * chunkSize; - var take = skip + chunkSize; - var chunk = new List(chunkSize); - - for (int e = skip; e < take && e < list.Length; e++) - { - chunk.Add(list.GetValue(e)); - } - splitList.Add(chunk); - } - - return splitList; - } - - public static string[] load_attributes_from_hdf5_group(long group, string name) - { - var (success, attr) = Hdf5.ReadStringAttributes(group, name, "", true); - if (success) - return attr.ToArray(); - - return null; - } - - public static void load_attributes_from_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static List _legacy_weights(ILayer layer) - { - var weights = layer.TrainableWeights.Select(x => x).ToList(); - weights.AddRange(layer.NonTrainableWeights); - return weights; - } - } -} - + } + WriteDataset(crDataGroup, name_split[name_split.Length - 1], tensor); + Hdf5.CloseGroup(crDataGroup); + } + else + { + WriteDataset(g, name, tensor); + } + } + Hdf5.CloseGroup(g); + } + } + + private static void save_attributes_to_hdf5_group(long f, string name, Array data) + { + int num_chunks = 1; + + var chunked_data = Split(data, num_chunks); + int getSize = 0; + + string getType = data.Length > 0 ? data.GetValue(0).GetType().Name.ToLower() : "string"; + + switch (getType) + { + case "single": + getSize = sizeof(float); + break; + case "double": + getSize = sizeof(double); + break; + case "string": + getSize = -1; + break; + case "int32": + getSize = sizeof(int); + break; + case "int64": + getSize = sizeof(long); + break; + default: + getSize = -1; + break; + } + int getCount = chunked_data.Count; + + if (getSize != -1) + { + num_chunks = (int)Math.Ceiling((double)(getCount * getSize) / HDF5_OBJECT_HEADER_LIMIT); + if (num_chunks > 1) chunked_data = Split(data, num_chunks); + } + + if (num_chunks > 1) + { + foreach (var (chunk_id, chunk_data) in enumerate(chunked_data)) + WriteAttrs(f, getType, $"{name}{chunk_id}", chunk_data.ToArray()); + } + else + { + WriteAttrs(f, getType, name, data); + } + } + + private static void WriteDataset(long f, string name, Tensor data) + { + switch (data.dtype) + { + case TF_DataType.TF_FLOAT: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + case TF_DataType.TF_DOUBLE: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + case TF_DataType.TF_INT32: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + case TF_DataType.TF_INT64: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + default: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + } + } + + private static void WriteAttrs(long f,string typename, string name, Array data) + { + switch (typename) + { + case "single": + Hdf5.WriteAttributes(f, name, data); + break; + case "double": + Hdf5.WriteAttributes(f, name, data); + break; + case "string": + Hdf5.WriteAttributes(f, name, data); + break; + case "int32": + Hdf5.WriteAttributes(f, name, data); + break; + case "int64": + Hdf5.WriteAttributes(f, name, data); + break; + default: + Hdf5.WriteAttributes(f, name,data); + break; + } + } + + private static List> Split(Array list, int chunkSize) + { + var splitList = new List>(); + var chunkCount = (int)Math.Ceiling((double)list.Length / (double)chunkSize); + + for (int c = 0; c < chunkCount; c++) + { + var skip = c * chunkSize; + var take = skip + chunkSize; + var chunk = new List(chunkSize); + + for (int e = skip; e < take && e < list.Length; e++) + { + chunk.Add(list.GetValue(e)); + } + splitList.Add(chunk); + } + + return splitList; + } + + public static string[] load_attributes_from_hdf5_group(long group, string name) + { + var (success, attr) = Hdf5.ReadStringAttributes(group, name, "", true); + if (success) + return attr.ToArray(); + + return null; + } + + public static void load_attributes_from_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static List _legacy_weights(ILayer layer) + { + var weights = layer.TrainableWeights.Select(x => x).ToList(); + weights.AddRange(layer.NonTrainableWeights); + return weights; + } + } +} + From 8b17b14f30e288705552a5ca417264b35b8447bc Mon Sep 17 00:00:00 2001 From: Beacontownfc <89081023+Beacontownfc@users.noreply.github.com> Date: Sat, 22 Jul 2023 14:34:08 +0800 Subject: [PATCH 17/98] Update hdf5_format.cs --- src/TensorFlowNET.Keras/Saving/hdf5_format.cs | 708 +++++++++--------- 1 file changed, 354 insertions(+), 354 deletions(-) diff --git a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs index c80f653f8..bab0efecf 100644 --- a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs +++ b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs @@ -1,354 +1,354 @@ -using System; -using System.Collections.Generic; -using System.Text; -using HDF.PInvoke; -using Tensorflow.NumPy; -using HDF5CSharp; -using static Tensorflow.Binding; -using static Tensorflow.KerasApi; -using System.Linq; -using System.Text.RegularExpressions; - -namespace Tensorflow.Keras.Saving -{ - public class hdf5_format - { - private static int HDF5_OBJECT_HEADER_LIMIT = 64512; - public static void load_model_from_hdf5(string filepath = "", Dictionary custom_objects = null, bool compile = false) - { - long root = Hdf5.OpenFile(filepath,true); - load_model_from_hdf5(root, custom_objects, compile); - } - public static void load_model_from_hdf5(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - //long fileId = filepath; - //try - //{ - // groupId = H5G.open(fileId, "/"); - // (bool success, string[] attrId) = Hdf5.ReadStringAttributes(groupId, "model_config", ""); - // H5G.close(groupId); - // if (success == true) { - // Console.WriteLine(attrId[0]); - // } - //} - //catch (Exception ex) - //{ - // if (filepath != -1) { - // Hdf5.CloseFile(filepath); - // } - // if (groupId != -1) { - // H5G.close(groupId); - // } - // throw new Exception(ex.ToString()); - //} - - } - public static void save_model_to_hdf5(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - /// - /// Preprocess layer weights between different Keras formats. - /// - /// - /// - /// - /// - public static List preprocess_weights_for_loading(ILayer layer, List weights, string original_keras_version = null, string original_backend = null) - { - // convert CuDNN layers - return _convert_rnn_weights(layer, weights); - } - - /// - /// Converts weights for RNN layers between native and CuDNN format. - /// - /// - /// - static List _convert_rnn_weights(ILayer layer, List weights) - { - var target_class = layer.GetType().Name; - return weights; - } - - public static void save_optimizer_weights_to_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static void load_optimizer_weights_from_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static void load_weights_from_hdf5_group(long f, List layers) - { - string original_keras_version = "2.5.0"; - string original_backend = null; - var (success, attr) = Hdf5.ReadStringAttributes(f, "keras_version", "", true); - if (success) - original_keras_version = attr.First(); - // keras version should be 2.5.0+ - var ver_major = int.Parse(original_keras_version.Split('.')[0]); - var ver_minor = int.Parse(original_keras_version.Split('.')[1]); - if (ver_major < 2 || (ver_major == 2 && ver_minor < 5)) - throw new ValueError("keras version should be 2.5.0 or later."); - - (success, attr) = Hdf5.ReadStringAttributes(f, "backend", "", true); - if (success) - original_backend = attr.First(); - - var filtered_layers = new List(); - foreach (var layer in layers) - { - var weights = _legacy_weights(layer); - if (weights.Count > 0) - filtered_layers.append(layer); - } - - string[] layer_names = load_attributes_from_hdf5_group(f, "layer_names"); - var filtered_layer_names = new List(); - foreach(var name in layer_names) - { - if (!filtered_layers.Select(x => x.Name).Contains(name)) - continue; - long g = H5G.open(f, name); - var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); - if (weight_names.Count() > 0) - filtered_layer_names.Add(name); - H5G.close(g); - } - - layer_names = filtered_layer_names.ToArray(); - if (layer_names.Length != filtered_layers.Count()) - throw new ValueError("You are trying to load a weight file " + - $"containing {layer_names}" + - $" layers into a model with {filtered_layers.Count} layers."); - - var weight_value_tuples = new List<(IVariableV1, NDArray)>(); - foreach (var (k, name) in enumerate(layer_names)) - { - var weight_values = new List(); - long g = H5G.open(f, name); - var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); - foreach (var i_ in weight_names) - { - (success, Array result) = Hdf5.ReadDataset(g, i_); - if (success) - weight_values.Add(np.array(result)); - } - H5G.close(g); - var layer = filtered_layers[k]; - var symbolic_weights = _legacy_weights(layer); - preprocess_weights_for_loading(layer, weight_values, original_keras_version, original_backend); - if (weight_values.Count() != symbolic_weights.Count()) - throw new ValueError($"Layer #{k} (named {layer.Name}" + - "in the current model) was found to " + - $"correspond to layer {name} in the save file." + - $"However the new layer {layer.Name} expects " + - $"{symbolic_weights.Count()} weights, but the saved weights have " + - $"{weight_values.Count()} elements."); - weight_value_tuples.AddRange(zip(symbolic_weights, weight_values)); - } - - keras.backend.batch_set_value(weight_value_tuples); - } - - public static void toarrayf4(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static void load_weights_from_hdf5_group_by_name(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static void save_weights_to_hdf5_group(long f, List layers) - { - List layerName=new List(); - foreach (var layer in layers) - { - layerName.Add(layer.Name); - } - save_attributes_to_hdf5_group(f, "layer_names", layerName.ToArray()); - Hdf5.WriteAttribute(f, "backend", "tensorflow"); - Hdf5.WriteAttribute(f, "keras_version", "2.5.0"); - - foreach (var layer in layers) - { - var weights = _legacy_weights(layer); - if (weights.Count == 0) - continue; - - var weight_names = new List(); - // weight_values= keras.backend.batch_get_value(weights); - foreach (var weight in weights) - weight_names.Add(weight.Name); - - var g = Hdf5.CreateOrOpenGroup(f, Hdf5Utils.NormalizedName(layer.Name)); - save_attributes_to_hdf5_group(g, "weight_names", weight_names.ToArray()); - foreach (var (name, val) in zip(weight_names, weights)) - { - var tensor = val.AsTensor(); - if (name.IndexOf("/") > 1) - { - var crDataGroup = g; - string[] name_split = name.Split('/'); - for(int i = 0; i < name_split.Length - 1; i++) - { - crDataGroup = Hdf5.CreateOrOpenGroup(crDataGroup, Hdf5Utils.NormalizedName(name_split[i])); - } - WriteDataset(crDataGroup, name_split[name_split.Length - 1], tensor); - Hdf5.CloseGroup(crDataGroup); - } - else - { - WriteDataset(g, name, tensor); - } - } - Hdf5.CloseGroup(g); - } - } - - private static void save_attributes_to_hdf5_group(long f, string name, Array data) - { - int num_chunks = 1; - - var chunked_data = Split(data, num_chunks); - int getSize = 0; - - string getType = data.Length > 0 ? data.GetValue(0).GetType().Name.ToLower() : "string"; - - switch (getType) - { - case "single": - getSize = sizeof(float); - break; - case "double": - getSize = sizeof(double); - break; - case "string": - getSize = -1; - break; - case "int32": - getSize = sizeof(int); - break; - case "int64": - getSize = sizeof(long); - break; - default: - getSize = -1; - break; - } - int getCount = chunked_data.Count; - - if (getSize != -1) - { - num_chunks = (int)Math.Ceiling((double)(getCount * getSize) / HDF5_OBJECT_HEADER_LIMIT); - if (num_chunks > 1) chunked_data = Split(data, num_chunks); - } - - if (num_chunks > 1) - { - foreach (var (chunk_id, chunk_data) in enumerate(chunked_data)) - WriteAttrs(f, getType, $"{name}{chunk_id}", chunk_data.ToArray()); - } - else - { - WriteAttrs(f, getType, name, data); - } - } - - private static void WriteDataset(long f, string name, Tensor data) - { - switch (data.dtype) - { - case TF_DataType.TF_FLOAT: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - case TF_DataType.TF_DOUBLE: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - case TF_DataType.TF_INT32: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - case TF_DataType.TF_INT64: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - default: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - } - } - - private static void WriteAttrs(long f,string typename, string name, Array data) - { - switch (typename) - { - case "single": - Hdf5.WriteAttributes(f, name, data); - break; - case "double": - Hdf5.WriteAttributes(f, name, data); - break; - case "string": - Hdf5.WriteAttributes(f, name, data); - break; - case "int32": - Hdf5.WriteAttributes(f, name, data); - break; - case "int64": - Hdf5.WriteAttributes(f, name, data); - break; - default: - Hdf5.WriteAttributes(f, name,data); - break; - } - } - - private static List> Split(Array list, int chunkSize) - { - var splitList = new List>(); - var chunkCount = (int)Math.Ceiling((double)list.Length / (double)chunkSize); - - for (int c = 0; c < chunkCount; c++) - { - var skip = c * chunkSize; - var take = skip + chunkSize; - var chunk = new List(chunkSize); - - for (int e = skip; e < take && e < list.Length; e++) - { - chunk.Add(list.GetValue(e)); - } - splitList.Add(chunk); - } - - return splitList; - } - - public static string[] load_attributes_from_hdf5_group(long group, string name) - { - var (success, attr) = Hdf5.ReadStringAttributes(group, name, "", true); - if (success) - return attr.ToArray(); - - return null; - } - - public static void load_attributes_from_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static List _legacy_weights(ILayer layer) - { - var weights = layer.TrainableWeights.Select(x => x).ToList(); - weights.AddRange(layer.NonTrainableWeights); - return weights; - } - } -} - +using System; +using System.Collections.Generic; +using System.Text; +using HDF.PInvoke; +using Tensorflow.NumPy; +using HDF5CSharp; +using static Tensorflow.Binding; +using static Tensorflow.KerasApi; +using System.Linq; +using System.Text.RegularExpressions; + +namespace Tensorflow.Keras.Saving +{ + public class hdf5_format + { + private static int HDF5_OBJECT_HEADER_LIMIT = 64512; + public static void load_model_from_hdf5(string filepath = "", Dictionary custom_objects = null, bool compile = false) + { + long root = Hdf5.OpenFile(filepath,true); + load_model_from_hdf5(root, custom_objects, compile); + } + public static void load_model_from_hdf5(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + //long fileId = filepath; + //try + //{ + // groupId = H5G.open(fileId, "/"); + // (bool success, string[] attrId) = Hdf5.ReadStringAttributes(groupId, "model_config", ""); + // H5G.close(groupId); + // if (success == true) { + // Console.WriteLine(attrId[0]); + // } + //} + //catch (Exception ex) + //{ + // if (filepath != -1) { + // Hdf5.CloseFile(filepath); + // } + // if (groupId != -1) { + // H5G.close(groupId); + // } + // throw new Exception(ex.ToString()); + //} + + } + public static void save_model_to_hdf5(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + /// + /// Preprocess layer weights between different Keras formats. + /// + /// + /// + /// + /// + public static List preprocess_weights_for_loading(ILayer layer, List weights, string original_keras_version = null, string original_backend = null) + { + // convert CuDNN layers + return _convert_rnn_weights(layer, weights); + } + + /// + /// Converts weights for RNN layers between native and CuDNN format. + /// + /// + /// + static List _convert_rnn_weights(ILayer layer, List weights) + { + var target_class = layer.GetType().Name; + return weights; + } + + public static void save_optimizer_weights_to_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static void load_optimizer_weights_from_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static void load_weights_from_hdf5_group(long f, List layers) + { + string original_keras_version = "2.5.0"; + string original_backend = null; + var (success, attr) = Hdf5.ReadStringAttributes(f, "keras_version", "", true); + if (success) + original_keras_version = attr.First(); + // keras version should be 2.5.0+ + var ver_major = int.Parse(original_keras_version.Split('.')[0]); + var ver_minor = int.Parse(original_keras_version.Split('.')[1]); + if (ver_major < 2 || (ver_major == 2 && ver_minor < 5)) + throw new ValueError("keras version should be 2.5.0 or later."); + + (success, attr) = Hdf5.ReadStringAttributes(f, "backend", "", true); + if (success) + original_backend = attr.First(); + + var filtered_layers = new List(); + foreach (var layer in layers) + { + var weights = _legacy_weights(layer); + if (weights.Count > 0) + filtered_layers.append(layer); + } + + string[] layer_names = load_attributes_from_hdf5_group(f, "layer_names"); + var filtered_layer_names = new List(); + foreach(var name in layer_names) + { + if (!filtered_layers.Select(x => x.Name).Contains(name)) + continue; + long g = H5G.open(f, name); + var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); + if (weight_names.Count() > 0) + filtered_layer_names.Add(name); + H5G.close(g); + } + + layer_names = filtered_layer_names.ToArray(); + if (layer_names.Length != filtered_layers.Count()) + throw new ValueError("You are trying to load a weight file " + + $"containing {layer_names}" + + $" layers into a model with {filtered_layers.Count} layers."); + + var weight_value_tuples = new List<(IVariableV1, NDArray)>(); + foreach (var (k, name) in enumerate(layer_names)) + { + var weight_values = new List(); + long g = H5G.open(f, name); + var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); + foreach (var i_ in weight_names) + { + (success, Array result) = Hdf5.ReadDataset(g, i_); + if (success) + weight_values.Add(np.array(result)); + } + H5G.close(g); + var layer = filtered_layers[k]; + var symbolic_weights = _legacy_weights(layer); + preprocess_weights_for_loading(layer, weight_values, original_keras_version, original_backend); + if (weight_values.Count() != symbolic_weights.Count()) + throw new ValueError($"Layer #{k} (named {layer.Name}" + + "in the current model) was found to " + + $"correspond to layer {name} in the save file." + + $"However the new layer {layer.Name} expects " + + $"{symbolic_weights.Count()} weights, but the saved weights have " + + $"{weight_values.Count()} elements."); + weight_value_tuples.AddRange(zip(symbolic_weights, weight_values)); + } + + keras.backend.batch_set_value(weight_value_tuples); + } + + public static void toarrayf4(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static void load_weights_from_hdf5_group_by_name(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static void save_weights_to_hdf5_group(long f, List layers) + { + List layerName=new List(); + foreach (var layer in layers) + { + layerName.Add(layer.Name); + } + save_attributes_to_hdf5_group(f, "layer_names", layerName.ToArray()); + Hdf5.WriteAttribute(f, "backend", "tensorflow"); + Hdf5.WriteAttribute(f, "keras_version", "2.5.0"); + + foreach (var layer in layers) + { + var weights = _legacy_weights(layer); + if (weights.Count == 0) + continue; + + var weight_names = new List(); + // weight_values= keras.backend.batch_get_value(weights); + foreach (var weight in weights) + weight_names.Add(weight.Name); + + var g = Hdf5.CreateOrOpenGroup(f, Hdf5Utils.NormalizedName(layer.Name)); + save_attributes_to_hdf5_group(g, "weight_names", weight_names.ToArray()); + foreach (var (name, val) in zip(weight_names, weights)) + { + var tensor = val.AsTensor(); + if (name.IndexOf("/") > 1) + { + var crDataGroup = g; + string[] name_split = name.Split('/'); + for(int i = 0; i < name_split.Length - 1; i++) + { + crDataGroup = Hdf5.CreateOrOpenGroup(crDataGroup, Hdf5Utils.NormalizedName(name_split[i])); + } + WriteDataset(crDataGroup, name_split[name_split.Length - 1], tensor); + Hdf5.CloseGroup(crDataGroup); + } + else + { + WriteDataset(g, name, tensor); + } + } + Hdf5.CloseGroup(g); + } + } + + private static void save_attributes_to_hdf5_group(long f, string name, Array data) + { + int num_chunks = 1; + + var chunked_data = Split(data, num_chunks); + int getSize = 0; + + string getType = data.Length > 0 ? data.GetValue(0).GetType().Name.ToLower() : "string"; + + switch (getType) + { + case "single": + getSize = sizeof(float); + break; + case "double": + getSize = sizeof(double); + break; + case "string": + getSize = -1; + break; + case "int32": + getSize = sizeof(int); + break; + case "int64": + getSize = sizeof(long); + break; + default: + getSize = -1; + break; + } + int getCount = chunked_data.Count; + + if (getSize != -1) + { + num_chunks = (int)Math.Ceiling((double)(getCount * getSize) / HDF5_OBJECT_HEADER_LIMIT); + if (num_chunks > 1) chunked_data = Split(data, num_chunks); + } + + if (num_chunks > 1) + { + foreach (var (chunk_id, chunk_data) in enumerate(chunked_data)) + WriteAttrs(f, getType, $"{name}{chunk_id}", chunk_data.ToArray()); + } + else + { + WriteAttrs(f, getType, name, data); + } + } + + private static void WriteDataset(long f, string name, Tensor data) + { + switch (data.dtype) + { + case TF_DataType.TF_FLOAT: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + case TF_DataType.TF_DOUBLE: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + case TF_DataType.TF_INT32: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + case TF_DataType.TF_INT64: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + default: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + } + } + + private static void WriteAttrs(long f,string typename, string name, Array data) + { + switch (typename) + { + case "single": + Hdf5.WriteAttributes(f, name, data); + break; + case "double": + Hdf5.WriteAttributes(f, name, data); + break; + case "string": + Hdf5.WriteAttributes(f, name, data); + break; + case "int32": + Hdf5.WriteAttributes(f, name, data); + break; + case "int64": + Hdf5.WriteAttributes(f, name, data); + break; + default: + Hdf5.WriteAttributes(f, name,data); + break; + } + } + + private static List> Split(Array list, int chunkSize) + { + var splitList = new List>(); + var chunkCount = (int)Math.Ceiling((double)list.Length / (double)chunkSize); + + for (int c = 0; c < chunkCount; c++) + { + var skip = c * chunkSize; + var take = skip + chunkSize; + var chunk = new List(chunkSize); + + for (int e = skip; e < take && e < list.Length; e++) + { + chunk.Add(list.GetValue(e)); + } + splitList.Add(chunk); + } + + return splitList; + } + + public static string[] load_attributes_from_hdf5_group(long group, string name) + { + var (success, attr) = Hdf5.ReadStringAttributes(group, name, "", true); + if (success) + return attr.ToArray(); + + return null; + } + + public static void load_attributes_from_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static List _legacy_weights(ILayer layer) + { + var weights = layer.TrainableWeights.Select(x => x).ToList(); + weights.AddRange(layer.NonTrainableWeights); + return weights; + } + } +} + From 482899eab734f1b6f3a39ef52a4f9ae28e332ed5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Sat, 22 Jul 2023 15:03:50 +0800 Subject: [PATCH 18/98] fix: revise np.amin, np.amax and add np.argmin --- .../NumPy/NumPy.Sorting.Searching.Counting.cs | 4 ++++ src/TensorFlowNET.Core/NumPy/NumPy.Statistics.cs | 4 ++-- src/TensorFlowNET.Core/Operations/math_ops.cs | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/TensorFlowNET.Core/NumPy/NumPy.Sorting.Searching.Counting.cs b/src/TensorFlowNET.Core/NumPy/NumPy.Sorting.Searching.Counting.cs index 5182d5726..4cad36e0b 100644 --- a/src/TensorFlowNET.Core/NumPy/NumPy.Sorting.Searching.Counting.cs +++ b/src/TensorFlowNET.Core/NumPy/NumPy.Sorting.Searching.Counting.cs @@ -13,6 +13,10 @@ public partial class np public static NDArray argmax(NDArray a, Axis? axis = null) => new NDArray(math_ops.argmax(a, axis ?? 0)); + [AutoNumPy] + public static NDArray argmin(NDArray a, Axis? axis = null) + => new NDArray(math_ops.argmin(a, axis ?? 0)); + [AutoNumPy] public static NDArray argsort(NDArray a, Axis? axis = null) => new NDArray(sort_ops.argsort(a, axis: axis ?? -1)); diff --git a/src/TensorFlowNET.Core/NumPy/NumPy.Statistics.cs b/src/TensorFlowNET.Core/NumPy/NumPy.Statistics.cs index 5d86b1b39..bce16ec9f 100644 --- a/src/TensorFlowNET.Core/NumPy/NumPy.Statistics.cs +++ b/src/TensorFlowNET.Core/NumPy/NumPy.Statistics.cs @@ -10,10 +10,10 @@ namespace Tensorflow.NumPy public partial class np { [AutoNumPy] - public static NDArray amin(NDArray x, int axis = 0) => new NDArray(tf.arg_min(x, axis)); + public static NDArray amin(NDArray x, int axis = 0) => new NDArray(tf.min(x, axis)); [AutoNumPy] - public static NDArray amax(NDArray x, int axis = 0) => new NDArray(tf.math.argmax(x, axis)); + public static NDArray amax(NDArray x, int axis = 0) => new NDArray(tf.max(x, axis)); [AutoNumPy] public static NDArray average(NDArray a, int axis = -1, NDArray? weights = null, bool returned = false) diff --git a/src/TensorFlowNET.Core/Operations/math_ops.cs b/src/TensorFlowNET.Core/Operations/math_ops.cs index 092137bf2..e77df702f 100644 --- a/src/TensorFlowNET.Core/Operations/math_ops.cs +++ b/src/TensorFlowNET.Core/Operations/math_ops.cs @@ -77,6 +77,9 @@ public static Tensor add_n(Tensor[] inputs, string name = null) public static Tensor argmax(Tensor input, Axis dimension, TF_DataType output_type = TF_DataType.TF_INT64, string name = null) => gen_math_ops.arg_max(input, dimension, output_type: output_type, name: name); + public static Tensor argmin(Tensor input, Axis dimension, TF_DataType output_type = TF_DataType.TF_INT64, string name = null) + => gen_math_ops.arg_min(input, dimension, output_type: output_type, name: name); + public static Tensor round(Tensor x, string name = null) { x = ops.convert_to_tensor(x, name: "x"); From b0ce73caff995d8b5b8080dd41812af4c48908e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Mon, 24 Jul 2023 23:38:58 +0800 Subject: [PATCH 19/98] feat: add adjust_contrast, adjust_hue, combined_non_max_suppression, crop_and_resize image oprs --- src/TensorFlowNET.Core/APIs/tf.image.cs | 131 +++++++- .../Operations/gen_image_ops.cs | 298 +++++++++++++++++- .../TensorFlowNET.Graph.UnitTest/ImageTest.cs | 65 +++- 3 files changed, 479 insertions(+), 15 deletions(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.image.cs b/src/TensorFlowNET.Core/APIs/tf.image.cs index 9230b50dc..ac9cbc60d 100644 --- a/src/TensorFlowNET.Core/APIs/tf.image.cs +++ b/src/TensorFlowNET.Core/APIs/tf.image.cs @@ -14,6 +14,10 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using OneOf.Types; +using System; +using System.Buffers.Text; +using Tensorflow.Contexts; using static Tensorflow.Binding; namespace Tensorflow @@ -162,17 +166,108 @@ public Tensor ssim_multiscale(Tensor img1, Tensor img2, float max_val, float[] p public Tensor sobel_edges(Tensor image) => image_ops_impl.sobel_edges(image); - public Tensor decode_jpeg(Tensor contents, - int channels = 0, - int ratio = 1, - bool fancy_upscaling = true, - bool try_recover_truncated = false, - int acceptable_fraction = 1, - string dct_method = "", - string name = null) - => gen_image_ops.decode_jpeg(contents, channels: channels, ratio: ratio, - fancy_upscaling: fancy_upscaling, try_recover_truncated: try_recover_truncated, - acceptable_fraction: acceptable_fraction, dct_method: dct_method); + /// + /// Adjust contrast of RGB or grayscale images. + /// + /// Images to adjust. At least 3-D. + /// + /// A float multiplier for adjusting contrast. + /// The contrast-adjusted image or images. + public Tensor adjust_contrast(Tensor images, float contrast_factor, string name = null) + => gen_image_ops.adjust_contrastv2(images, contrast_factor, name); + + /// + /// Adjust hue of RGB images. + /// + /// RGB image or images. The size of the last dimension must be 3. + /// float. How much to add to the hue channel. + /// A name for this operation (optional). + /// Adjusted image(s), same shape and DType as `image`. + /// if `delta` is not in the interval of `[-1, 1]`. + public Tensor adjust_hue(Tensor images, float delta, string name = null) + { + if (tf.Context.executing_eagerly()) + { + if (delta < -1f || delta > 1f) + throw new ValueError("delta must be in the interval [-1, 1]"); + } + return gen_image_ops.adjust_hue(images, delta, name: name); + } + + /// + /// Adjust saturation of RGB images. + /// + /// RGB image or images. The size of the last dimension must be 3. + /// float. Factor to multiply the saturation by. + /// A name for this operation (optional). + /// Adjusted image(s), same shape and DType as `image`. + public Tensor adjust_saturation(Tensor image, float saturation_factor, string name = null) + => gen_image_ops.adjust_saturation(image, saturation_factor, name); + + /// + /// Greedily selects a subset of bounding boxes in descending order of score. + /// + /// + /// A 4-D float `Tensor` of shape `[batch_size, num_boxes, q, 4]`. If `q` + /// is 1 then same boxes are used for all classes otherwise, if `q` is equal + /// to number of classes, class-specific boxes are used. + /// + /// + /// A 3-D float `Tensor` of shape `[batch_size, num_boxes, num_classes]` + /// representing a single score corresponding to each box(each row of boxes). + /// + /// + /// A scalar integer `Tensor` representing the + /// maximum number of boxes to be selected by non-max suppression per class + /// + /// + /// A int32 scalar representing maximum number of boxes retained + /// over all classes.Note that setting this value to a large number may + /// result in OOM error depending on the system workload. + /// + /// + /// A float representing the threshold for deciding whether boxes + /// overlap too much with respect to IOU. + /// + /// + /// A float representing the threshold for deciding when to + /// remove boxes based on score. + /// + /// + /// If false, the output nmsed boxes, scores and classes are + /// padded/clipped to `max_total_size`. If true, the output nmsed boxes, scores and classes are padded to be of length `max_size_per_class`*`num_classes`, + /// unless it exceeds `max_total_size` in which case it is clipped to `max_total_size`. Defaults to false. + /// + /// + /// If true, the coordinates of output nmsed boxes will be clipped + /// to[0, 1]. If false, output the box coordinates as it is. Defaults to true. + /// + /// + /// 'nmsed_boxes': A [batch_size, max_detections, 4] float32 tensor containing the non-max suppressed boxes. + /// 'nmsed_scores': A [batch_size, max_detections] float32 tensor containing the scores for the boxes. + /// 'nmsed_classes': A [batch_size, max_detections] float32 tensor containing the class for boxes. + /// 'valid_detections': A [batch_size] int32 tensor indicating the number of + /// valid detections per batch item. Only the top valid_detections[i] entries + /// in nms_boxes[i], nms_scores[i] and nms_class[i] are valid. The rest of the + /// entries are zero paddings. + /// + public (Tensor, Tensor, Tensor, Tensor) combined_non_max_suppression( + Tensor boxes, + Tensor scores, + int max_output_size_per_class, + int max_total_size, + float iou_threshold, + float score_threshold, + bool pad_per_class = false, + bool clip_boxes = true) + { + var iou_threshold_t = ops.convert_to_tensor(iou_threshold, TF_DataType.TF_FLOAT, name: "iou_threshold"); + var score_threshold_t = ops.convert_to_tensor(score_threshold, TF_DataType.TF_FLOAT, name: "score_threshold"); + var max_total_size_t = ops.convert_to_tensor(max_total_size); + var max_output_size_per_class_t = ops.convert_to_tensor(max_output_size_per_class); + return gen_image_ops.combined_non_max_suppression(boxes, scores, max_output_size_per_class_t, max_total_size_t, + iou_threshold_t, score_threshold_t, pad_per_class, clip_boxes); + } /// /// Extracts crops from the input image tensor and resizes them using bilinear sampling or nearest neighbor sampling (possibly with aspect ratio change) to a common output size specified by crop_size. This is more general than the crop_to_bounding_box op which extracts a fixed size slice from the input image and does not allow resizing or aspect ratio change. @@ -187,7 +282,19 @@ public Tensor decode_jpeg(Tensor contents, /// A name for the operation (optional). /// A 4-D tensor of shape [num_boxes, crop_height, crop_width, depth]. public Tensor crop_and_resize(Tensor image, Tensor boxes, Tensor box_ind, Tensor crop_size, string method = "bilinear", float extrapolation_value = 0f, string name = null) => - image_ops_impl.crop_and_resize(image, boxes, box_ind, crop_size, method, extrapolation_value, name); + gen_image_ops.crop_and_resize(image, boxes, box_ind, crop_size, method, extrapolation_value, name); + + public Tensor decode_jpeg(Tensor contents, + int channels = 0, + int ratio = 1, + bool fancy_upscaling = true, + bool try_recover_truncated = false, + int acceptable_fraction = 1, + string dct_method = "", + string name = null) + => gen_image_ops.decode_jpeg(contents, channels: channels, ratio: ratio, + fancy_upscaling: fancy_upscaling, try_recover_truncated: try_recover_truncated, + acceptable_fraction: acceptable_fraction, dct_method: dct_method); public Tensor extract_glimpse(Tensor input, Tensor size, Tensor offsets, bool centered = true, bool normalized = true, bool uniform_noise = true, string name = null) diff --git a/src/TensorFlowNET.Core/Operations/gen_image_ops.cs b/src/TensorFlowNET.Core/Operations/gen_image_ops.cs index 9240b5905..cbe661ae5 100644 --- a/src/TensorFlowNET.Core/Operations/gen_image_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_image_ops.cs @@ -16,18 +16,312 @@ limitations under the License. using System; using System.Linq; +using Tensorflow.Eager; using static Tensorflow.Binding; +using Tensorflow.Exceptions; +using Tensorflow.Contexts; +using System.Xml.Linq; +using Google.Protobuf; namespace Tensorflow { public class gen_image_ops { + public static Tensor adjust_contrastv2(Tensor images, Tensor contrast_factor, string name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "AdjustContrastv2", name) { + args = new object[] { images, contrast_factor }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return adjust_contrastv2_eager_fallback(images, contrast_factor, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["images"] = images; + keywords["contrast_factor"] = contrast_factor; + var _op = tf.OpDefLib._apply_op_helper("AdjustContrastv2", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "T", _op._get_attr_type("T") }; + _execute.record_gradient("AdjustContrastv2", _op.inputs, _attrs, _result); + } + return _result[0]; + } + public static Tensor adjust_contrastv2(Tensor image, float contrast_factor, string name = null) + { + return adjust_contrastv2(image, tf.convert_to_tensor(contrast_factor), name: name); + } + + public static Tensor adjust_contrastv2_eager_fallback(Tensor images, Tensor contrast_factor, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { images, contrast_factor}; + object[] _attrs = new object[] { "T", images.dtype }; + var _result = _execute.execute("AdjustContrastv2", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("AdjustContrastv2", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + + public static Tensor adjust_hue(Tensor images, Tensor delta, string name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "AdjustHue", name) { + args = new object[] { images, delta }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return adjust_hue_eager_fallback(images, delta, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["images"] = images; + keywords["delta"] = delta; + var _op = tf.OpDefLib._apply_op_helper("AdjustHue", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "T", _op._get_attr_type("T") }; + _execute.record_gradient("AdjustHue", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor adjust_hue(Tensor images, float delta, string name = null) + => adjust_hue(images, delta, name: name); + + public static Tensor adjust_hue_eager_fallback(Tensor images, Tensor delta, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { images, delta}; + object[] _attrs = new object[] { "T", images.dtype }; + var _result = _execute.execute("AdjustHue", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("AdjustHue", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + + public static Tensor adjust_saturation(Tensor images, Tensor scale, string name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "AdjustSaturation", name) + { + args = new object[] { images, scale }, + attrs = new Dictionary() { } + }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return adjust_hue_eager_fallback(images, scale, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["images"] = images; + keywords["scale"] = scale; + var _op = tf.OpDefLib._apply_op_helper("AdjustSaturation", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "T", _op._get_attr_type("T") }; + _execute.record_gradient("AdjustSaturation", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor adjust_saturation(Tensor images, float scale, string name = null) + => adjust_saturation(images, ops.convert_to_tensor(scale), name: name); + + public static Tensor adjust_saturation_eager_fallback(Tensor images, Tensor scale, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { images, scale }; + object[] _attrs = new object[] { "T", images.dtype }; + var _result = _execute.execute("AdjustSaturation", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("AdjustSaturation", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + public static (Tensor, Tensor, Tensor, Tensor) combined_non_max_suppression(Tensor boxes, Tensor scores, Tensor max_output_size_per_class, Tensor max_total_size, - Tensor iou_threshold, Tensor score_threshold, bool pad_per_class, bool clip_boxes) + Tensor iou_threshold, Tensor score_threshold, bool pad_per_class = false, bool clip_boxes = true, string name = null) { - throw new NotImplementedException("combined_non_max_suppression"); + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "CombinedNonMaxSuppression", name){ + args = new object[] { + boxes, scores, max_output_size_per_class, max_total_size, iou_threshold, score_threshold, + "pad_per_class", pad_per_class, "clip_boxes", clip_boxes}, + attrs = new Dictionary() { }}); + return (_fast_path_result[0], _fast_path_result[1], _fast_path_result[2], _fast_path_result[3]); + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return combined_non_max_suppression_eager_fallback( + boxes, scores, max_output_size_per_class, max_total_size, iou_threshold, + score_threshold, pad_per_class, clip_boxes, name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["boxes"] = boxes; + keywords["scores"] = scores; + keywords["max_output_size_per_class"] = max_output_size_per_class; + keywords["max_total_size"] = max_total_size; + keywords["iou_threshold"] = iou_threshold; + keywords["score_threshold"] = score_threshold; + keywords["pad_per_class"] = pad_per_class; + keywords["clip_boxes"] = clip_boxes; + + var _op = tf.OpDefLib._apply_op_helper("CombinedNonMaxSuppression", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "pad_per_class", _op._get_attr_type("pad_per_class") ,"clip_boxes", _op._get_attr_type("clip_boxes")}; + _execute.record_gradient("CombinedNonMaxSuppression", _op.inputs, _attrs, _result); + } + return (_result[0], _result[1], _result[2], _result[3]); } + public static (Tensor, Tensor, Tensor, Tensor) combined_non_max_suppression_eager_fallback(Tensor boxes, Tensor scores, Tensor max_output_size_per_class, Tensor max_total_size, + Tensor iou_threshold, Tensor score_threshold, bool pad_per_class, bool clip_boxes, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { boxes, scores, max_output_size_per_class, max_total_size, iou_threshold, score_threshold }; + object[] _attrs = new object[] { "pad_per_class", pad_per_class, "clip_boxes", clip_boxes }; + var _result = _execute.execute("CombinedNonMaxSuppression", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("CombinedNonMaxSuppression", _inputs_flat, _attrs, _result); + } + return (_result[0], _result[1], _result[2], _result[3]); + } + + public static Tensor crop_and_resize(Tensor image, Tensor boxes, Tensor box_ind, Tensor crop_size, string method = "bilinear", float extrapolation_value = 0f, string name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "CropAndResize", name) { + args = new object[] { + image, boxes, box_ind, crop_size, "method", method, "extrapolation_value", extrapolation_value }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return crop_and_resize_eager_fallback( + image, boxes, box_ind, crop_size, method: method, extrapolation_value: extrapolation_value, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["image"] = image; + keywords["boxes"] = boxes; + keywords["box_ind"] = box_ind; + keywords["crop_size"] = crop_size; + keywords["method"] = method; + keywords["extrapolation_value"] = extrapolation_value; + var _op = tf.OpDefLib._apply_op_helper("CropAndResize", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "T", _op._get_attr_type("T") ,"method", _op._get_attr_type("method") , + "extrapolation_value", _op.get_attr("extrapolation_value")}; + _execute.record_gradient("CropAndResize", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor crop_and_resize_eager_fallback(Tensor image, Tensor boxes, Tensor box_ind, Tensor crop_size, string method, float extrapolation_value, string name, Context ctx) + { + if (method is null) + method = "bilinear"; + //var method_cpmpat = ByteString.CopyFromUtf8(method ?? string.Empty); + //var extrapolation_value_float = (float)extrapolation_value; + + Tensor[] _inputs_flat = new Tensor[] { image, boxes, box_ind, crop_size, tf.convert_to_tensor(method), tf.convert_to_tensor(extrapolation_value) }; + object[] _attrs = new object[] { "T", image.dtype }; + var _result = _execute.execute("CropAndResize", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("CropAndResize", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + + public static Tensor convert_image_dtype(Tensor image, TF_DataType dtype, bool saturate = false, string name = null) { if (dtype == image.dtype) diff --git a/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs b/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs index c42445cf1..151ea834b 100644 --- a/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs +++ b/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs @@ -3,6 +3,7 @@ using System.Linq; using Tensorflow; using static Tensorflow.Binding; +using System; namespace TensorFlowNET.UnitTest { @@ -22,13 +23,75 @@ public void Initialize() contents = tf.io.read_file(imgPath); } + [TestMethod] + public void adjust_contrast() + { + var input = np.array(0f, 1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f); + var image = tf.reshape(input, new int[] { 3, 3, 1 }); + var img = tf.image.adjust_contrast(image, 2.0f); + var res = np.array(-4f, -2f, 0f, 2f, 4f, 6f, 8f, 10f, 12f).reshape((3,3,1)); + Assert.AreEqual(img.numpy(), res); + } + + [Ignore] + [TestMethod] + public void adjust_hue() + { + var image = tf.constant(new int[] {1,2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18}); + image = tf.reshape(image, new int[] { 3, 2, 3 }); + var adjusted_image = tf.image.adjust_hue(image, 0.2f); + var res = tf.constant(new int[] {2,1,3, 4, 5, 6,8,7,9,11,10,12,14,13,15,17,16,18}); + res = tf.reshape(res,(3,2,3)); + Assert.AreEqual(adjusted_image, res); + } + + [TestMethod] + public void combined_non_max_suppression() + { + var boxesX = tf.constant(new float[,] { { 200, 100, 150, 100 }, { 220, 120, 150, 100 }, { 190, 110, 150, 100 },{ 210, 112, 150, 100 } }); + var boxes1 = tf.reshape(boxesX, (1, 4, 1, 4)); + var scoresX = tf.constant(new float[,] { { 0.2f, 0.7f, 0.1f },{ 0.1f, 0.8f, 0.1f },{ 0.3f, 0.6f, 0.1f },{ 0.05f, 0.9f, 0.05f } }); + var scores1 = tf.reshape(scoresX, (1, 4, 3)); + var (boxes, scores, classes, valid_detections) = tf.image.combined_non_max_suppression(boxes1, scores1, 10, 10, 0.5f, 0.2f, clip_boxes:false); + + var boxes_gt = tf.constant(new float[,] { { 210f, 112f, 150f, 100f }, { 200f, 100f, 150f, 100f }, { 190f, 110f, 150f, 100f }, + { 0f, 0f, 0f, 0f},{ 0f, 0f, 0f, 0f},{ 0f, 0f, 0f, 0f},{ 0f, 0f, 0f , 0f},{ 0f, 0f, 0f, 0f},{ 0f , 0f, 0f, 0f},{ 0f, 0f, 0f, 0f} }); + boxes_gt = tf.reshape(boxes_gt,(1, 10, 4)); + Assert.AreEqual(boxes.numpy(), boxes_gt.numpy()); + var scores_gt = tf.constant(new float[,] { { 0.9f, 0.7f, 0.3f, 0f, 0f, 0f, 0f, 0f, 0f, 0f } }); + scores_gt = tf.reshape(scores_gt, (1, 10)); + Assert.AreEqual(scores.numpy(), scores_gt.numpy()); + var classes_gt = tf.constant(new float[,] { { 1f, 1f, 0f, 0f, 0f, 0f, 0f, 0f, 0f, 0f } }); + classes_gt = tf.reshape(classes_gt, (1, 10)); + Assert.AreEqual(classes.numpy(), classes_gt.numpy()); + var valid_detections_gt = tf.constant(new int[,] { { 3 } }); + valid_detections_gt = tf.reshape(valid_detections_gt, (1)); + Assert.AreEqual(valid_detections.numpy(), valid_detections_gt.numpy()); + } + + [TestMethod] + public void crop_and_resize() + { + int BATCH_SIZE = 1; + int NUM_BOXES = 5; + int IMAGE_HEIGHT = 256; + int IMAGE_WIDTH = 256; + int CHANNELS = 3; + var crop_size = tf.constant(new int[] { 24, 24 }); + var image = tf.random.uniform((BATCH_SIZE, IMAGE_HEIGHT, IMAGE_WIDTH, CHANNELS)); + var boxes = tf.random.uniform((NUM_BOXES, 4)); + var box_ind = tf.random.uniform((NUM_BOXES), minval: 0, maxval: BATCH_SIZE, dtype: TF_DataType.TF_INT32); + var output = tf.image.crop_and_resize(image, boxes, box_ind, crop_size); + Assert.AreEqual((5,24,24,3), output.shape); + } + [TestMethod] public void decode_image() { var img = tf.image.decode_image(contents); Assert.AreEqual(img.name, "decode_image/DecodeImage:0"); } - + [TestMethod] public void resize_image() { From 3273cbc7f2e14eb030dfc9967ce5bf550186a93e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Tue, 25 Jul 2023 00:09:50 +0800 Subject: [PATCH 20/98] fix: fix ci error --- .../TensorFlowNET.Graph.UnitTest/ImageTest.cs | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs b/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs index 151ea834b..d671b6096 100644 --- a/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs +++ b/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs @@ -28,9 +28,14 @@ public void adjust_contrast() { var input = np.array(0f, 1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f); var image = tf.reshape(input, new int[] { 3, 3, 1 }); - var img = tf.image.adjust_contrast(image, 2.0f); + + var init = tf.global_variables_initializer(); + var sess = tf.Session(); + sess.run(init); + var adjust_contrast = tf.image.adjust_contrast(image, 2.0f); + var result = sess.run(adjust_contrast); var res = np.array(-4f, -2f, 0f, 2f, 4f, 6f, 8f, 10f, 12f).reshape((3,3,1)); - Assert.AreEqual(img.numpy(), res); + Assert.AreEqual(result.numpy(), res); } [Ignore] @@ -48,25 +53,31 @@ public void adjust_hue() [TestMethod] public void combined_non_max_suppression() { - var boxesX = tf.constant(new float[,] { { 200, 100, 150, 100 }, { 220, 120, 150, 100 }, { 190, 110, 150, 100 },{ 210, 112, 150, 100 } }); + var boxesX = tf.constant(new float[,] { { 200, 100, 150, 100 }, { 220, 120, 150, 100 }, { 190, 110, 150, 100 }, { 210, 112, 150, 100 } }); var boxes1 = tf.reshape(boxesX, (1, 4, 1, 4)); - var scoresX = tf.constant(new float[,] { { 0.2f, 0.7f, 0.1f },{ 0.1f, 0.8f, 0.1f },{ 0.3f, 0.6f, 0.1f },{ 0.05f, 0.9f, 0.05f } }); + var scoresX = tf.constant(new float[,] { { 0.2f, 0.7f, 0.1f }, { 0.1f, 0.8f, 0.1f }, { 0.3f, 0.6f, 0.1f }, { 0.05f, 0.9f, 0.05f } }); var scores1 = tf.reshape(scoresX, (1, 4, 3)); - var (boxes, scores, classes, valid_detections) = tf.image.combined_non_max_suppression(boxes1, scores1, 10, 10, 0.5f, 0.2f, clip_boxes:false); + + var init = tf.global_variables_initializer(); + var sess = tf.Session(); + sess.run(init); + + var (boxes, scores, classes, valid_detections) = tf.image.combined_non_max_suppression(boxes1, scores1, 10, 10, 0.5f, 0.2f, clip_boxes: false); + var result = sess.run((boxes, scores, classes, valid_detections)); var boxes_gt = tf.constant(new float[,] { { 210f, 112f, 150f, 100f }, { 200f, 100f, 150f, 100f }, { 190f, 110f, 150f, 100f }, { 0f, 0f, 0f, 0f},{ 0f, 0f, 0f, 0f},{ 0f, 0f, 0f, 0f},{ 0f, 0f, 0f , 0f},{ 0f, 0f, 0f, 0f},{ 0f , 0f, 0f, 0f},{ 0f, 0f, 0f, 0f} }); - boxes_gt = tf.reshape(boxes_gt,(1, 10, 4)); - Assert.AreEqual(boxes.numpy(), boxes_gt.numpy()); + boxes_gt = tf.reshape(boxes_gt, (1, 10, 4)); + Assert.AreEqual(result.Item1.numpy(), boxes_gt.numpy()); var scores_gt = tf.constant(new float[,] { { 0.9f, 0.7f, 0.3f, 0f, 0f, 0f, 0f, 0f, 0f, 0f } }); scores_gt = tf.reshape(scores_gt, (1, 10)); - Assert.AreEqual(scores.numpy(), scores_gt.numpy()); + Assert.AreEqual(result.Item2.numpy(), scores_gt.numpy()); var classes_gt = tf.constant(new float[,] { { 1f, 1f, 0f, 0f, 0f, 0f, 0f, 0f, 0f, 0f } }); classes_gt = tf.reshape(classes_gt, (1, 10)); - Assert.AreEqual(classes.numpy(), classes_gt.numpy()); + Assert.AreEqual(result.Item3.numpy(), classes_gt.numpy()); var valid_detections_gt = tf.constant(new int[,] { { 3 } }); valid_detections_gt = tf.reshape(valid_detections_gt, (1)); - Assert.AreEqual(valid_detections.numpy(), valid_detections_gt.numpy()); + Assert.AreEqual(result.Item4.numpy(), valid_detections_gt.numpy()); } [TestMethod] From 005476cbcd71f4bcdfeda8f41461ea20dbdc09df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Wed, 26 Jul 2023 15:31:06 +0800 Subject: [PATCH 21/98] fix: add the gradient of the tf.gradient opr --- src/TensorFlowNET.Core/Gradients/array_grad.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/TensorFlowNET.Core/Gradients/array_grad.cs b/src/TensorFlowNET.Core/Gradients/array_grad.cs index 1b6bc95ee..4b7027992 100644 --- a/src/TensorFlowNET.Core/Gradients/array_grad.cs +++ b/src/TensorFlowNET.Core/Gradients/array_grad.cs @@ -373,5 +373,13 @@ public static Tensor[] _TransposeGrad(Operation op, Tensor[] grads) var p = op.inputs[1]; return new Tensor[] { array_ops.transpose(grads[0], array_ops.invert_permutation(p)), null }; } + + [RegisterGradient("ReverseV2")] + public static Tensor[] _ReverseV2Grad(Operation op, Tensor[] grads) + { + var grad = grads[0]; + var axis = op.inputs[1]; + return new Tensor[] { array_ops.reverse(grad, axis), null }; + } } } From f3b3d8be65f8d037dd456d6380bb93d2e888b53c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Fri, 28 Jul 2023 12:42:11 +0800 Subject: [PATCH 22/98] fix: add the momentum parameter's implemention of SGD --- src/TensorFlowNET.Core/Keras/IOptimizerApi.cs | 2 +- .../Training/gen_training_ops.cs | 4 ++++ .../Optimizers/OptimizerApi.cs | 4 ++-- src/TensorFlowNET.Keras/Optimizers/SGD.cs | 19 ++++++++++++++++++- 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs b/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs index d0d3a74f1..19e3a7b8c 100644 --- a/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs +++ b/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs @@ -63,6 +63,6 @@ IOptimizer RMSprop(float learning_rate = 0.001f, bool centered = false, string name = "RMSprop"); - IOptimizer SGD(float learning_rate); + IOptimizer SGD(float learning_rate, float momentum); } } diff --git a/src/TensorFlowNET.Core/Training/gen_training_ops.cs b/src/TensorFlowNET.Core/Training/gen_training_ops.cs index abe85a141..df7dd9e65 100644 --- a/src/TensorFlowNET.Core/Training/gen_training_ops.cs +++ b/src/TensorFlowNET.Core/Training/gen_training_ops.cs @@ -51,5 +51,9 @@ public static Tensor apply_gradient_descent(IVariableV1 var, Tensor alpha, Tenso public static Tensor resource_apply_gradient_descent(Tensor var, Tensor alpha, Tensor delta, bool use_locking = false, string name = null) => tf.Context.ExecuteOp("ResourceApplyGradientDescent", name, new ExecuteOpArgs(var, alpha, delta).SetAttributes(new { use_locking })); + + public static Tensor resource_apply_keras_momentum(Tensor var, Tensor accum, Tensor lr, Tensor grad, Tensor momentum, bool use_locking = false, bool use_nesterov = false, string name = null) + => tf.Context.ExecuteOp("ResourceApplyKerasMomentum", name, + new ExecuteOpArgs(var, accum, lr, grad, momentum).SetAttributes(new { use_locking, use_nesterov })); } } diff --git a/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs b/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs index 280694268..affd43a4f 100644 --- a/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs +++ b/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs @@ -71,7 +71,7 @@ public IOptimizer RMSprop(float learning_rate = 0.001f, Name = name }); - public IOptimizer SGD(float learning_rate) - => new SGD(learning_rate); + public IOptimizer SGD(float learning_rate, float momentum) + => new SGD(learning_rate, momentum); } } diff --git a/src/TensorFlowNET.Keras/Optimizers/SGD.cs b/src/TensorFlowNET.Keras/Optimizers/SGD.cs index f97f4b15f..1d9ceb810 100644 --- a/src/TensorFlowNET.Keras/Optimizers/SGD.cs +++ b/src/TensorFlowNET.Keras/Optimizers/SGD.cs @@ -22,6 +22,8 @@ public SGD(float learning_rate, _set_hyper("decay", decay); _momentum = momentum > 0; + if (momentum < 0 || momentum > 1) + throw new ValueError($"momentum must be a number between 0 and 1, got {momentum}."); _set_hyper("momentum", momentum); @@ -30,6 +32,13 @@ public SGD(float learning_rate, #pragma warning restore CS1717 // Assignment made to same variable } + protected override void _create_slots(IVariableV1[] var_list) + { + if (_momentum) + foreach (var var in var_list) + add_slot(var, "momentum"); + } + protected override void _prepare_local(DeviceDType device_dtype, Dictionary> _apply_state) { @@ -43,7 +52,15 @@ protected override Operation _resource_apply_dense(IVariableV1 var, Tensor grad, { if (_momentum) { - throw new NotImplementedException("_resource_apply_dense"); + var momentum_var = get_slot(var, "momentum"); + return gen_training_ops.resource_apply_keras_momentum( + var.Handle, + momentum_var.Handle, + _get_hyper("learning_rate", var.dtype), + grad, + _get_hyper("momentum", var.dtype), + use_locking: _use_locking, + use_nesterov: nesterov); } var device_dtype = _apply_state.Keys.FirstOrDefault(x => x.Device == var.Device && x.DType == var.dtype.as_base_dtype()); From 6d3f134637308c4a4f01f49ca9e3b0222644a87b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <583087864@qq.com> Date: Sat, 29 Jul 2023 15:48:13 +0800 Subject: [PATCH 23/98] fix: remove the reflection in the implemention of Bidirectional --- .../Layers/Rnn/Bidirectional.cs | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/Bidirectional.cs b/src/TensorFlowNET.Keras/Layers/Rnn/Bidirectional.cs index 6114d9c7c..0566b08ad 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/Bidirectional.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/Bidirectional.cs @@ -13,17 +13,17 @@ namespace Tensorflow.Keras.Layers /// public class Bidirectional: Wrapper { - BidirectionalArgs _args; - RNN _forward_layer; - RNN _backward_layer; - RNN _layer; - bool _support_masking = true; int _num_constants = 0; + bool _support_masking = true; bool _return_state; bool _stateful; bool _return_sequences; - InputSpec _input_spec; + BidirectionalArgs _args; RNNArgs _layer_args_copy; + RNN _forward_layer; + RNN _backward_layer; + RNN _layer; + InputSpec _input_spec; public Bidirectional(BidirectionalArgs args):base(args) { _args = args; @@ -66,12 +66,16 @@ public Bidirectional(BidirectionalArgs args):base(args) // Recreate the forward layer from the original layer config, so that it // will not carry over any state from the layer. - var actualType = _layer.GetType(); - if (actualType == typeof(LSTM)) + if (_layer is LSTM) { var arg = _layer_args_copy as LSTMArgs; _forward_layer = new LSTM(arg); } + else if(_layer is SimpleRNN) + { + var arg = _layer_args_copy as SimpleRNNArgs; + _forward_layer = new SimpleRNN(arg); + } // TODO(Wanglongzhi2001), add GRU if case. else { @@ -154,12 +158,18 @@ private RNN _recreate_layer_from_config(RNN layer, bool go_backwards = false) { config.GoBackwards = !config.GoBackwards; } - var actualType = layer.GetType(); - if (actualType == typeof(LSTM)) + + if (layer is LSTM) { var arg = config as LSTMArgs; return new LSTM(arg); } + else if(layer is SimpleRNN) + { + var arg = config as SimpleRNNArgs; + return new SimpleRNN(arg); + } + // TODO(Wanglongzhi2001), add GRU if case. else { return new RNN(cell, config); @@ -183,7 +193,6 @@ public override void build(KerasShapesWrapper input_shape) protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { // `Bidirectional.call` implements the same API as the wrapped `RNN`. - Tensors forward_inputs; Tensors backward_inputs; Tensors forward_state; From f5eb4ff0a0950fa1b0c3af9b67950e4f4dc90a1a Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Sat, 26 Aug 2023 10:35:45 +0800 Subject: [PATCH 24/98] fix: partially fix the bug of load_model --- .../ArgsDefinition/Activation/ExponentialArgs.cs | 10 ++++++++++ .../ArgsDefinition/Activation/HardSigmoidArgs.cs | 10 ++++++++++ .../Keras/ArgsDefinition/Activation/SELUArgs.cs | 11 +++++++++++ .../Keras/ArgsDefinition/Activation/SoftplusArgs.cs | 10 ++++++++++ .../Keras/ArgsDefinition/Activation/SoftsignArgs.cs | 10 ++++++++++ .../Keras/ArgsDefinition/Activation/SwishArgs.cs | 10 ++++++++++ .../Keras/ArgsDefinition/Activation/TanhArgs.cs | 10 ++++++++++ .../ArgsDefinition/Convolution/Conv2DTransposeArgs.cs | 10 ++++++++++ .../Keras/ArgsDefinition/Merging/AddArgs.cs | 10 ++++++++++ .../Keras/ArgsDefinition/Merging/ConcatenateArgs.cs | 10 ++++++++++ .../Keras/ArgsDefinition/Merging/SubtractArgs.cs | 10 ++++++++++ .../Pooling/GlobalAveragePooling1DArgs.cs | 10 ++++++++++ .../Pooling/GlobalAveragePooling2DArgs.cs | 10 ++++++++++ .../ArgsDefinition/Pooling/GlobalMaxPooling1DArgs.cs | 10 ++++++++++ .../ArgsDefinition/Pooling/GlobalMaxPooling2DArgs.cs | 10 ++++++++++ .../Keras/ArgsDefinition/Pooling/MaxPooling1DArgs.cs | 10 ++++++++++ 16 files changed, 161 insertions(+) create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/ExponentialArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/HardSigmoidArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SELUArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftplusArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftsignArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SwishArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/TanhArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Convolution/Conv2DTransposeArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/AddArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/ConcatenateArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/SubtractArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalAveragePooling1DArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalAveragePooling2DArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalMaxPooling1DArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalMaxPooling2DArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/MaxPooling1DArgs.cs diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/ExponentialArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/ExponentialArgs.cs new file mode 100644 index 000000000..ef024971d --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/ExponentialArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class ExponentialArgs : LayerArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/HardSigmoidArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/HardSigmoidArgs.cs new file mode 100644 index 000000000..788e0f36d --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/HardSigmoidArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class HardSigmoidArgs : LayerArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SELUArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SELUArgs.cs new file mode 100644 index 000000000..eb0e18446 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SELUArgs.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class SELUArgs : LayerArgs + { + + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftplusArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftplusArgs.cs new file mode 100644 index 000000000..7b4f20795 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftplusArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class SoftplusArgs : LayerArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftsignArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftsignArgs.cs new file mode 100644 index 000000000..4e23d261d --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftsignArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class SoftsignArgs : LayerArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SwishArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SwishArgs.cs new file mode 100644 index 000000000..3dea06a23 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SwishArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class SwishArgs : LayerArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/TanhArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/TanhArgs.cs new file mode 100644 index 000000000..5df41b71b --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/TanhArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class TanhArgs : LayerArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Convolution/Conv2DTransposeArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Convolution/Conv2DTransposeArgs.cs new file mode 100644 index 000000000..3daba9465 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Convolution/Conv2DTransposeArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class Conv2DTransposeArgs : Conv2DArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/AddArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/AddArgs.cs new file mode 100644 index 000000000..016d58203 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/AddArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class AddArgs : MergeArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/ConcatenateArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/ConcatenateArgs.cs new file mode 100644 index 000000000..4a81d139d --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/ConcatenateArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class ConcatenateArgs : MergeArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/SubtractArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/SubtractArgs.cs new file mode 100644 index 000000000..1e3621cb6 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/SubtractArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class SubtractArgs : MergeArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalAveragePooling1DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalAveragePooling1DArgs.cs new file mode 100644 index 000000000..e73aff766 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalAveragePooling1DArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class GlobalAveragePooling1DArgs : Pooling1DArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalAveragePooling2DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalAveragePooling2DArgs.cs new file mode 100644 index 000000000..d143cf471 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalAveragePooling2DArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class GlobalAveragePooling2DArgs : Pooling2DArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalMaxPooling1DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalMaxPooling1DArgs.cs new file mode 100644 index 000000000..e03227feb --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalMaxPooling1DArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class GlobalMaxPooling1DArgs : Pooling1DArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalMaxPooling2DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalMaxPooling2DArgs.cs new file mode 100644 index 000000000..a95cac836 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalMaxPooling2DArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class GlobalMaxPooling2DArgs : Pooling2DArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/MaxPooling1DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/MaxPooling1DArgs.cs new file mode 100644 index 000000000..4cfff2c15 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/MaxPooling1DArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class MaxPooling1DArgs : Pooling1DArgs + { + } +} From f679af67e61c51bee1aca254f993d6d137df07ff Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Sat, 26 Aug 2023 11:36:41 +0800 Subject: [PATCH 25/98] fix: partially fix the bug of load_model --- .../Layers/LayersApi.Activation.cs | 14 +++++++------- .../Layers/LayersApi.Merging.cs | 2 +- src/TensorFlowNET.Keras/Layers/LayersApi.cs | 18 +++++++++--------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.Activation.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.Activation.cs index 280e91e2c..2c55f8fd5 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.Activation.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.Activation.cs @@ -10,14 +10,14 @@ public partial class LayersApi { public ILayer ELU ( float alpha = 0.1f ) => new ELU(new ELUArgs { Alpha = alpha }); public ILayer SELU () - => new SELU(new LayerArgs { }); + => new SELU(new SELUArgs { }); public ILayer Softmax(int axis = -1) => new Softmax(new SoftmaxArgs { axis = axis }); public ILayer Softmax ( Axis axis ) => new Softmax(new SoftmaxArgs { axis = axis }); - public ILayer Softplus () => new Softplus(new LayerArgs { }); - public ILayer HardSigmoid () => new HardSigmoid(new LayerArgs { }); - public ILayer Softsign () => new Softsign(new LayerArgs { }); - public ILayer Swish () => new Swish(new LayerArgs { }); - public ILayer Tanh () => new Tanh(new LayerArgs { }); - public ILayer Exponential () => new Exponential(new LayerArgs { }); + public ILayer Softplus () => new Softplus(new SoftplusArgs { }); + public ILayer HardSigmoid () => new HardSigmoid(new HardSigmoidArgs { }); + public ILayer Softsign () => new Softsign(new SoftsignArgs { }); + public ILayer Swish () => new Swish(new SwishArgs { }); + public ILayer Tanh () => new Tanh(new TanhArgs { }); + public ILayer Exponential () => new Exponential(new ExponentialArgs { }); } } diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.Merging.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.Merging.cs index d94bfb4d8..bf06b1418 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.Merging.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.Merging.cs @@ -14,7 +14,7 @@ public partial class LayersApi /// Axis along which to concatenate. /// public ILayer Concatenate(int axis = -1) - => new Concatenate(new MergeArgs + => new Concatenate(new ConcatenateArgs { Axis = axis }); diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index a04a9c051..9155c7742 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -240,7 +240,7 @@ public ILayer Conv2DTranspose(int filters, string kernel_regularizer = null, string bias_regularizer = null, string activity_regularizer = null) - => new Conv2DTranspose(new Conv2DArgs + => new Conv2DTranspose(new Conv2DTransposeArgs { Rank = 2, Filters = filters, @@ -568,7 +568,7 @@ public ILayer MaxPooling1D(int? pool_size = null, int? strides = null, string padding = "valid", string data_format = null) - => new MaxPooling1D(new Pooling1DArgs + => new MaxPooling1D(new MaxPooling1DArgs { PoolSize = pool_size ?? 2, Strides = strides ?? (pool_size ?? 2), @@ -944,21 +944,21 @@ public ILayer Rescaling(float scale, /// /// public ILayer Add() - => new Add(new MergeArgs { }); + => new Add(new AddArgs { }); /// /// /// /// public ILayer Subtract() - => new Subtract(new MergeArgs { }); + => new Subtract(new SubtractArgs { }); /// /// Global max pooling operation for spatial data. /// /// public ILayer GlobalAveragePooling2D() - => new GlobalAveragePooling2D(new Pooling2DArgs { }); + => new GlobalAveragePooling2D(new GlobalAveragePooling2DArgs { }); /// /// Global average pooling operation for temporal data. @@ -968,7 +968,7 @@ public ILayer GlobalAveragePooling2D() /// /// public ILayer GlobalAveragePooling1D(string data_format = "channels_last") - => new GlobalAveragePooling1D(new Pooling1DArgs { DataFormat = data_format }); + => new GlobalAveragePooling1D(new GlobalAveragePooling1DArgs { DataFormat = data_format }); /// /// Global max pooling operation for spatial data. @@ -977,7 +977,7 @@ public ILayer GlobalAveragePooling1D(string data_format = "channels_last") /// channels_last corresponds to inputs with shape (batch, height, width, channels) while channels_first corresponds to inputs with shape (batch, channels, height, width). /// public ILayer GlobalAveragePooling2D(string data_format = "channels_last") - => new GlobalAveragePooling2D(new Pooling2DArgs { DataFormat = data_format }); + => new GlobalAveragePooling2D(new GlobalAveragePooling2DArgs { DataFormat = data_format }); /// /// Global max pooling operation for 1D temporal data. @@ -988,7 +988,7 @@ public ILayer GlobalAveragePooling2D(string data_format = "channels_last") /// /// public ILayer GlobalMaxPooling1D(string data_format = "channels_last") - => new GlobalMaxPooling1D(new Pooling1DArgs { DataFormat = data_format }); + => new GlobalMaxPooling1D(new GlobalMaxPooling1DArgs { DataFormat = data_format }); /// /// Global max pooling operation for spatial data. @@ -997,7 +997,7 @@ public ILayer GlobalMaxPooling1D(string data_format = "channels_last") /// channels_last corresponds to inputs with shape (batch, height, width, channels) while channels_first corresponds to inputs with shape (batch, channels, height, width). /// public ILayer GlobalMaxPooling2D(string data_format = "channels_last") - => new GlobalMaxPooling2D(new Pooling2DArgs { DataFormat = data_format }); + => new GlobalMaxPooling2D(new GlobalMaxPooling2DArgs { DataFormat = data_format }); /// /// Get an weights initializer from its name. From 8e3ba22c832e6d34598644686e00182924b08c3a Mon Sep 17 00:00:00 2001 From: lingbai-kong Date: Sat, 26 Aug 2023 16:29:28 +0800 Subject: [PATCH 26/98] fix: validate dataset of `Imdb` do not load bug & add: custom `Imdb` path --- src/TensorFlowNET.Keras/Datasets/Imdb.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/TensorFlowNET.Keras/Datasets/Imdb.cs b/src/TensorFlowNET.Keras/Datasets/Imdb.cs index 61ce39475..a62f3f87d 100644 --- a/src/TensorFlowNET.Keras/Datasets/Imdb.cs +++ b/src/TensorFlowNET.Keras/Datasets/Imdb.cs @@ -31,7 +31,7 @@ public class Imdb /// /// /// - public DatasetPass load_data(string path = "imdb.npz", + public DatasetPass load_data(string? path = "imdb.npz", int num_words = -1, int skip_top = 0, int maxlen = -1, @@ -42,7 +42,7 @@ public DatasetPass load_data(string path = "imdb.npz", { if (maxlen == -1) throw new InvalidArgumentError("maxlen must be assigned."); - var dst = Download(); + var dst = path ?? Download(); var lines = File.ReadAllLines(Path.Combine(dst, "imdb_train.txt")); var x_train_string = new string[lines.Length]; @@ -55,7 +55,7 @@ public DatasetPass load_data(string path = "imdb.npz", var x_train = keras.preprocessing.sequence.pad_sequences(PraseData(x_train_string), maxlen: maxlen); - File.ReadAllLines(Path.Combine(dst, "imdb_test.txt")); + lines = File.ReadAllLines(Path.Combine(dst, "imdb_test.txt")); var x_test_string = new string[lines.Length]; var y_test = np.zeros(new int[] { lines.Length }, np.int64); for (int i = 0; i < lines.Length; i++) From ba1ddb44488bbb2f528065ac2be07e9e6965722e Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 26 Aug 2023 11:20:12 -0500 Subject: [PATCH 27/98] Set SGD default value. --- src/TensorFlowNET.Core/Keras/IOptimizerApi.cs | 2 +- .../Tensorflow.Binding.csproj | 10 ++--- .../Optimizers/OptimizerApi.cs | 2 +- .../Tensorflow.Keras.csproj | 39 ++++++++++--------- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs b/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs index 19e3a7b8c..6c15fd469 100644 --- a/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs +++ b/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs @@ -63,6 +63,6 @@ IOptimizer RMSprop(float learning_rate = 0.001f, bool centered = false, string name = "RMSprop"); - IOptimizer SGD(float learning_rate, float momentum); + IOptimizer SGD(float learning_rate = 0.01f, float momentum = 0f); } } diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index ca5aa47a9..babb52561 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -5,13 +5,13 @@ Tensorflow.Binding Tensorflow 2.11.0 - 0.110.2 + 0.110.3 10.0 enable Haiping Chen, Eli Belash, Yaohui Liu, Meinrad Recheis SciSharp STACK False - Apache 2.0, Haiping Chen $([System.DateTime]::UtcNow.ToString(yyyy)) + Apache 2.0, Haiping Chen since 2018 https://github.com/SciSharp/TensorFlow.NET git http://scisharpstack.org @@ -20,7 +20,7 @@ Google's TensorFlow full binding in .NET Standard. Building, training and infering deep learning models. https://tensorflownet.readthedocs.io - 0.110.1.0 + 0.110.3.0 tf.net 0.110.x and above are based on tensorflow native 2.11.0 * Support RNN, LSTM model. @@ -43,7 +43,7 @@ https://tensorflownet.readthedocs.io tf.net 0.10x.x aligns with TensorFlow v2.10.x native library. tf.net 0.11x.x aligns with TensorFlow v2.11.x native library. - 0.110.2.0 + 0.110.3.0 LICENSE true packages @@ -172,7 +172,7 @@ https://tensorflownet.readthedocs.io - + diff --git a/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs b/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs index affd43a4f..a237499f9 100644 --- a/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs +++ b/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs @@ -71,7 +71,7 @@ public IOptimizer RMSprop(float learning_rate = 0.001f, Name = name }); - public IOptimizer SGD(float learning_rate, float momentum) + public IOptimizer SGD(float learning_rate = 0.01f, float momentum = 0f) => new SGD(learning_rate, momentum); } } diff --git a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj index eeb7c559f..36d1bc1d4 100644 --- a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj +++ b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj @@ -7,27 +7,30 @@ enable Tensorflow.Keras AnyCPU;x64 - 0.11.2 + 0.11.3 Haiping Chen Keras for .NET - Apache 2.0, Haiping Chen 2023 + Apache 2.0, Haiping Chen since 2018 TensorFlow.Keras https://github.com/SciSharp/TensorFlow.NET https://avatars3.githubusercontent.com/u/44989469?s=200&v=4 https://github.com/SciSharp/TensorFlow.NET - Keras for .NET is a C# version of Keras ported from the python version. - -* Support CIFAR-10 dataset in keras.datasets. -* Support Conv2D functional API. -* Support BatchNormalization layer. -* Building keras model in subclass, functional and sequential api -* Implemented backward_function. -* Support model.load_weights. -* Add Subtract layer -* Text preprocessing -* Preprocessing.timeseries_dataset_from_array -* Fixed memory leak for YOLOv3 model. -* Support RNN and LSTM models + + Keras for .NET is a C# version of Keras ported from the python version. + + * Support CIFAR-10 dataset in keras.datasets. + * Support Conv2D functional API. + * Support BatchNormalization layer. + * Building keras model in subclass, functional and sequential api + * Implemented backward_function. + * Support model.load_weights. + * Add Subtract layer + * Text preprocessing + * Preprocessing.timeseries_dataset_from_array + * Fixed memory leak for YOLOv3 model. + * Support RNN and LSTM models + * Support Transformer model + Keras for .NET Keras is an API designed for human beings, not machines. Keras follows best practices for reducing cognitive load: it offers consistent & simple APIs, it minimizes the number of user actions required for common use cases, and it provides clear & actionable error messages. @@ -39,8 +42,8 @@ Keras is an API designed for human beings, not machines. Keras follows best prac Git False Open.snk - 0.11.2.0 - 0.11.2.0 + 0.11.3.0 + 0.11.3.0 LICENSE Debug;Release;GPU @@ -140,7 +143,7 @@ Keras is an API designed for human beings, not machines. Keras follows best prac - + From 7b077eac7e6a9e60d9d34be9782e222317fbe353 Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Mon, 4 Sep 2023 00:05:22 +0800 Subject: [PATCH 28/98] feat: implement GRU layer --- .../Keras/ArgsDefinition/Rnn/GRUArgs.cs | 29 +++ .../ArgsDefinition/Rnn/GRUOptionalArgs.cs | 13 ++ .../Keras/Layers/ILayersApi.cs | 19 ++ src/TensorFlowNET.Keras/Layers/LayersApi.cs | 61 ++++++- src/TensorFlowNET.Keras/Layers/Rnn/GRU.cs | 168 ++++++++++++++++++ src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs | 42 +---- .../Layers/Rnn.Test.cs | 9 + 7 files changed, 300 insertions(+), 41 deletions(-) create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUOptionalArgs.cs create mode 100644 src/TensorFlowNET.Keras/Layers/Rnn/GRU.cs diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUArgs.cs new file mode 100644 index 000000000..cdc3097e9 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUArgs.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class GRUArgs : AutoSerializeLayerArgs + { + public int Units { get; set; } + public Activation Activation { get; set; } + public Activation RecurrentActivation { get; set; } + public bool UseBias { get; set; } = true; + public float Dropout { get; set; } = .0f; + public float RecurrentDropout { get; set; } = .0f; + public IInitializer KernelInitializer { get; set; } + public IInitializer RecurrentInitializer { get; set; } + public IInitializer BiasInitializer { get; set; } + public bool ReturnSequences { get;set; } + public bool ReturnState { get;set; } + public bool GoBackwards { get;set; } + public bool Stateful { get;set; } + public bool Unroll { get;set; } + public bool TimeMajor { get;set; } + public bool ResetAfter { get;set; } + public int Implementation { get; set; } = 2; + + } + +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUOptionalArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUOptionalArgs.cs new file mode 100644 index 000000000..d441dc828 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUOptionalArgs.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class GRUOptionalArgs + { + public string Identifier => "GRU"; + + public Tensor Mask { get; set; } = null; + } +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index b8aff5fb6..5e08eadc4 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -259,6 +259,25 @@ public IRnnCell GRUCell( float recurrent_dropout = 0f, bool reset_after = true); + public ILayer GRU( + int units, + string activation = "tanh", + string recurrent_activation = "sigmoid", + bool use_bias = true, + string kernel_initializer = "glorot_uniform", + string recurrent_initializer = "orthogonal", + string bias_initializer = "zeros", + float dropout = 0f, + float recurrent_dropout = 0f, + bool return_sequences = false, + bool return_state = false, + bool go_backwards = false, + bool stateful = false, + bool unroll = false, + bool time_major = false, + bool reset_after = true + ); + /// /// Bidirectional wrapper for RNNs. /// diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index 9155c7742..928e7e337 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -784,7 +784,7 @@ public IRnnCell LSTMCell(int uints, string recurrent_activation = "sigmoid", bool use_bias = true, string kernel_initializer = "glorot_uniform", - string recurrent_initializer = "orthogonal", // TODO(Wanglongzhi2001),glorot_uniform has not been developed. + string recurrent_initializer = "orthogonal", string bias_initializer = "zeros", bool unit_forget_bias = true, float dropout = 0f, @@ -908,6 +908,65 @@ public IRnnCell GRUCell( ResetAfter = reset_after }); + /// + /// Gated Recurrent Unit - Cho et al. 2014. + /// + /// Positive integer, dimensionality of the output space. + /// Activation function to use. If you pass `None`, no activation is applied.(ie. "linear" activation: `a(x) = x`). + /// Activation function to use for the recurrent step. If you pass `None`, no activation is applied. (ie. "linear" activation: `a(x) = x`). + /// Boolean, (default `True`), whether the layer uses a bias vector. + /// Initializer for the `kernel` weights matrix, used for the linear transformation of the inputs. Default: `glorot_uniform`. + /// Initializer for the `recurrent_kernel` weights matrix, used for the linear transformation of the recurrent state. Default: `orthogonal`. + /// Initializer for the bias vector. Default: `zeros`. + /// Float between 0 and 1. Fraction of the units to drop for the linear transformation of the inputs. Default: 0. + /// Float between 0 and 1. Fraction of the units to drop for the linear transformation of the recurrent state. Default: 0. + /// + /// Boolean. Whether to return the last output in the output sequence, or the full sequence. Default: `False`. + /// Boolean. Whether to return the last state in addition to the output. Default: `False`. + /// Boolean (default `False`). If True, process the input sequence backwards and return the reversed sequence. + /// Boolean (default False). If True, the last state for each sample at index i in a batch will be used as initial state for the sample of index i in the following batch. + /// Boolean (default False). If True, the network will be unrolled, else a symbolic loop will be used. Unrolling can speed-up a RNN, + /// The shape format of the `inputs` and `outputs` tensors. + /// GRU convention (whether to apply reset gate after or before matrix multiplication). False = "before", True = "after" (default and cuDNN compatible). + /// + public ILayer GRU( + int units, + string activation = "tanh", + string recurrent_activation = "sigmoid", + bool use_bias = true, + string kernel_initializer = "glorot_uniform", + string recurrent_initializer = "orthogonal", + string bias_initializer = "zeros", + float dropout = 0f, + float recurrent_dropout = 0f, + bool return_sequences = false, + bool return_state = false, + bool go_backwards = false, + bool stateful = false, + bool unroll = false, + bool time_major = false, + bool reset_after = true + ) + => new GRU(new GRUArgs + { + Units = units, + Activation = keras.activations.GetActivationFromName(activation), + RecurrentActivation = keras.activations.GetActivationFromName(recurrent_activation), + KernelInitializer = GetInitializerByName(kernel_initializer), + RecurrentInitializer = GetInitializerByName(recurrent_initializer), + BiasInitializer = GetInitializerByName(bias_initializer), + UseBias = use_bias, + Dropout = dropout, + RecurrentDropout = recurrent_dropout, + ReturnSequences = return_sequences, + ReturnState = return_state, + GoBackwards = go_backwards, + Stateful = stateful, + TimeMajor = time_major, + Unroll = unroll, + ResetAfter = reset_after + }); + public ILayer Bidirectional( ILayer layer, string merge_mode = "concat", diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/GRU.cs b/src/TensorFlowNET.Keras/Layers/Rnn/GRU.cs new file mode 100644 index 000000000..0919883d2 --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Rnn/GRU.cs @@ -0,0 +1,168 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Common.Extensions; +using Tensorflow.Common.Types; +using Tensorflow.Keras.Saving; + + +namespace Tensorflow.Keras.Layers +{ + public class GRU : RNN + { + GRUArgs _args; + private static GRUCell _cell; + + bool _return_runtime; + public GRUCell Cell { get => _cell; } + public int units { get => _args.Units; } + public Activation activation { get => _args.Activation; } + public Activation recurrent_activation { get => _args.RecurrentActivation; } + public bool use_bias { get => _args.UseBias; } + public float dropout { get => _args.Dropout; } + public float recurrent_dropout { get => _args.RecurrentDropout; } + public IInitializer kernel_initializer { get => _args.KernelInitializer; } + public IInitializer recurrent_initializer { get => _args.RecurrentInitializer; } + public IInitializer bias_initializer { get => _args.BiasInitializer; } + public int implementation { get => _args.Implementation; } + public bool reset_after { get => _args.ResetAfter; } + + public GRU(GRUArgs args) : base(CreateCell(args), PreConstruct(args)) + { + _args = args; + + if (_args.Implementation == 0) + { + // Use the red output to act as a warning message that can also be used under the release version + Console.ForegroundColor = ConsoleColor.Red; + Console.WriteLine("Warning: `implementation=0` has been deprecated, "+ + "and now defaults to `implementation=2`."+ + "Please update your layer call."); + Console.ResetColor(); + } + + GRUCell cell = new GRUCell(new GRUCellArgs + { + Units = _args.Units, + Activation = _args.Activation, + RecurrentActivation = _args.RecurrentActivation, + UseBias = _args.UseBias, + Dropout = _args.Dropout, + RecurrentDropout = _args.RecurrentDropout, + KernelInitializer = _args.KernelInitializer, + RecurrentInitializer = _args.RecurrentInitializer, + BiasInitializer = _args.BiasInitializer, + ResetAfter = _args.ResetAfter, + Implementation = _args.Implementation + }); + _cell = cell; + } + + protected override Tensors Call(Tensors inputs, Tensors initial_state = null, bool? training = null, IOptionalArgs? optional_args = null) + { + GRUOptionalArgs? gru_optional_args = optional_args as GRUOptionalArgs; + if (optional_args is not null && gru_optional_args is null) + { + throw new ArgumentException("The type of optional args should be `GRUOptionalArgs`."); + } + Tensors? mask = gru_optional_args?.Mask; + + // Not support ragger input temporarily; + int row_length = 0; + bool is_ragged_input = false; + + _validate_args_if_ragged(is_ragged_input, mask); + + // GRU does not support constants.Ignore it during process. + (inputs, initial_state, _) = this._process_inputs(inputs, initial_state, null); + + if (mask.Length > 1) + { + mask = mask[0]; + } + + var input_shape = inputs.shape; + var timesteps = _args.TimeMajor ? input_shape[0] : input_shape[1]; + + + // TODO(Wanglongzhi2001), finish _could_use_gpu_kernel part + Func step = (cell_inputs, cell_states) => + { + var res = Cell.Apply(cell_inputs, cell_states, training is null ? true : training.Value); + var (output, state) = res; + return (output, state); + }; + + var (last_output, outputs, states) = keras.backend.rnn( + step, + inputs, + initial_state, + constants: null, + go_backwards: _args.GoBackwards, + mask: mask, + unroll: _args.Unroll, + input_length: ops.convert_to_tensor(timesteps), + time_major: _args.TimeMajor, + zero_output_for_mask: base.Args.ZeroOutputForMask, + return_all_outputs: _args.ReturnSequences + ); + + Tensors output; + if (_args.ReturnSequences) + { + output = outputs; + } + else + { + output = last_output; + } + + if (_args.ReturnState) + { + output = new Tensors { output, states }; + } + return output; + } + + private static IRnnCell CreateCell(GRUArgs gruArgs) + { + return new GRUCell(new GRUCellArgs + { + Units = gruArgs.Units, + Activation = gruArgs.Activation, + RecurrentActivation = gruArgs.RecurrentActivation, + UseBias = gruArgs.UseBias, + Dropout = gruArgs.Dropout, + RecurrentDropout = gruArgs.RecurrentDropout, + KernelInitializer = gruArgs.KernelInitializer, + RecurrentInitializer = gruArgs.RecurrentInitializer, + BiasInitializer = gruArgs.BiasInitializer, + ResetAfter = gruArgs.ResetAfter, + Implementation = gruArgs.Implementation + }); + } + + private static RNNArgs PreConstruct(GRUArgs args) + { + return new RNNArgs + { + ReturnSequences = args.ReturnSequences, + ReturnState = args.ReturnState, + GoBackwards = args.GoBackwards, + Stateful = args.Stateful, + Unroll = args.Unroll, + TimeMajor = args.TimeMajor, + Units = args.Units, + Activation = args.Activation, + RecurrentActivation = args.RecurrentActivation, + UseBias = args.UseBias, + Dropout = args.Dropout, + RecurrentDropout = args.RecurrentDropout, + KernelInitializer = args.KernelInitializer, + RecurrentInitializer = args.RecurrentInitializer, + BiasInitializer = args.BiasInitializer + }; + } + } +} diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index c19222614..fec75559c 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -25,8 +25,8 @@ public class RNN : RnnBase private RNNArgs _args; private object _input_spec = null; // or NoneValue?? private object _state_spec = null; - private Tensors _states = null; private object _constants_spec = null; + private Tensors _states = null; private int _num_constants; protected IVariableV1 _kernel; protected IVariableV1 _bias; @@ -469,7 +469,7 @@ public override Tensors Apply(Tensors inputs, Tensors initial_states = null, boo return (inputs, initial_state, constants); } - private void _validate_args_if_ragged(bool is_ragged_input, Tensors mask) + protected void _validate_args_if_ragged(bool is_ragged_input, Tensors mask) { if (!is_ragged_input) { @@ -528,44 +528,6 @@ public Tensors __call__(Tensors inputs, Tensor state = null, Tensor training = n throw new NotImplementedException(); } - // 好像不能cell不能传接口类型 - //public RNN New(IRnnArgCell cell, - // bool return_sequences = false, - // bool return_state = false, - // bool go_backwards = false, - // bool stateful = false, - // bool unroll = false, - // bool time_major = false) - // => new RNN(new RNNArgs - // { - // Cell = cell, - // ReturnSequences = return_sequences, - // ReturnState = return_state, - // GoBackwards = go_backwards, - // Stateful = stateful, - // Unroll = unroll, - // TimeMajor = time_major - // }); - - //public RNN New(List cell, - // bool return_sequences = false, - // bool return_state = false, - // bool go_backwards = false, - // bool stateful = false, - // bool unroll = false, - // bool time_major = false) - // => new RNN(new RNNArgs - // { - // Cell = cell, - // ReturnSequences = return_sequences, - // ReturnState = return_state, - // GoBackwards = go_backwards, - // Stateful = stateful, - // Unroll = unroll, - // TimeMajor = time_major - // }); - - protected Tensors get_initial_state(Tensors inputs) { var input = inputs[0]; diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs index 03159346a..dbf5cae1e 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs @@ -146,6 +146,15 @@ public void GRUCell() } + [TestMethod] + public void GRU() + { + var inputs = tf.ones((32, 10, 8)); + var gru = tf.keras.layers.GRU(4); + var output = gru.Apply(inputs); + Assert.AreEqual((32, 4), output.shape); + } + [TestMethod] public void Bidirectional() { From 9d10daf30f02ebf078d56aadca59cc269ae23b4d Mon Sep 17 00:00:00 2001 From: lingbai-kong Date: Wed, 6 Sep 2023 23:12:00 +0800 Subject: [PATCH 29/98] add reconstruction and setstate of NDArray for loading pickled npy file. --- .../NumPy/DtypeConstructor.cs | 55 ++++++++--- .../Implementation/NumPyImpl.Creation.cs | 3 - .../NumPy/Implementation/NumPyImpl.load.cs | 24 ++--- .../NumPy/MultiArrayConstructor.cs | 35 ++++--- .../NumPy/NDArray.Pickle.cs | 99 ++++++++++++++++++- .../NumPy/NDArrayConverter.cs | 1 + src/TensorFlowNET.Core/Numpy/Numpy.cs | 4 +- src/TensorFlowNET.Keras/Datasets/Imdb.cs | 10 +- 8 files changed, 178 insertions(+), 53 deletions(-) diff --git a/src/TensorFlowNET.Core/NumPy/DtypeConstructor.cs b/src/TensorFlowNET.Core/NumPy/DtypeConstructor.cs index f84f408e1..30ef82df4 100644 --- a/src/TensorFlowNET.Core/NumPy/DtypeConstructor.cs +++ b/src/TensorFlowNET.Core/NumPy/DtypeConstructor.cs @@ -16,25 +16,50 @@ class DtypeConstructor : IObjectConstructor { public object construct(object[] args) { - Console.WriteLine("DtypeConstructor"); - Console.WriteLine(args.Length); - for (int i = 0; i < args.Length; i++) - { - Console.WriteLine(args[i]); - } - return new demo(); + var typeCode = (string)args[0]; + TF_DataType dtype; + if (typeCode == "b1") + dtype = np.@bool; + else if (typeCode == "i1") + dtype = np.@byte; + else if (typeCode == "i2") + dtype = np.int16; + else if (typeCode == "i4") + dtype = np.int32; + else if (typeCode == "i8") + dtype = np.int64; + else if (typeCode == "u1") + dtype = np.ubyte; + else if (typeCode == "u2") + dtype = np.uint16; + else if (typeCode == "u4") + dtype = np.uint32; + else if (typeCode == "u8") + dtype = np.uint64; + else if (typeCode == "f4") + dtype = np.float32; + else if (typeCode == "f8") + dtype = np.float64; + else if (typeCode.StartsWith("S")) + dtype = np.@string; + else if (typeCode.StartsWith("O")) + dtype = np.@object; + else + throw new NotSupportedException(); + return new TF_DataType_Warpper(dtype); } } - class demo + public class TF_DataType_Warpper { - public void __setstate__(object[] args) + TF_DataType dtype { get; set; } + public TF_DataType_Warpper(TF_DataType dtype) { - Console.WriteLine("demo __setstate__"); - Console.WriteLine(args.Length); - for (int i = 0; i < args.Length; i++) - { - Console.WriteLine(args[i]); - } + this.dtype = dtype; + } + public void __setstate__(object[] args) { } + public static implicit operator TF_DataType(TF_DataType_Warpper dtypeWarpper) + { + return dtypeWarpper.dtype; } } } diff --git a/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs b/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs index 80b62198a..7b79f83c6 100644 --- a/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs +++ b/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs @@ -99,9 +99,6 @@ Array ReadValueMatrix(BinaryReader reader, Array matrix, int bytes, Type type, i NDArray ReadObjectMatrix(BinaryReader reader, Array matrix, int[] shape) { - //int data = reader.ReadByte(); - //Console.WriteLine(data); - //Console.WriteLine(reader.ReadByte()); Stream stream = reader.BaseStream; Unpickler.registerConstructor("numpy.core.multiarray", "_reconstruct", new MultiArrayConstructor()); Unpickler.registerConstructor("numpy", "dtype", new DtypeConstructor()); diff --git a/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.load.cs b/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.load.cs index 789f119a1..bbe48e6a4 100644 --- a/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.load.cs +++ b/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.load.cs @@ -28,17 +28,17 @@ public Array LoadMatrix(Stream stream) //if (type == typeof(String)) //return ReadStringMatrix(reader, matrix, bytes, type, shape); - NDArray res = ReadObjectMatrix(reader, matrix, shape); - Console.WriteLine("LoadMatrix"); - Console.WriteLine(res.dims[0]); - Console.WriteLine((int)res[0][0]); - Console.WriteLine(res.dims[1]); - //if (type == typeof(Object)) - //{ - - //} - //else - return ReadValueMatrix(reader, matrix, bytes, type, shape); + + if (type == typeof(Object)) + { + NDArray res = ReadObjectMatrix(reader, matrix, shape); + // res = res.reconstructedNDArray; + return res.reconstructedArray; + } + else + { + return ReadValueMatrix(reader, matrix, bytes, type, shape); + } } } @@ -133,7 +133,7 @@ Type GetType(string dtype, out int bytes, out bool? isLittleEndian) return typeof(Double); if (typeCode.StartsWith("S")) return typeof(String); - if (typeCode == "O") + if (typeCode.StartsWith("O")) return typeof(Object); throw new NotSupportedException(); diff --git a/src/TensorFlowNET.Core/NumPy/MultiArrayConstructor.cs b/src/TensorFlowNET.Core/NumPy/MultiArrayConstructor.cs index 92927cd5a..43eda23e0 100644 --- a/src/TensorFlowNET.Core/NumPy/MultiArrayConstructor.cs +++ b/src/TensorFlowNET.Core/NumPy/MultiArrayConstructor.cs @@ -3,6 +3,7 @@ using System.Diagnostics.CodeAnalysis; using System.Text; using Razorvine.Pickle; +using Razorvine.Pickle.Objects; namespace Tensorflow.NumPy { @@ -17,28 +18,36 @@ public class MultiArrayConstructor : IObjectConstructor { public object construct(object[] args) { - //Console.WriteLine(args.Length); - //for (int i = 0; i < args.Length; i++) - //{ - // Console.WriteLine(args[i]); - //} - Console.WriteLine("MultiArrayConstructor"); - + if (args.Length != 3) + throw new InvalidArgumentError($"Invalid number of arguments in MultiArrayConstructor._reconstruct. Expected three arguments. Given {args.Length} arguments."); + + var types = (ClassDictConstructor)args[0]; + if (types.module != "numpy" || types.name != "ndarray") + throw new RuntimeError("_reconstruct: First argument must be a sub-type of ndarray"); + var arg1 = (Object[])args[1]; var dims = new int[arg1.Length]; for (var i = 0; i < arg1.Length; i++) { dims[i] = (int)arg1[i]; } + var shape = new Shape(dims); - var dtype = TF_DataType.DtInvalid; - switch (args[2]) + TF_DataType dtype; + string identifier; + if (args[2].GetType() == typeof(string)) + identifier = (string)args[2]; + else + identifier = Encoding.UTF8.GetString((byte[])args[2]); + switch (identifier) { - case "b": dtype = TF_DataType.DtUint8Ref; break; - default: throw new NotImplementedException("cannot parse" + args[2]); + case "u": dtype = np.uint32; break; + case "c": dtype = np.complex_; break; + case "f": dtype = np.float32; break; + case "b": dtype = np.@bool; break; + default: throw new NotImplementedException($"Unsupported data type: {args[2]}"); } - return new NDArray(new Shape(dims), dtype); - + return new NDArray(shape, dtype); } } } diff --git a/src/TensorFlowNET.Core/NumPy/NDArray.Pickle.cs b/src/TensorFlowNET.Core/NumPy/NDArray.Pickle.cs index b4d66243a..62720826a 100644 --- a/src/TensorFlowNET.Core/NumPy/NDArray.Pickle.cs +++ b/src/TensorFlowNET.Core/NumPy/NDArray.Pickle.cs @@ -1,4 +1,7 @@ -using System; +using Newtonsoft.Json.Linq; +using Serilog.Debugging; +using System; +using System.Collections; using System.Collections.Generic; using System.Text; @@ -6,14 +9,100 @@ namespace Tensorflow.NumPy { public partial class NDArray { + public NDArray reconstructedNDArray { get; set; } + public Array reconstructedArray { get; set; } public void __setstate__(object[] args) { - Console.WriteLine("NDArray __setstate__"); - Console.WriteLine(args.Length); - for (int i = 0; i < args.Length; i++) + if (args.Length != 5) + throw new InvalidArgumentError($"Invalid number of arguments in NDArray.__setstate__. Expected five arguments. Given {args.Length} arguments."); + + var version = (int)args[0]; // version + + var arg1 = (Object[])args[1]; + var dims = new int[arg1.Length]; + for (var i = 0; i < arg1.Length; i++) + { + dims[i] = (int)arg1[i]; + } + var _ShapeLike = new Shape(dims); // shape + + TF_DataType _DType_co = (TF_DataType_Warpper)args[2]; // DType + + var F_continuous = (bool)args[3]; // F-continuous + if (F_continuous) + throw new InvalidArgumentError("Fortran Continuous memory layout is not supported. Please use C-continuous layout or check the data format."); + + var data = args[4]; // Data + /* + * If we ever need another pickle format, increment the version + * number. But we should still be able to handle the old versions. + */ + if (version < 0 || version > 4) + throw new ValueError($"can't handle version {version} of numpy.dtype pickle"); + + // TODO: Implement the missing details and checks from the official Numpy C code here. + // https://github.com/numpy/numpy/blob/2f0bd6e86a77e4401d0384d9a75edf9470c5deb6/numpy/core/src/multiarray/descriptor.c#L2761 + + if (data.GetType() == typeof(ArrayList)) + { + SetState((ArrayList)data); + } + else + throw new NotImplementedException(""); + } + private void SetState(ArrayList arrayList) + { + int ndim = 1; + var subArrayList = arrayList; + while (subArrayList.Count > 0 && subArrayList[0] != null && subArrayList[0].GetType() == typeof(ArrayList)) + { + subArrayList = (ArrayList)subArrayList[0]; + ndim += 1; + } + var type = subArrayList[0].GetType(); + if (type == typeof(int)) { - Console.WriteLine(args[i]); + if (ndim == 1) + { + int[] list = (int[])arrayList.ToArray(typeof(int)); + Shape shape = new Shape(new int[] { arrayList.Count }); + reconstructedArray = list; + reconstructedNDArray = new NDArray(list, shape); + //SetData(new[] { new Slice() }, new NDArray(list, shape)); + //set_shape(shape); + } + if (ndim == 2) + { + int secondDim = 0; + foreach (ArrayList subArray in arrayList) + { + secondDim = subArray.Count > secondDim ? subArray.Count : secondDim; + } + int[,] list = new int[arrayList.Count, secondDim]; + for (int i = 0; i < arrayList.Count; i++) + { + var subArray = (ArrayList?)arrayList[i]; + if (subArray == null) + throw new NullReferenceException(""); + for (int j = 0; j < subArray.Count; j++) + { + var element = subArray[j]; + if (element == null) + throw new NoNullAllowedException("the element of ArrayList cannot be null."); + list[i,j] = (int) element; + } + } + Shape shape = new Shape(new int[] { arrayList.Count, secondDim }); + reconstructedArray = list; + reconstructedNDArray = new NDArray(list, shape); + //SetData(new[] { new Slice() }, new NDArray(list, shape)); + //set_shape(shape); + } + if (ndim > 2) + throw new NotImplementedException("can't handle ArrayList with more than two dimensions."); } + else + throw new NotImplementedException(""); } } } diff --git a/src/TensorFlowNET.Core/NumPy/NDArrayConverter.cs b/src/TensorFlowNET.Core/NumPy/NDArrayConverter.cs index c8c2d45fa..4c64eba74 100644 --- a/src/TensorFlowNET.Core/NumPy/NDArrayConverter.cs +++ b/src/TensorFlowNET.Core/NumPy/NDArrayConverter.cs @@ -10,6 +10,7 @@ public class NDArrayConverter public unsafe static T Scalar(NDArray nd) where T : unmanaged => nd.dtype switch { + TF_DataType.TF_BOOL => Scalar(*(bool*)nd.data), TF_DataType.TF_UINT8 => Scalar(*(byte*)nd.data), TF_DataType.TF_FLOAT => Scalar(*(float*)nd.data), TF_DataType.TF_INT32 => Scalar(*(int*)nd.data), diff --git a/src/TensorFlowNET.Core/Numpy/Numpy.cs b/src/TensorFlowNET.Core/Numpy/Numpy.cs index 72d2e981c..fee2d63fc 100644 --- a/src/TensorFlowNET.Core/Numpy/Numpy.cs +++ b/src/TensorFlowNET.Core/Numpy/Numpy.cs @@ -43,7 +43,9 @@ public partial class np public static readonly TF_DataType @decimal = TF_DataType.TF_DOUBLE; public static readonly TF_DataType complex_ = TF_DataType.TF_COMPLEX; public static readonly TF_DataType complex64 = TF_DataType.TF_COMPLEX64; - public static readonly TF_DataType complex128 = TF_DataType.TF_COMPLEX128; + public static readonly TF_DataType complex128 = TF_DataType.TF_COMPLEX128; + public static readonly TF_DataType @string = TF_DataType.TF_STRING; + public static readonly TF_DataType @object = TF_DataType.TF_VARIANT; #endregion public static double nan => double.NaN; diff --git a/src/TensorFlowNET.Keras/Datasets/Imdb.cs b/src/TensorFlowNET.Keras/Datasets/Imdb.cs index 016b352d9..6808035c6 100644 --- a/src/TensorFlowNET.Keras/Datasets/Imdb.cs +++ b/src/TensorFlowNET.Keras/Datasets/Imdb.cs @@ -70,7 +70,7 @@ namespace Tensorflow.Keras.Datasets public class Imdb { string origin_folder = "https://storage.googleapis.com/tensorflow/tf-keras-datasets/"; - string file_name = "imdb.npz"; + string file_name = "simple.npz"; string dest_folder = "imdb"; /// /// Loads the [IMDB dataset](https://ai.stanford.edu/~amaas/data/sentiment/). @@ -128,13 +128,15 @@ public DatasetPass load_data(string path = "imdb.npz", (NDArray, NDArray) LoadX(byte[] bytes) { - var y = np.Load_Npz(bytes); - return (y["x_train.npy"], y["x_test.npy"]); + var y = np.Load_Npz(bytes); + var x_train = y["x_train.npy"]; + var x_test = y["x_test.npy"]; + return (x_train, x_test); } (NDArray, NDArray) LoadY(byte[] bytes) { - var y = np.Load_Npz(bytes); + var y = np.Load_Npz(bytes); return (y["y_train.npy"], y["y_test.npy"]); } From ea978bbf214a75ead94c568755255a6f3c6fed58 Mon Sep 17 00:00:00 2001 From: lingbai-kong Date: Thu, 7 Sep 2023 21:33:29 +0800 Subject: [PATCH 30/98] optimize code structure of reconstruction ndarray from pickled npy file --- .../Implementation/NumPyImpl.Creation.cs | 12 ++---- .../NumPy/Implementation/NumPyImpl.load.cs | 10 +---- .../NumPy/Pickle/DTypePickleWarpper.cs | 20 ++++++++++ .../NumPy/{ => Pickle}/DtypeConstructor.cs | 17 +------- .../{ => Pickle}/MultiArrayConstructor.cs | 14 +++---- .../MultiArrayPickleWarpper.cs} | 39 ++++++++++++------- src/TensorFlowNET.Core/tensorflow.cs | 6 +++ src/TensorFlowNET.Keras/Datasets/Imdb.cs | 19 +++------ .../Dataset/DatasetTest.cs | 6 +-- 9 files changed, 75 insertions(+), 68 deletions(-) create mode 100644 src/TensorFlowNET.Core/NumPy/Pickle/DTypePickleWarpper.cs rename src/TensorFlowNET.Core/NumPy/{ => Pickle}/DtypeConstructor.cs (77%) rename src/TensorFlowNET.Core/NumPy/{ => Pickle}/MultiArrayConstructor.cs (91%) rename src/TensorFlowNET.Core/NumPy/{NDArray.Pickle.cs => Pickle/MultiArrayPickleWarpper.cs} (77%) diff --git a/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs b/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs index 7b79f83c6..fa4ef0191 100644 --- a/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs +++ b/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs @@ -5,6 +5,7 @@ using System.Text; using Tensorflow.Util; using Razorvine.Pickle; +using Tensorflow.NumPy.Pickle; using static Tensorflow.Binding; namespace Tensorflow.NumPy @@ -94,20 +95,15 @@ Array ReadValueMatrix(BinaryReader reader, Array matrix, int bytes, Type type, i var buffer = reader.ReadBytes(bytes * total); System.Buffer.BlockCopy(buffer, 0, matrix, 0, buffer.Length); + return matrix; } - NDArray ReadObjectMatrix(BinaryReader reader, Array matrix, int[] shape) + Array ReadObjectMatrix(BinaryReader reader, Array matrix, int[] shape) { Stream stream = reader.BaseStream; - Unpickler.registerConstructor("numpy.core.multiarray", "_reconstruct", new MultiArrayConstructor()); - Unpickler.registerConstructor("numpy", "dtype", new DtypeConstructor()); - var unpickler = new Unpickler(); - - NDArray result = (NDArray) unpickler.load(stream); - Console.WriteLine(result.dims); - return result; + return (MultiArrayPickleWarpper)unpickler.load(stream); } public (NDArray, NDArray) meshgrid(T[] array, bool copy = true, bool sparse = false) diff --git a/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.load.cs b/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.load.cs index bbe48e6a4..199e5ced3 100644 --- a/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.load.cs +++ b/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.load.cs @@ -30,17 +30,12 @@ public Array LoadMatrix(Stream stream) //return ReadStringMatrix(reader, matrix, bytes, type, shape); if (type == typeof(Object)) - { - NDArray res = ReadObjectMatrix(reader, matrix, shape); - // res = res.reconstructedNDArray; - return res.reconstructedArray; - } + return ReadObjectMatrix(reader, matrix, shape); else { return ReadValueMatrix(reader, matrix, bytes, type, shape); } } - } public T Load(Stream stream) @@ -59,7 +54,7 @@ bool ParseReader(BinaryReader reader, out int bytes, out Type t, out int[] shape shape = null; // The first 6 bytes are a magic string: exactly "x93NUMPY" - if (reader.ReadByte() != 0x93) return false; + if (reader.ReadChar() != 63) return false; if (reader.ReadChar() != 'N') return false; if (reader.ReadChar() != 'U') return false; if (reader.ReadChar() != 'M') return false; @@ -75,7 +70,6 @@ bool ParseReader(BinaryReader reader, out int bytes, out Type t, out int[] shape ushort len = reader.ReadUInt16(); string header = new String(reader.ReadChars(len)); - Console.WriteLine(header); string mark = "'descr': '"; int s = header.IndexOf(mark) + mark.Length; int e = header.IndexOf("'", s + 1); diff --git a/src/TensorFlowNET.Core/NumPy/Pickle/DTypePickleWarpper.cs b/src/TensorFlowNET.Core/NumPy/Pickle/DTypePickleWarpper.cs new file mode 100644 index 000000000..5dff6c16b --- /dev/null +++ b/src/TensorFlowNET.Core/NumPy/Pickle/DTypePickleWarpper.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.NumPy.Pickle +{ + public class DTypePickleWarpper + { + TF_DataType dtype { get; set; } + public DTypePickleWarpper(TF_DataType dtype) + { + this.dtype = dtype; + } + public void __setstate__(object[] args) { } + public static implicit operator TF_DataType(DTypePickleWarpper dTypeWarpper) + { + return dTypeWarpper.dtype; + } + } +} diff --git a/src/TensorFlowNET.Core/NumPy/DtypeConstructor.cs b/src/TensorFlowNET.Core/NumPy/Pickle/DtypeConstructor.cs similarity index 77% rename from src/TensorFlowNET.Core/NumPy/DtypeConstructor.cs rename to src/TensorFlowNET.Core/NumPy/Pickle/DtypeConstructor.cs index 30ef82df4..160c7d4e9 100644 --- a/src/TensorFlowNET.Core/NumPy/DtypeConstructor.cs +++ b/src/TensorFlowNET.Core/NumPy/Pickle/DtypeConstructor.cs @@ -4,7 +4,7 @@ using System.Text; using Razorvine.Pickle; -namespace Tensorflow.NumPy +namespace Tensorflow.NumPy.Pickle { /// /// @@ -46,20 +46,7 @@ public object construct(object[] args) dtype = np.@object; else throw new NotSupportedException(); - return new TF_DataType_Warpper(dtype); - } - } - public class TF_DataType_Warpper - { - TF_DataType dtype { get; set; } - public TF_DataType_Warpper(TF_DataType dtype) - { - this.dtype = dtype; - } - public void __setstate__(object[] args) { } - public static implicit operator TF_DataType(TF_DataType_Warpper dtypeWarpper) - { - return dtypeWarpper.dtype; + return new DTypePickleWarpper(dtype); } } } diff --git a/src/TensorFlowNET.Core/NumPy/MultiArrayConstructor.cs b/src/TensorFlowNET.Core/NumPy/Pickle/MultiArrayConstructor.cs similarity index 91% rename from src/TensorFlowNET.Core/NumPy/MultiArrayConstructor.cs rename to src/TensorFlowNET.Core/NumPy/Pickle/MultiArrayConstructor.cs index 43eda23e0..885f368c4 100644 --- a/src/TensorFlowNET.Core/NumPy/MultiArrayConstructor.cs +++ b/src/TensorFlowNET.Core/NumPy/Pickle/MultiArrayConstructor.cs @@ -5,7 +5,7 @@ using Razorvine.Pickle; using Razorvine.Pickle.Objects; -namespace Tensorflow.NumPy +namespace Tensorflow.NumPy.Pickle { /// /// Creates multiarrays of objects. Returns a primitive type multiarray such as int[][] if @@ -18,14 +18,14 @@ public class MultiArrayConstructor : IObjectConstructor { public object construct(object[] args) { - if (args.Length != 3) + if (args.Length != 3) throw new InvalidArgumentError($"Invalid number of arguments in MultiArrayConstructor._reconstruct. Expected three arguments. Given {args.Length} arguments."); - + var types = (ClassDictConstructor)args[0]; - if (types.module != "numpy" || types.name != "ndarray") + if (types.module != "numpy" || types.name != "ndarray") throw new RuntimeError("_reconstruct: First argument must be a sub-type of ndarray"); - - var arg1 = (Object[])args[1]; + + var arg1 = (object[])args[1]; var dims = new int[arg1.Length]; for (var i = 0; i < arg1.Length; i++) { @@ -47,7 +47,7 @@ public object construct(object[] args) case "b": dtype = np.@bool; break; default: throw new NotImplementedException($"Unsupported data type: {args[2]}"); } - return new NDArray(shape, dtype); + return new MultiArrayPickleWarpper(shape, dtype); } } } diff --git a/src/TensorFlowNET.Core/NumPy/NDArray.Pickle.cs b/src/TensorFlowNET.Core/NumPy/Pickle/MultiArrayPickleWarpper.cs similarity index 77% rename from src/TensorFlowNET.Core/NumPy/NDArray.Pickle.cs rename to src/TensorFlowNET.Core/NumPy/Pickle/MultiArrayPickleWarpper.cs index 62720826a..af8d1ecc2 100644 --- a/src/TensorFlowNET.Core/NumPy/NDArray.Pickle.cs +++ b/src/TensorFlowNET.Core/NumPy/Pickle/MultiArrayPickleWarpper.cs @@ -5,12 +5,19 @@ using System.Collections.Generic; using System.Text; -namespace Tensorflow.NumPy +namespace Tensorflow.NumPy.Pickle { - public partial class NDArray + public class MultiArrayPickleWarpper { + public Shape reconstructedShape { get; set; } + public TF_DataType reconstructedDType { get; set; } public NDArray reconstructedNDArray { get; set; } - public Array reconstructedArray { get; set; } + public Array reconstructedMultiArray { get; set; } + public MultiArrayPickleWarpper(Shape shape, TF_DataType dtype) + { + reconstructedShape = shape; + reconstructedDType = dtype; + } public void __setstate__(object[] args) { if (args.Length != 5) @@ -18,7 +25,7 @@ public void __setstate__(object[] args) var version = (int)args[0]; // version - var arg1 = (Object[])args[1]; + var arg1 = (object[])args[1]; var dims = new int[arg1.Length]; for (var i = 0; i < arg1.Length; i++) { @@ -26,7 +33,7 @@ public void __setstate__(object[] args) } var _ShapeLike = new Shape(dims); // shape - TF_DataType _DType_co = (TF_DataType_Warpper)args[2]; // DType + TF_DataType _DType_co = (DTypePickleWarpper)args[2]; // DType var F_continuous = (bool)args[3]; // F-continuous if (F_continuous) @@ -45,12 +52,12 @@ public void __setstate__(object[] args) if (data.GetType() == typeof(ArrayList)) { - SetState((ArrayList)data); + Reconstruct((ArrayList)data); } else throw new NotImplementedException(""); } - private void SetState(ArrayList arrayList) + private void Reconstruct(ArrayList arrayList) { int ndim = 1; var subArrayList = arrayList; @@ -66,10 +73,8 @@ private void SetState(ArrayList arrayList) { int[] list = (int[])arrayList.ToArray(typeof(int)); Shape shape = new Shape(new int[] { arrayList.Count }); - reconstructedArray = list; + reconstructedMultiArray = list; reconstructedNDArray = new NDArray(list, shape); - //SetData(new[] { new Slice() }, new NDArray(list, shape)); - //set_shape(shape); } if (ndim == 2) { @@ -89,14 +94,12 @@ private void SetState(ArrayList arrayList) var element = subArray[j]; if (element == null) throw new NoNullAllowedException("the element of ArrayList cannot be null."); - list[i,j] = (int) element; + list[i, j] = (int)element; } } Shape shape = new Shape(new int[] { arrayList.Count, secondDim }); - reconstructedArray = list; + reconstructedMultiArray = list; reconstructedNDArray = new NDArray(list, shape); - //SetData(new[] { new Slice() }, new NDArray(list, shape)); - //set_shape(shape); } if (ndim > 2) throw new NotImplementedException("can't handle ArrayList with more than two dimensions."); @@ -104,5 +107,13 @@ private void SetState(ArrayList arrayList) else throw new NotImplementedException(""); } + public static implicit operator Array(MultiArrayPickleWarpper arrayWarpper) + { + return arrayWarpper.reconstructedMultiArray; + } + public static implicit operator NDArray(MultiArrayPickleWarpper arrayWarpper) + { + return arrayWarpper.reconstructedNDArray; + } } } diff --git a/src/TensorFlowNET.Core/tensorflow.cs b/src/TensorFlowNET.Core/tensorflow.cs index dc4e48da8..e368b37cd 100644 --- a/src/TensorFlowNET.Core/tensorflow.cs +++ b/src/TensorFlowNET.Core/tensorflow.cs @@ -14,6 +14,7 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using Razorvine.Pickle; using Serilog; using Serilog.Core; using System.Reflection; @@ -22,6 +23,7 @@ limitations under the License. using Tensorflow.Eager; using Tensorflow.Gradients; using Tensorflow.Keras; +using Tensorflow.NumPy.Pickle; namespace Tensorflow { @@ -98,6 +100,10 @@ public tensorflow() "please visit https://github.com/SciSharp/TensorFlow.NET. If it still not work after installing the backend, please submit an " + "issue to https://github.com/SciSharp/TensorFlow.NET/issues"); } + + // register numpy reconstructor for pickle + Unpickler.registerConstructor("numpy.core.multiarray", "_reconstruct", new MultiArrayConstructor()); + Unpickler.registerConstructor("numpy", "dtype", new DtypeConstructor()); } public string VERSION => c_api.StringPiece(c_api.TF_Version()); diff --git a/src/TensorFlowNET.Keras/Datasets/Imdb.cs b/src/TensorFlowNET.Keras/Datasets/Imdb.cs index 6808035c6..a992ae84a 100644 --- a/src/TensorFlowNET.Keras/Datasets/Imdb.cs +++ b/src/TensorFlowNET.Keras/Datasets/Imdb.cs @@ -5,13 +5,6 @@ using Tensorflow.Keras.Utils; using Tensorflow.NumPy; using System.Linq; -using Google.Protobuf.Collections; -using Microsoft.VisualBasic; -using OneOf.Types; -using static HDF.PInvoke.H5; -using System.Data; -using System.Reflection.Emit; -using System.Xml.Linq; namespace Tensorflow.Keras.Datasets { @@ -70,8 +63,9 @@ namespace Tensorflow.Keras.Datasets public class Imdb { string origin_folder = "https://storage.googleapis.com/tensorflow/tf-keras-datasets/"; - string file_name = "simple.npz"; + string file_name = "imdb.npz"; string dest_folder = "imdb"; + /// /// Loads the [IMDB dataset](https://ai.stanford.edu/~amaas/data/sentiment/). /// @@ -95,8 +89,9 @@ public DatasetPass load_data(string path = "imdb.npz", { var dst = Download(); var fileBytes = File.ReadAllBytes(Path.Combine(dst, file_name)); - var (x_train, x_test) = LoadX(fileBytes); var (y_train, y_test) = LoadY(fileBytes); + var (x_train, x_test) = LoadX(fileBytes); + /*var lines = File.ReadAllLines(Path.Combine(dst, "imdb_train.txt")); var x_train_string = new string[lines.Length]; var y_train = np.zeros(new int[] { lines.Length }, np.int64); @@ -129,14 +124,12 @@ public DatasetPass load_data(string path = "imdb.npz", (NDArray, NDArray) LoadX(byte[] bytes) { var y = np.Load_Npz(bytes); - var x_train = y["x_train.npy"]; - var x_test = y["x_test.npy"]; - return (x_train, x_test); + return (y["x_train.npy"], y["x_test.npy"]); } (NDArray, NDArray) LoadY(byte[] bytes) { - var y = np.Load_Npz(bytes); + var y = np.Load_Npz(bytes); return (y["y_train.npy"], y["y_test.npy"]); } diff --git a/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs b/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs index 778290bb8..db6252efc 100644 --- a/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs +++ b/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs @@ -1,6 +1,5 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using System; -using System.Collections.Generic; using System.Linq; using static Tensorflow.Binding; using static Tensorflow.KerasApi; @@ -197,6 +196,7 @@ public void Shuffle() Assert.IsFalse(allEqual); } + [Ignore] [TestMethod] public void GetData() { @@ -209,8 +209,8 @@ public void GetData() var y_val = dataset.Test.Item2; print(len(x_train) + "Training sequences"); print(len(x_val) + "Validation sequences"); - x_train = keras.preprocessing.sequence.pad_sequences((IEnumerable)x_train, maxlen: maxlen); - x_val = keras.preprocessing.sequence.pad_sequences((IEnumerable)x_val, maxlen: maxlen); + //x_train = keras.preprocessing.sequence.pad_sequences((IEnumerable)x_train, maxlen: maxlen); + //x_val = keras.preprocessing.sequence.pad_sequences((IEnumerable)x_val, maxlen: maxlen); } } } From 28c77f53d64dbe78284bf46b00c8c945d76fb31c Mon Sep 17 00:00:00 2001 From: lingbai-kong Date: Fri, 8 Sep 2023 17:38:54 +0800 Subject: [PATCH 31/98] implement Imdb dataset loader --- .../NumPy/Implementation/RandomizedImpl.cs | 4 +- src/TensorFlowNET.Keras/Datasets/Imdb.cs | 186 ++++++++++++------ src/TensorFlowNET.Keras/Utils/data_utils.cs | 47 +++++ .../Dataset/DatasetTest.cs | 28 ++- 4 files changed, 198 insertions(+), 67 deletions(-) diff --git a/src/TensorFlowNET.Core/NumPy/Implementation/RandomizedImpl.cs b/src/TensorFlowNET.Core/NumPy/Implementation/RandomizedImpl.cs index 064c7362f..a707e8aae 100644 --- a/src/TensorFlowNET.Core/NumPy/Implementation/RandomizedImpl.cs +++ b/src/TensorFlowNET.Core/NumPy/Implementation/RandomizedImpl.cs @@ -14,9 +14,9 @@ public class RandomizedImpl public NDArray permutation(NDArray x) => new NDArray(random_ops.random_shuffle(x)); [AutoNumPy] - public void shuffle(NDArray x) + public void shuffle(NDArray x, int? seed = null) { - var y = random_ops.random_shuffle(x); + var y = random_ops.random_shuffle(x, seed); Marshal.Copy(y.BufferToArray(), 0, x.TensorDataPointer, (int)x.bytesize); } diff --git a/src/TensorFlowNET.Keras/Datasets/Imdb.cs b/src/TensorFlowNET.Keras/Datasets/Imdb.cs index 68364ea67..0266b48bd 100644 --- a/src/TensorFlowNET.Keras/Datasets/Imdb.cs +++ b/src/TensorFlowNET.Keras/Datasets/Imdb.cs @@ -3,8 +3,6 @@ using System.IO; using System.Text; using Tensorflow.Keras.Utils; -using Tensorflow.NumPy; -using System.Linq; namespace Tensorflow.Keras.Datasets { @@ -41,14 +39,14 @@ namespace Tensorflow.Keras.Datasets /// `skip_top` limits will be replaced with this character. /// index_from: int. Index actual words with this index and higher. /// Returns: - /// Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`. + /// Tuple of Numpy arrays: `(x_train, labels_train), (x_test, labels_test)`. /// /// ** x_train, x_test**: lists of sequences, which are lists of indexes /// (integers). If the num_words argument was specific, the maximum /// possible index value is `num_words - 1`. If the `maxlen` argument was /// specified, the largest possible sequence length is `maxlen`. /// - /// ** y_train, y_test**: lists of integer labels(1 or 0). + /// ** labels_train, labels_test**: lists of integer labels(1 or 0). /// /// Raises: /// ValueError: in case `maxlen` is so low @@ -63,7 +61,6 @@ namespace Tensorflow.Keras.Datasets public class Imdb { string origin_folder = "https://storage.googleapis.com/tensorflow/tf-keras-datasets/"; - string file_name = "imdb.npz"; string dest_folder = "imdb"; /// @@ -78,43 +75,139 @@ public class Imdb /// /// /// - public DatasetPass load_data(string? path = "imdb.npz", - int num_words = -1, + public DatasetPass load_data( + string path = "imdb.npz", + int? num_words = null, int skip_top = 0, - int maxlen = -1, + int? maxlen = null, int seed = 113, - int start_char = 1, - int oov_char= 2, + int? start_char = 1, + int? oov_char = 2, int index_from = 3) { - if (maxlen == -1) throw new InvalidArgumentError("maxlen must be assigned."); - - var dst = path ?? Download(); - var fileBytes = File.ReadAllBytes(Path.Combine(dst, file_name)); - var (y_train, y_test) = LoadY(fileBytes); + path = data_utils.get_file( + path, + origin: Path.Combine(origin_folder, "imdb.npz"), + file_hash: "69664113be75683a8fe16e3ed0ab59fda8886cb3cd7ada244f7d9544e4676b9f" + ); + path = Path.Combine(path, "imdb.npz"); + var fileBytes = File.ReadAllBytes(path); var (x_train, x_test) = LoadX(fileBytes); - - /*var lines = File.ReadAllLines(Path.Combine(dst, "imdb_train.txt")); - var x_train_string = new string[lines.Length]; - var y_train = np.zeros(new int[] { lines.Length }, np.int64); - for (int i = 0; i < lines.Length; i++) + var (labels_train, labels_test) = LoadY(fileBytes); + x_test.astype(np.int32); + labels_test.astype(np.int32); + + var indices = np.arange(len(x_train)); + np.random.shuffle(indices, seed); + x_train = x_train[indices]; + labels_train = labels_train[indices]; + + indices = np.arange(len(x_test)); + np.random.shuffle(indices, seed); + x_test = x_test[indices]; + labels_test = labels_test[indices]; + + if (start_char != null) + { + int[,] new_x_train = new int[x_train.shape[0], x_train.shape[1] + 1]; + for (var i = 0; i < x_train.shape[0]; i++) + { + new_x_train[i, 0] = (int)start_char; + for (var j = 0; j < x_train.shape[1]; j++) + { + new_x_train[i, j + 1] = x_train[i][j]; + } + } + int[,] new_x_test = new int[x_test.shape[0], x_test.shape[1] + 1]; + for (var i = 0; i < x_test.shape[0]; i++) + { + new_x_test[i, 0] = (int)start_char; + for (var j = 0; j < x_test.shape[1]; j++) + { + new_x_test[i, j + 1] = x_test[i][j]; + } + } + x_train = new NDArray(new_x_train); + x_test = new NDArray(new_x_test); + } + else if (index_from != 0) + { + for (var i = 0; i < x_train.shape[0]; i++) + { + for (var j = 0; j < x_train.shape[1]; j++) + { + if (x_train[i, j] != 0) + x_train[i, j] += index_from; + } + } + for (var i = 0; i < x_test.shape[0]; i++) + { + for (var j = 0; j < x_test.shape[1]; j++) + { + if (x_test[i, j] != 0) + x_test[i, j] += index_from; + } + } + } + + if (maxlen != null) { - y_train[i] = long.Parse(lines[i].Substring(0, 1)); - x_train_string[i] = lines[i].Substring(2); + (x_train, labels_train) = data_utils._remove_long_seq((int)maxlen, x_train, labels_train); + (x_test, labels_test) = data_utils._remove_long_seq((int)maxlen, x_test, labels_test); + if (x_train.size == 0 || x_test.size == 0) + throw new ValueError("After filtering for sequences shorter than maxlen=" + + $"{maxlen}, no sequence was kept. Increase maxlen."); } - var x_train = keras.preprocessing.sequence.pad_sequences(PraseData(x_train_string), maxlen: maxlen); + var xs = np.concatenate(new[] { x_train, x_test }); + var labels = np.concatenate(new[] { labels_train, labels_test }); - lines = File.ReadAllLines(Path.Combine(dst, "imdb_test.txt")); - var x_test_string = new string[lines.Length]; - var y_test = np.zeros(new int[] { lines.Length }, np.int64); - for (int i = 0; i < lines.Length; i++) + if(num_words == null) { - y_test[i] = long.Parse(lines[i].Substring(0, 1)); - x_test_string[i] = lines[i].Substring(2); + num_words = 0; + for (var i = 0; i < xs.shape[0]; i++) + for (var j = 0; j < xs.shape[1]; j++) + num_words = max((int)num_words, (int)xs[i][j]); } - var x_test = np.array(x_test_string);*/ + // by convention, use 2 as OOV word + // reserve 'index_from' (=3 by default) characters: + // 0 (padding), 1 (start), 2 (OOV) + if (oov_char != null) + { + int[,] new_xs = new int[xs.shape[0], xs.shape[1]]; + for(var i = 0; i < xs.shape[0]; i++) + { + for(var j = 0; j < xs.shape[1]; j++) + { + if ((int)xs[i][j] == 0 || skip_top <= (int)xs[i][j] && (int)xs[i][j] < num_words) + new_xs[i, j] = (int)xs[i][j]; + else + new_xs[i, j] = (int)oov_char; + } + } + xs = new NDArray(new_xs); + } + else + { + int[,] new_xs = new int[xs.shape[0], xs.shape[1]]; + for (var i = 0; i < xs.shape[0]; i++) + { + int k = 0; + for (var j = 0; j < xs.shape[1]; j++) + { + if ((int)xs[i][j] == 0 || skip_top <= (int)xs[i][j] && (int)xs[i][j] < num_words) + new_xs[i, k++] = (int)xs[i][j]; + } + } + xs = new NDArray(new_xs); + } + + var idx = len(x_train); + x_train = xs[$"0:{idx}"]; + x_test = xs[$"{idx}:"]; + var y_train = labels[$"0:{idx}"]; + var y_test = labels[$"{idx}:"]; return new DatasetPass { @@ -125,8 +218,8 @@ public DatasetPass load_data(string? path = "imdb.npz", (NDArray, NDArray) LoadX(byte[] bytes) { - var y = np.Load_Npz(bytes); - return (y["x_train.npy"], y["x_test.npy"]); + var x = np.Load_Npz(bytes); + return (x["x_train.npy"], x["x_test.npy"]); } (NDArray, NDArray) LoadY(byte[] bytes) @@ -134,34 +227,5 @@ public DatasetPass load_data(string? path = "imdb.npz", var y = np.Load_Npz(bytes); return (y["y_train.npy"], y["y_test.npy"]); } - - string Download() - { - var dst = Path.Combine(Path.GetTempPath(), dest_folder); - Directory.CreateDirectory(dst); - - Web.Download(origin_folder + file_name, dst, file_name); - - return dst; - // return Path.Combine(dst, file_name); - } - - protected IEnumerable PraseData(string[] x) - { - var data_list = new List(); - for (int i = 0; i < len(x); i++) - { - var list_string = x[i]; - var cleaned_list_string = list_string.Replace("[", "").Replace("]", "").Replace(" ", ""); - string[] number_strings = cleaned_list_string.Split(','); - int[] numbers = new int[number_strings.Length]; - for (int j = 0; j < number_strings.Length; j++) - { - numbers[j] = int.Parse(number_strings[j]); - } - data_list.Add(numbers); - } - return data_list; - } } } diff --git a/src/TensorFlowNET.Keras/Utils/data_utils.cs b/src/TensorFlowNET.Keras/Utils/data_utils.cs index 5b84c601f..16b121b07 100644 --- a/src/TensorFlowNET.Keras/Utils/data_utils.cs +++ b/src/TensorFlowNET.Keras/Utils/data_utils.cs @@ -39,5 +39,52 @@ public static string get_file(string fname, string origin, return datadir; } + + public static (NDArray, NDArray) _remove_long_seq(int maxlen, NDArray seq, NDArray label) + { + /*Removes sequences that exceed the maximum length. + + Args: + maxlen: Int, maximum length of the output sequences. + seq: List of lists, where each sublist is a sequence. + label: List where each element is an integer. + + Returns: + new_seq, new_label: shortened lists for `seq` and `label`. + + */ + List new_seq = new List(); + List new_label = new List(); + + for (var i = 0; i < seq.shape[0]; i++) + { + if (maxlen < seq.shape[1] && seq[i][maxlen] != 0) + continue; + int[] sentence = new int[maxlen]; + for (var j = 0; j < maxlen && j < seq.shape[1]; j++) + { + sentence[j] = seq[i, j]; + } + new_seq.Add(sentence); + new_label.Add(label[i]); + } + + int[,] new_seq_array = new int[new_seq.Count, maxlen]; + int[] new_label_array = new int[new_label.Count]; + + for (var i = 0; i < new_seq.Count; i++) + { + for (var j = 0; j < maxlen; j++) + { + new_seq_array[i, j] = new_seq[i][j]; + } + } + + for (var i = 0; i < new_label.Count; i++) + { + new_label_array[i] = new_label[i]; + } + return (new_seq_array, new_label_array); + } } } diff --git a/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs b/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs index db6252efc..251eeff90 100644 --- a/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs +++ b/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs @@ -1,6 +1,8 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using System; +using System.Collections.Generic; using System.Linq; +using Tensorflow.NumPy; using static Tensorflow.Binding; using static Tensorflow.KerasApi; @@ -207,10 +209,28 @@ public void GetData() var y_train = dataset.Train.Item2; var x_val = dataset.Test.Item1; var y_val = dataset.Test.Item2; - print(len(x_train) + "Training sequences"); - print(len(x_val) + "Validation sequences"); - //x_train = keras.preprocessing.sequence.pad_sequences((IEnumerable)x_train, maxlen: maxlen); - //x_val = keras.preprocessing.sequence.pad_sequences((IEnumerable)x_val, maxlen: maxlen); + + x_train = keras.preprocessing.sequence.pad_sequences(RemoveZeros(x_train), maxlen: maxlen); + x_val = keras.preprocessing.sequence.pad_sequences(RemoveZeros(x_val), maxlen: maxlen); + print(len(x_train) + " Training sequences"); + print(len(x_val) + " Validation sequences"); + } + IEnumerable RemoveZeros(NDArray data) + { + List new_data = new List(); + for (var i = 0; i < data.shape[0]; i++) + { + List new_array = new List(); + for (var j = 0; j < data.shape[1]; j++) + { + if (data[i][j] == 0) + break; + else + new_array.Add((int)data[i][j]); + } + new_data.Add(new_array.ToArray()); + } + return new_data; } } } From f57a6fe6ed006f79511f4cc9550eeda312b11e98 Mon Sep 17 00:00:00 2001 From: lingbai-kong Date: Sat, 9 Sep 2023 18:31:46 +0800 Subject: [PATCH 32/98] optimize the time complexity of Imdb dataset loader --- src/TensorFlowNET.Keras/Datasets/Imdb.cs | 101 ++++++++++-------- src/TensorFlowNET.Keras/Utils/data_utils.cs | 16 +-- .../Dataset/DatasetTest.cs | 11 +- 3 files changed, 71 insertions(+), 57 deletions(-) diff --git a/src/TensorFlowNET.Keras/Datasets/Imdb.cs b/src/TensorFlowNET.Keras/Datasets/Imdb.cs index 0266b48bd..49fc79251 100644 --- a/src/TensorFlowNET.Keras/Datasets/Imdb.cs +++ b/src/TensorFlowNET.Keras/Datasets/Imdb.cs @@ -94,8 +94,6 @@ public DatasetPass load_data( var fileBytes = File.ReadAllBytes(path); var (x_train, x_test) = LoadX(fileBytes); var (labels_train, labels_test) = LoadY(fileBytes); - x_test.astype(np.int32); - labels_test.astype(np.int32); var indices = np.arange(len(x_train)); np.random.shuffle(indices, seed); @@ -107,67 +105,80 @@ public DatasetPass load_data( x_test = x_test[indices]; labels_test = labels_test[indices]; + var x_train_array = (int[,])x_train.ToMultiDimArray(); + var x_test_array = (int[,])x_test.ToMultiDimArray(); + var labels_train_array = (long[])labels_train.ToArray(); + var labels_test_array = (long[])labels_test.ToArray(); + if (start_char != null) { - int[,] new_x_train = new int[x_train.shape[0], x_train.shape[1] + 1]; - for (var i = 0; i < x_train.shape[0]; i++) + int[,] new_x_train_array = new int[x_train_array.GetLength(0), x_train_array.GetLength(1) + 1]; + for (var i = 0; i < x_train_array.GetLength(0); i++) { - new_x_train[i, 0] = (int)start_char; - for (var j = 0; j < x_train.shape[1]; j++) + new_x_train_array[i, 0] = (int)start_char; + for (var j = 0; j < x_train_array.GetLength(1); j++) { - new_x_train[i, j + 1] = x_train[i][j]; + if (x_train_array[i, j] == 0) + break; + new_x_train_array[i, j + 1] = x_train_array[i, j]; } } - int[,] new_x_test = new int[x_test.shape[0], x_test.shape[1] + 1]; - for (var i = 0; i < x_test.shape[0]; i++) + int[,] new_x_test_array = new int[x_test_array.GetLength(0), x_test_array.GetLength(1) + 1]; + for (var i = 0; i < x_test_array.GetLength(0); i++) { - new_x_test[i, 0] = (int)start_char; - for (var j = 0; j < x_test.shape[1]; j++) + new_x_test_array[i, 0] = (int)start_char; + for (var j = 0; j < x_test_array.GetLength(1); j++) { - new_x_test[i, j + 1] = x_test[i][j]; + if (x_test_array[i, j] == 0) + break; + new_x_test_array[i, j + 1] = x_test_array[i, j]; } } - x_train = new NDArray(new_x_train); - x_test = new NDArray(new_x_test); + x_train_array = new_x_train_array; + x_test_array = new_x_test_array; } else if (index_from != 0) { - for (var i = 0; i < x_train.shape[0]; i++) + for (var i = 0; i < x_train_array.GetLength(0); i++) { - for (var j = 0; j < x_train.shape[1]; j++) + for (var j = 0; j < x_train_array.GetLength(1); j++) { - if (x_train[i, j] != 0) - x_train[i, j] += index_from; + if (x_train_array[i, j] == 0) + break; + x_train_array[i, j] += index_from; } } - for (var i = 0; i < x_test.shape[0]; i++) + for (var i = 0; i < x_test_array.GetLength(0); i++) { - for (var j = 0; j < x_test.shape[1]; j++) + for (var j = 0; j < x_test_array.GetLength(1); j++) { - if (x_test[i, j] != 0) - x_test[i, j] += index_from; + if (x_test_array[i, j] == 0) + break; + x_test[i, j] += index_from; } } } - if (maxlen != null) + if (maxlen == null) { - (x_train, labels_train) = data_utils._remove_long_seq((int)maxlen, x_train, labels_train); - (x_test, labels_test) = data_utils._remove_long_seq((int)maxlen, x_test, labels_test); - if (x_train.size == 0 || x_test.size == 0) - throw new ValueError("After filtering for sequences shorter than maxlen=" + - $"{maxlen}, no sequence was kept. Increase maxlen."); + maxlen = max(x_train_array.GetLength(1), x_test_array.GetLength(1)); } + (x_train, labels_train) = data_utils._remove_long_seq((int)maxlen, x_train_array, labels_train_array); + (x_test, labels_test) = data_utils._remove_long_seq((int)maxlen, x_test_array, labels_test_array); + if (x_train.size == 0 || x_test.size == 0) + throw new ValueError("After filtering for sequences shorter than maxlen=" + + $"{maxlen}, no sequence was kept. Increase maxlen."); var xs = np.concatenate(new[] { x_train, x_test }); var labels = np.concatenate(new[] { labels_train, labels_test }); + var xs_array = (int[,])xs.ToMultiDimArray(); - if(num_words == null) + if (num_words == null) { num_words = 0; - for (var i = 0; i < xs.shape[0]; i++) - for (var j = 0; j < xs.shape[1]; j++) - num_words = max((int)num_words, (int)xs[i][j]); + for (var i = 0; i < xs_array.GetLength(0); i++) + for (var j = 0; j < xs_array.GetLength(1); j++) + num_words = max((int)num_words, (int)xs_array[i, j]); } // by convention, use 2 as OOV word @@ -175,32 +186,32 @@ public DatasetPass load_data( // 0 (padding), 1 (start), 2 (OOV) if (oov_char != null) { - int[,] new_xs = new int[xs.shape[0], xs.shape[1]]; - for(var i = 0; i < xs.shape[0]; i++) + int[,] new_xs_array = new int[xs_array.GetLength(0), xs_array.GetLength(1)]; + for (var i = 0; i < xs_array.GetLength(0); i++) { - for(var j = 0; j < xs.shape[1]; j++) + for (var j = 0; j < xs_array.GetLength(1); j++) { - if ((int)xs[i][j] == 0 || skip_top <= (int)xs[i][j] && (int)xs[i][j] < num_words) - new_xs[i, j] = (int)xs[i][j]; + if (xs_array[i, j] == 0 || skip_top <= xs_array[i, j] && xs_array[i, j] < num_words) + new_xs_array[i, j] = xs_array[i, j]; else - new_xs[i, j] = (int)oov_char; + new_xs_array[i, j] = (int)oov_char; } } - xs = new NDArray(new_xs); + xs = new NDArray(new_xs_array); } else { - int[,] new_xs = new int[xs.shape[0], xs.shape[1]]; - for (var i = 0; i < xs.shape[0]; i++) + int[,] new_xs_array = new int[xs_array.GetLength(0), xs_array.GetLength(1)]; + for (var i = 0; i < xs_array.GetLength(0); i++) { int k = 0; - for (var j = 0; j < xs.shape[1]; j++) + for (var j = 0; j < xs_array.GetLength(1); j++) { - if ((int)xs[i][j] == 0 || skip_top <= (int)xs[i][j] && (int)xs[i][j] < num_words) - new_xs[i, k++] = (int)xs[i][j]; + if (xs_array[i, j] == 0 || skip_top <= xs_array[i, j] && xs_array[i, j] < num_words) + new_xs_array[i, k++] = xs_array[i, j]; } } - xs = new NDArray(new_xs); + xs = new NDArray(new_xs_array); } var idx = len(x_train); diff --git a/src/TensorFlowNET.Keras/Utils/data_utils.cs b/src/TensorFlowNET.Keras/Utils/data_utils.cs index 16b121b07..57ae76695 100644 --- a/src/TensorFlowNET.Keras/Utils/data_utils.cs +++ b/src/TensorFlowNET.Keras/Utils/data_utils.cs @@ -54,23 +54,25 @@ public static (NDArray, NDArray) _remove_long_seq(int maxlen, NDArray seq, NDArr */ List new_seq = new List(); - List new_label = new List(); + List new_label = new List(); - for (var i = 0; i < seq.shape[0]; i++) + var seq_array = (int[,])seq.ToMultiDimArray(); + var label_array = (long[])label.ToArray(); + for (var i = 0; i < seq_array.GetLength(0); i++) { - if (maxlen < seq.shape[1] && seq[i][maxlen] != 0) + if (maxlen < seq_array.GetLength(1) && seq_array[i,maxlen] != 0) continue; int[] sentence = new int[maxlen]; - for (var j = 0; j < maxlen && j < seq.shape[1]; j++) + for (var j = 0; j < maxlen && j < seq_array.GetLength(1); j++) { - sentence[j] = seq[i, j]; + sentence[j] = seq_array[i, j]; } new_seq.Add(sentence); - new_label.Add(label[i]); + new_label.Add(label_array[i]); } int[,] new_seq_array = new int[new_seq.Count, maxlen]; - int[] new_label_array = new int[new_label.Count]; + long[] new_label_array = new long[new_label.Count]; for (var i = 0; i < new_seq.Count; i++) { diff --git a/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs b/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs index 251eeff90..183544ab6 100644 --- a/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs +++ b/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs @@ -204,7 +204,7 @@ public void GetData() { var vocab_size = 20000; // Only consider the top 20k words var maxlen = 200; // Only consider the first 200 words of each movie review - var dataset = keras.datasets.imdb.load_data(num_words: vocab_size); + var dataset = keras.datasets.imdb.load_data(num_words: vocab_size, maxlen: maxlen); var x_train = dataset.Train.Item1; var y_train = dataset.Train.Item2; var x_val = dataset.Test.Item1; @@ -217,16 +217,17 @@ public void GetData() } IEnumerable RemoveZeros(NDArray data) { + var data_array = (int[,])data.ToMultiDimArray(); List new_data = new List(); - for (var i = 0; i < data.shape[0]; i++) + for (var i = 0; i < data_array.GetLength(0); i++) { List new_array = new List(); - for (var j = 0; j < data.shape[1]; j++) + for (var j = 0; j < data_array.GetLength(1); j++) { - if (data[i][j] == 0) + if (data_array[i, j] == 0) break; else - new_array.Add((int)data[i][j]); + new_array.Add(data_array[i, j]); } new_data.Add(new_array.ToArray()); } From 114282885589956a29d7bcd015f55e966cb12532 Mon Sep 17 00:00:00 2001 From: Asaf Agami Date: Sun, 10 Sep 2023 18:09:38 +0300 Subject: [PATCH 33/98] fix: model does not stop on stop_training == true --- src/TensorFlowNET.Keras/Engine/Model.Fit.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs index de57f19ae..d6f89d8be 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs @@ -224,6 +224,10 @@ History FitInternal(DataHandler data_handler, int epochs, int validation_step, i GC.Collect(); GC.WaitForPendingFinalizers(); + if (stop_training) + { + break; + } } return callbacks.History; @@ -283,6 +287,10 @@ History FitInternal(DataHandler data_handler, int epochs, int verbose, List Date: Wed, 13 Sep 2023 17:18:43 +0000 Subject: [PATCH 34/98] cached_session for graph tests --- .../ControlFlowTest/WhileContextTestCase.cs | 3 +- .../GradientTest/GradientTest.cs | 21 ++- .../PythonTest.cs | 148 +++++++++++++++++- 3 files changed, 156 insertions(+), 16 deletions(-) diff --git a/test/TensorFlowNET.Graph.UnitTest/ControlFlowTest/WhileContextTestCase.cs b/test/TensorFlowNET.Graph.UnitTest/ControlFlowTest/WhileContextTestCase.cs index c637cf858..4dee61337 100644 --- a/test/TensorFlowNET.Graph.UnitTest/ControlFlowTest/WhileContextTestCase.cs +++ b/test/TensorFlowNET.Graph.UnitTest/ControlFlowTest/WhileContextTestCase.cs @@ -1,5 +1,6 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using System; +using System.Linq; using Tensorflow; using static Tensorflow.Binding; @@ -29,7 +30,7 @@ private void _testWhileContextHelper(int maximum_iterations) var b = new Func(x => math_ops.add(x, 1, name: "c")); //control_flow_ops.while_loop( // c, b, i , maximum_iterations: tf.constant(maximum_iterations)); - foreach (Operation op in sess.graph.get_operations()) + foreach (Operation op in sess.Single().graph.get_operations()) { var control_flow_context = op._get_control_flow_context(); /*if (control_flow_context != null) diff --git a/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs b/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs index f240817b4..37bc646dd 100644 --- a/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs +++ b/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs @@ -388,22 +388,19 @@ public void testBoundaryStop() } - [Ignore("TODO")] [TestMethod] public void testBoundaryContinue() { - //@test_util.run_v1_only("b/120545219") - //def testBoundaryContinue(self): - // # Test that we differentiate both 'x' and 'y' correctly when x is a - // # predecessor of y. - // with self.cached_session(): - // x = constant(1.0) - // y = x * 2.0 - // z = y * 3.0 - // grads = gradients.gradients(z, [x, y]) - // self.assertTrue(all(x is not None for x in grads)) - // self.assertEqual(6.0, grads[0].eval()) + // Test that we differentiate both 'x' and 'y' correctly when x is a + // predecessor of y. + self.cached_session(); + var x = tf.constant(1.0); + var y = x * 2.0; + var z = y * 3.0; + var grads = tf.gradients(z, new[] { x, y }); + self.assertTrue(all(grads.Select(x => x != null))); + self.assertEqual(6.0, grads[0].eval()); } [Ignore("TODO")] diff --git a/test/TensorFlowNET.Graph.UnitTest/PythonTest.cs b/test/TensorFlowNET.Graph.UnitTest/PythonTest.cs index 513791933..90abc0cc9 100644 --- a/test/TensorFlowNET.Graph.UnitTest/PythonTest.cs +++ b/test/TensorFlowNET.Graph.UnitTest/PythonTest.cs @@ -6,6 +6,8 @@ using System.Linq; using Tensorflow; using static Tensorflow.Binding; +using OneOf.Types; +using System.Collections.Generic; namespace TensorFlowNET.UnitTest { @@ -139,6 +141,21 @@ public void assertProtoEquals(object toProto, object o) #region tensor evaluation and test session + private Session _cached_session = null; + private Graph _cached_graph = null; + private object _cached_config = null; + private bool _cached_force_gpu = false; + + private void _ClearCachedSession() + { + if (self._cached_session != null) + { + self._cached_session.Dispose(); + self._cached_session = null; + } + } + + //protected object _eval_helper(Tensor[] tensors) //{ // if (tensors == null) @@ -203,10 +220,57 @@ public T evaluate(Tensor tensor) } } - - public Session cached_session() + ///Returns a TensorFlow Session for use in executing tests. + public IEnumerable cached_session( + Graph graph = null, object config = null, bool use_gpu = false, bool force_gpu = false) { - throw new NotImplementedException(); + // This method behaves differently than self.session(): for performance reasons + // `cached_session` will by default reuse the same session within the same + // test.The session returned by this function will only be closed at the end + // of the test(in the TearDown function). + + // Use the `use_gpu` and `force_gpu` options to control where ops are run.If + // `force_gpu` is True, all ops are pinned to `/ device:GPU:0`. Otherwise, if + // `use_gpu` is True, TensorFlow tries to run as many ops on the GPU as + // possible.If both `force_gpu and `use_gpu` are False, all ops are pinned to + // the CPU. + + // Example: + // python + // class MyOperatorTest(test_util.TensorFlowTestCase) : + // def testMyOperator(self): + // with self.cached_session() as sess: + // valid_input = [1.0, 2.0, 3.0, 4.0, 5.0] + // result = MyOperator(valid_input).eval() + // self.assertEqual(result, [1.0, 2.0, 3.0, 5.0, 8.0] + // invalid_input = [-1.0, 2.0, 7.0] + // with self.assertRaisesOpError("negative input not supported"): + // MyOperator(invalid_input).eval() + + + // Args: + // graph: Optional graph to use during the returned session. + // config: An optional config_pb2.ConfigProto to use to configure the + // session. + // use_gpu: If True, attempt to run as many ops as possible on GPU. + // force_gpu: If True, pin all ops to `/device:GPU:0`. + + // Yields: + // A Session object that should be used as a context manager to surround + // the graph building and execution code in a test case. + + + // TODO: + // if context.executing_eagerly(): + // return self._eval_helper(tensors) + // else: + { + var sess = self._get_cached_session( + graph, config, force_gpu, crash_if_inconsistent_args: true); + var cached = self._constrain_devices_and_set_default(sess, use_gpu, force_gpu); + return cached; + + } } //Returns a TensorFlow Session for use in executing tests. @@ -254,6 +318,40 @@ public Session session(Graph graph = null, object config = null, bool use_gpu = return s.as_default(); } + private IEnumerable _constrain_devices_and_set_default(Session sess, bool use_gpu, bool force_gpu) + { + // Set the session and its graph to global default and constrain devices.""" + // if context.executing_eagerly(): + // yield None + // else: + { + sess.graph.as_default(); + sess.as_default(); + { + if (force_gpu) + { + // TODO: + + // Use the name of an actual device if one is detected, or + // '/device:GPU:0' otherwise + /* var gpu_name = gpu_device_name(); + if (!gpu_name) + gpu_name = "/device:GPU:0" + using (sess.graph.device(gpu_name)) { + yield return sess; + }*/ + yield return sess; + } + else if (use_gpu) + yield return sess; + else + using (sess.graph.device("/device:CPU:0")) + yield return sess; + } + + } + } + // See session() for details. private Session _create_session(Graph graph, object cfg, bool forceGpu) { @@ -298,6 +396,50 @@ private Session _create_session(Graph graph, object cfg, bool forceGpu) return new Session(graph);//, config = prepare_config(config)) } + private Session _get_cached_session( + Graph graph = null, + object config = null, + bool force_gpu = false, + bool crash_if_inconsistent_args = true) + { + // See cached_session() for documentation. + if (self._cached_session == null) + { + var sess = self._create_session(graph, config, force_gpu); + self._cached_session = sess; + self._cached_graph = graph; + self._cached_config = config; + self._cached_force_gpu = force_gpu; + return sess; + } else { + + if (crash_if_inconsistent_args && !self._cached_graph.Equals(graph)) + throw new ValueError(@"The graph used to get the cached session is + different than the one that was used to create the + session. Maybe create a new session with + self.session()"); + if (crash_if_inconsistent_args && !self._cached_config.Equals(config)) { + throw new ValueError(@"The config used to get the cached session is + different than the one that was used to create the + session. Maybe create a new session with + self.session()"); + } + if (crash_if_inconsistent_args && !self._cached_force_gpu.Equals(force_gpu)) { + throw new ValueError(@"The force_gpu value used to get the cached session is + different than the one that was used to create the + session. Maybe create a new session with + self.session()"); + } + return _cached_session; + } + } + + [TestCleanup] + public void Cleanup() + { + _ClearCachedSession(); + } + #endregion public void AssetSequenceEqual(T[] a, T[] b) From ae50fa93bac27f9c7c77b7a38289f20d78480b3a Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Thu, 14 Sep 2023 03:58:15 +0000 Subject: [PATCH 35/98] fix fleaky test boundary continue --- test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs b/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs index 37bc646dd..0b4d79bb7 100644 --- a/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs +++ b/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs @@ -394,7 +394,7 @@ public void testBoundaryContinue() // Test that we differentiate both 'x' and 'y' correctly when x is a // predecessor of y. - self.cached_session(); + var sess = self.cached_session().Single(); var x = tf.constant(1.0); var y = x * 2.0; var z = y * 3.0; From 9d71cad96ecb69cd83c2b113fc808b608fbd7875 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Thu, 14 Sep 2023 11:21:18 +0000 Subject: [PATCH 36/98] using and no IEnumerable --- .../ControlFlowTest/WhileContextTestCase.cs | 4 ++-- .../GradientTest/GradientTest.cs | 16 ++++++++------ .../PythonTest.cs | 22 +++++++++---------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/test/TensorFlowNET.Graph.UnitTest/ControlFlowTest/WhileContextTestCase.cs b/test/TensorFlowNET.Graph.UnitTest/ControlFlowTest/WhileContextTestCase.cs index 4dee61337..e93324f3e 100644 --- a/test/TensorFlowNET.Graph.UnitTest/ControlFlowTest/WhileContextTestCase.cs +++ b/test/TensorFlowNET.Graph.UnitTest/ControlFlowTest/WhileContextTestCase.cs @@ -24,13 +24,13 @@ public void SimpleWhileLoop() private void _testWhileContextHelper(int maximum_iterations) { // TODO: implement missing code dependencies - var sess = this.cached_session(); + using var sess = this.cached_session(); var i = constant_op.constant(0, name: "i"); var c = new Func(x => gen_math_ops.less(x, ops.convert_to_tensor(10), name: "c")); var b = new Func(x => math_ops.add(x, 1, name: "c")); //control_flow_ops.while_loop( // c, b, i , maximum_iterations: tf.constant(maximum_iterations)); - foreach (Operation op in sess.Single().graph.get_operations()) + foreach (Operation op in sess.graph.get_operations()) { var control_flow_context = op._get_control_flow_context(); /*if (control_flow_context != null) diff --git a/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs b/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs index 0b4d79bb7..099c11627 100644 --- a/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs +++ b/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs @@ -394,13 +394,15 @@ public void testBoundaryContinue() // Test that we differentiate both 'x' and 'y' correctly when x is a // predecessor of y. - var sess = self.cached_session().Single(); - var x = tf.constant(1.0); - var y = x * 2.0; - var z = y * 3.0; - var grads = tf.gradients(z, new[] { x, y }); - self.assertTrue(all(grads.Select(x => x != null))); - self.assertEqual(6.0, grads[0].eval()); + using (self.cached_session()) + { + var x = tf.constant(1.0); + var y = x * 2.0; + var z = y * 3.0; + var grads = tf.gradients(z, new[] { x, y }); + self.assertTrue(all(grads.Select(x => x != null))); + self.assertEqual(6.0, grads[0].eval()); + } } [Ignore("TODO")] diff --git a/test/TensorFlowNET.Graph.UnitTest/PythonTest.cs b/test/TensorFlowNET.Graph.UnitTest/PythonTest.cs index 90abc0cc9..ccf59f5ae 100644 --- a/test/TensorFlowNET.Graph.UnitTest/PythonTest.cs +++ b/test/TensorFlowNET.Graph.UnitTest/PythonTest.cs @@ -221,7 +221,7 @@ public T evaluate(Tensor tensor) } ///Returns a TensorFlow Session for use in executing tests. - public IEnumerable cached_session( + public Session cached_session( Graph graph = null, object config = null, bool use_gpu = false, bool force_gpu = false) { // This method behaves differently than self.session(): for performance reasons @@ -267,9 +267,8 @@ public IEnumerable cached_session( { var sess = self._get_cached_session( graph, config, force_gpu, crash_if_inconsistent_args: true); - var cached = self._constrain_devices_and_set_default(sess, use_gpu, force_gpu); - return cached; - + using var cached = self._constrain_devices_and_set_default(sess, use_gpu, force_gpu); + return cached; } } @@ -318,13 +317,12 @@ public Session session(Graph graph = null, object config = null, bool use_gpu = return s.as_default(); } - private IEnumerable _constrain_devices_and_set_default(Session sess, bool use_gpu, bool force_gpu) + private Session _constrain_devices_and_set_default(Session sess, bool use_gpu, bool force_gpu) { // Set the session and its graph to global default and constrain devices.""" - // if context.executing_eagerly(): - // yield None - // else: - { + if (tf.executing_eagerly()) + return null; + else { sess.graph.as_default(); sess.as_default(); { @@ -340,13 +338,13 @@ private IEnumerable _constrain_devices_and_set_default(Session sess, bo using (sess.graph.device(gpu_name)) { yield return sess; }*/ - yield return sess; + return sess; } else if (use_gpu) - yield return sess; + return sess; else using (sess.graph.device("/device:CPU:0")) - yield return sess; + return sess; } } From adef5bcdc518d879ca385d37fe17ce5b2a329c44 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Thu, 14 Sep 2023 15:37:16 +0000 Subject: [PATCH 37/98] gradient tests --- .../GradientTest/GradientTest.cs | 383 +++++++++++------- 1 file changed, 236 insertions(+), 147 deletions(-) diff --git a/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs b/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs index 099c11627..b0827f2ab 100644 --- a/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs +++ b/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs @@ -5,6 +5,7 @@ using System.Linq; using Tensorflow; using static Tensorflow.Binding; +using Tensorflow.Framework; namespace TensorFlowNET.UnitTest.Gradient { @@ -394,6 +395,8 @@ public void testBoundaryContinue() // Test that we differentiate both 'x' and 'y' correctly when x is a // predecessor of y. + //TODO: @test_util.run_v1_only("b/120545219") + using (self.cached_session()) { var x = tf.constant(1.0); @@ -402,66 +405,61 @@ public void testBoundaryContinue() var grads = tf.gradients(z, new[] { x, y }); self.assertTrue(all(grads.Select(x => x != null))); self.assertEqual(6.0, grads[0].eval()); - } + } } - [Ignore("TODO")] [TestMethod] public void testAggregationMethodAccumulateN() { + //TODO: @test_util.run_v1_only("b/120545219") - //@test_util.run_v1_only("b/120545219") - //def testAggregationMethodAccumulateN(self): - // with self.cached_session(): - // x = constant(1.0) - // y = x * 2.0 - // z = y + y + y + y + y + y + y + y + y + y - // grads = gradients.gradients( - // z, [x, y], - // aggregation_method=gradients.AggregationMethod. - // EXPERIMENTAL_ACCUMULATE_N) - // self.assertTrue(all(x is not None for x in grads)) - // self.assertEqual(20.0, grads[0].eval()) - // self.assertEqual(10.0, grads[1].eval()) - + using (self.cached_session()) + { + var x = tf.constant(1.0); + var y = x * 2.0; + var z = y + y + y + y + y + y + y + y + y + y; + var grads = tf.gradients(z, new[] { x, y }, + aggregation_method: AggregationMethod.EXPERIMENTAL_ACCUMULATE_N); + self.assertTrue(all(grads.Select(x => x != null))); + self.assertEqual(20.0, grads[0].eval()); + self.assertEqual(10.0, grads[1].eval()); + } } - [Ignore("TODO")] [TestMethod] public void testAggregationMethodAddN() { - //@test_util.run_v1_only("b/120545219") - //def testAggregationMethodAddN(self): - // with self.cached_session(): - // x = constant(1.0) - // y = x * 2.0 - // z = y + y + y + y + y + y + y + y + y + y - // grads = gradients.gradients( - // z, [x, y], aggregation_method=gradients.AggregationMethod.ADD_N) - // self.assertTrue(all(x is not None for x in grads)) - // self.assertEqual(20.0, grads[0].eval()) - // self.assertEqual(10.0, grads[1].eval()) - + //TODO: @test_util.run_v1_only("b/120545219") + using (self.cached_session()) + { + var x = tf.constant(1.0); + var y = x * 2.0; + var z = y + y + y + y + y + y + y + y + y + y; + var grads = tf.gradients(z, new[] { x, y }, + aggregation_method: AggregationMethod.ADD_N); + self.assertTrue(grads.All(x => x != null)); + self.assertEqual(20.0, grads[0].eval()); + self.assertEqual(10.0, grads[1].eval()); + } } - [Ignore("TODO")] [TestMethod] public void testAggregationMethodTree() { - //@test_util.run_v1_only("b/120545219") - //def testAggregationMethodTree(self): - // with self.cached_session(): - // x = constant(1.0) - // y = x * 2.0 - // z = y + y + y + y + y + y + y + y + y + y - // grads = gradients.gradients( - // z, [x, y], - // aggregation_method=gradients.AggregationMethod.EXPERIMENTAL_TREE) - // self.assertTrue(all(x is not None for x in grads)) - // self.assertEqual(20.0, grads[0].eval()) - // self.assertEqual(10.0, grads[1].eval()) + //TODO: @test_util.run_v1_only("b/120545219") + using (self.cached_session()) + { + var x = tf.constant(1.0); + var y = x * 2.0; + var z = y + y + y + y + y + y + y + y + y + y; + var grads = tf.gradients(z, new[] { x, y }, + aggregation_method: AggregationMethod.EXPERIMENTAL_TREE); + self.assertTrue(grads.All(x => x != null)); + self.assertEqual(20.0, grads[0].eval()); + self.assertEqual(10.0, grads[1].eval()); + } } [Ignore("TODO")] @@ -490,24 +488,32 @@ public void testNoGradientForStringOutputs() // self.assertTrue(isinstance(grads[0], ops.Tensor)) } - [Ignore("TODO")] [TestMethod] public void testSingletonIndexedSlices() { + tf.Graph().as_default(); + + var x = tf.placeholder(TF_DataType.TF_FLOAT); + var y = tf.identity(x); + var dy_indices = tf.placeholder(TF_DataType.TF_INT32); + var dy_values = tf.placeholder(TF_DataType.TF_FLOAT); + Tensor dy = new IndexedSlices(dy_values, dy_indices); + var dx = tf.gradients(new[] { y }, new[] { x }, grad_ys: new[] { dy })[0]; + // The IndexedSlices gradient of tf.identity is the identity map. + using (var sess = self.cached_session()) + { + var feed_dict = new FeedItem[] + { + ( x, new Tensor(new float[] { 1.0f }) ), + (dy_indices, new Tensor(new int[] { 0 })), + (dy_values, new Tensor(new float[] { 2.0f })) + }; + var result = sess.run(new[] { dx, dy }, feed_dict); + var vdx = result[0]; + var vdy = result[1]; + self.assertEqual(vdx, vdy); + } - //def testSingletonIndexedSlices(self): - // with ops.Graph().as_default(): - // x = array_ops.placeholder(dtypes.float32) - // y = array_ops.identity(x) - // dy = ops.IndexedSlices( - // array_ops.placeholder(dtypes.float32), - // array_ops.placeholder(dtypes.int32)) - // dx, = gradients.gradients(y, x, grad_ys=dy) - // # The IndexedSlices gradient of tf.identity is the identity map. - // with self.cached_session() as sess: - // vdx, vdy = sess.run( - // [dx, dy], feed_dict={x: [1.0], dy.indices: [0], dy.values: [2.0]}) - // self.assertEqual(vdx, vdy) } [Ignore("TODO")] @@ -575,26 +581,25 @@ public void testVariableRefGradient() // self.assertIsNotNone(gradient) } - [Ignore("TODO")] [TestMethod] public void testDependentYs() { - //@test_util.run_v1_only("b/120545219") - //def testDependentYs(self): - // with self.cached_session(): - // x = constant_op.constant(3.0) - // y = math_ops.square(x) - // y1 = math_ops.square(y) - // y2 = math_ops.square(y1) - // g = gradients.gradients([y, y2], x) - // self.assertAllClose(17502.0, g[0].eval()) - // g = gradients.gradients(y + y2, x) - // self.assertAllClose(17502.0, g[0].eval()) - // z = array_ops.identity(y) - // z2 = array_ops.identity(y2) - // g = gradients.gradients([z, z2], x) - // self.assertAllClose(17502.0, g[0].eval()) - + //TODO: @test_util.run_v1_only("b/120545219") + using (self.cached_session()) + { + var x = constant_op.constant(3.0); + var y = math_ops.square(x); + var y1 = math_ops.square(y); + var y2 = math_ops.square(y1); + var g = tf.gradients(new[] { y, y2 }, new[] { x }); + self.assertAllClose(17502.0, g[0].eval()); + g = tf.gradients(y + y2, x); + self.assertAllClose(17502.0, g[0].eval()); + var z = array_ops.identity(y); + var z2 = array_ops.identity(y2); + g = tf.gradients(new[] { z, z2 }, new[] { x }); + self.assertAllClose(17502.0, g[0].eval()); + } } [Ignore("TODO")] @@ -602,75 +607,152 @@ public void testDependentYs() public void testPartialDerivatives() { - //@test_util.run_v1_only("b/120545219") - //def testPartialDerivatives(self): - // with self.cached_session(): - // x = constant_op.constant(1.) - // y = 2 * x - // z = x + y - // totalg = gradients.gradients(z, [x, y]) - // self.assertEqual([3.0, 1.0], [g.eval() for g in totalg]) - // partialg = gradients.gradients(z, [x, y], stop_gradients=[x, y]) - // self.assertEqual([1.0, 1.0], [g.eval() for g in partialg]) + //TODO: @test_util.run_v1_only("b/120545219") + using (self.cached_session()) + { + var x = tf.constant(1.0); + var y = 2 * x; + var z = x + y; + var totalg = tf.gradients(z, new[] { x, y }); + self.assertEqual(new[] { 3.0, 1.0 }, totalg.Select(g => g.eval())); + var partialg = tf.gradients(z, new[] { x, y }, stop_gradients: new[] { x, y }); + self.assertEqual(new[] { 1.0, 1.0 }, partialg.Select(g => g.eval())); + } } - [Ignore("TODO")] + // TODO: remove when np.testing.assert_allclose(a, b) is implemented + private class CollectionComparer : System.Collections.IComparer + { + private readonly double _epsilon = 1e-07; + + public int Compare(object x, object y) + { + var a = (double)x; + var b = (double)y; + + double delta = Math.Abs(a - b); + if (delta < _epsilon) + { + return 0; + } + return a.CompareTo(b); + } + } + + private struct Case + { + public Tensor[] grad1; + public Tensor[] grad2; + public string constants; + public string variables; + } + + [Ignore("FIXME")] [TestMethod] public void testStopGradients() { + + //TODO: @test_util.run_v1_only("b/120545219") + Dictionary makeGraph(RandomizedImpl rng, string stop_gradients) + { + Tensor functionOf(Tensor[] xs, int k) + { + var shape = new Shape(k, k); + // TODO: replace by DefaultIfEmpty() before Aggregate(). + if (!xs.Any()) + { + return rng.random(shape).astype(np.float32); + } + return xs.Select(x => gen_math_ops.mat_mul(rng.random(shape).astype(np.float32), x)) + .Aggregate((t1, t2) => t1 + t2) + + rng.random(shape).astype(np.float32); + } + var a = functionOf(Array.Empty(), 3); + if (stop_gradients.Contains('a')) a = array_ops.stop_gradient(a); + var b = functionOf(new Tensor[] { a }, 3); + if (stop_gradients.Contains('b')) b = array_ops.stop_gradient(b); + var c = functionOf(new Tensor[] { a, b }, 3); + if (stop_gradients.Contains('c')) c = array_ops.stop_gradient(c); + var d = functionOf(new Tensor[] { b, c }, 3); + if (stop_gradients.Contains('d')) d = array_ops.stop_gradient(d); - //@test_util.run_v1_only("b/120545219") - //def testStopGradients(self): - // def _MakeGraph(rng, stop_gradients=()): - // def _FunctionOf(xs, k=3): - // return ops.convert_to_tensor( - // sum(math_ops.matmul(rng.rand(k, k), x) for x in xs) - // + rng.rand(k, k)) - - // a = _FunctionOf([]) - // if "a" in stop_gradients: a = array_ops.stop_gradient(a) - // b = _FunctionOf([a]) - // if "b" in stop_gradients: b = array_ops.stop_gradient(b) - // c = _FunctionOf([a, b]) - // if "c" in stop_gradients: c = array_ops.stop_gradient(c) - // d = _FunctionOf([b, c]) - // if "d" in stop_gradients: d = array_ops.stop_gradient(d) - // return dict(a=a, b=b, c=c, d=d) - - // def _Gradients(ys, xs, **kwargs): - // dydxs = gradients.gradients(ys, xs, **kwargs) - // dydxs = [0. * x if dydx is None else dydx - // for x, dydx in zip(xs, dydxs)] - // return dydxs - // seed = np.random.randint(1000) - // cases = [] - // subsets = [""] + "a b c d ab ac ad bc bd cd abc abd acd bcd abcd".split() - // graph = _MakeGraph(np.random.RandomState(seed)) - // for constants in subsets: - // graph_with_stops = _MakeGraph(np.random.RandomState(seed), constants) - // for variables_ in subsets: - // # compute the gradient when stopped using tf.stop_gradients - // grad1 = _Gradients([graph_with_stops["d"]], - // [graph_with_stops[v] for v in variables_]) - // # compute the gradient when stopped using the stop_gradients kwarg - // grad2 = _Gradients([graph["d"]], - // [graph[v] for v in variables_], - // stop_gradients=[graph[v] for v in constants]) - // cases.append(dict(grad1=grad1, grad2=grad2, - // constants=constants, variables=variables_)) - - // # evaluate all tensors in one call to session.run for speed - // with self.cached_session() as sess: - // results = sess.run([(case["grad1"], case["grad2"]) for case in cases]) - - // for (npgrad1, npgrad2), case in zip(results, cases): - // for a, b in zip(npgrad1, npgrad2): - // np.testing.assert_allclose(a, b) + return new Dictionary + { + { 'a', a }, + { 'b', b }, + { 'c', c }, + { 'd', d } + }; + } + + Tensor[] gradients(Tensor[] ys, Tensor[] xs, Tensor[] stop_gradients = null) + { + var dydxs = tf.gradients(ys, xs, stop_gradients); + dydxs = dydxs.Select((dydx, i) => dydx == null ? xs[i] * 0 : dydx).ToArray(); + return dydxs; + } + + var seed = np.random.randint(1000); + // TODO: remove next line when np.random.RandomState implemented. + tf.set_random_seed(seed); + var cases = new List(); + // TODO: add "" case. + var subsets = new List { "" }.Concat("a b c d ab ac ad bc bd cd abc abd acd bcd abcd".Split()); + // TODO: pass np.random.RandomState(seed) instead of np.random + var graph = makeGraph(np.random, string.Empty); + foreach (var constants in subsets) + { + var graphWithStops = makeGraph(np.random, constants); + foreach (var variables_ in subsets) + { + // compute the gradient when stopped using tf.stop_gradients + var grad1 = gradients( + new[] { graphWithStops['d'] }, + variables_.ToCharArray().Select(v => graphWithStops[v]).ToArray() + ); + // compute the gradient when stopped using the stop_gradients from args + var grad2 = gradients( + new[] { graph['d'] }, + variables_.ToCharArray().Select(v => graph[v]).ToArray(), + constants.ToCharArray().Select(c => graph[c]).DefaultIfEmpty(null)?.ToArray() + ); + cases.Add(new Case + { + grad1 = grad1, + grad2 = grad2, + variables = variables_, + constants = constants, + }) ; + } + } + // evaluate all tensors in one call to session.run for speed + using (var sess = self.cached_session()) + { + var results = sess.run( + cases.Select(case_ => ( + case_.grad1, + case_.grad2 + )).ToArray() + ); + + foreach (var (result, case_) in results.Zip(cases)) + { + var npgrad1 = result[0]; + var npgrad2 = result[1]; + foreach (var (a, b) in npgrad1.Zip(npgrad2)) + { + // TODO: np.testing.assert_allclose(a, b); + CollectionAssert.AreEqual(a.ToArray(), b.ToArray(), new CollectionComparer()); + } + } + } } - [Ignore("TODO")] + + + [Ignore("TODO: Unconnected gradients are not implemented")] [TestMethod] public void testUnconnectedGradientsNoneUnconnectedGradients() { @@ -685,7 +767,7 @@ public void testUnconnectedGradientsNoneUnconnectedGradients() // self.assertIsNone(grad[0]) } - [Ignore("TODO")] + [Ignore("TODO: Unconnected gradients are not implemented")] [TestMethod] public void testUnconnectedGradientsZerosUnconnectedGradients() { @@ -699,15 +781,21 @@ public void testUnconnectedGradientsZerosUnconnectedGradients() // [y], [x], unconnected_gradients="zero") // with self.cached_session() as sess: // self.assertAllEqual([[0.0, 0.0], [0.0, 0.0]], self.evaluate(grads)[0]) + + // tf.Graph().as_default(); + // var x = tf.constant(1.0, shape: new long[] { 2, 2 }); + // var y = tf.constant(3.0, shape: new long[] { 3, 1 }); + // var grads = tf.gradients(new[] { y }, new[] { x }, unconnected_gradients: "zero"); + // using (self.cached_session()) + // { + // self.assertAllEqual(new[,] { { 0.0, 0.0 }, { 0.0, 0.0 } }, self.evaluate(grads)[0]); + // } } - [Ignore("TODO")] + [Ignore("TODO: Unconnected gradients are not implemented")] [TestMethod] public void testUnconnectedGradientsZeroConnectedGradients() { - - - //def testUnconnectedGradientsZeroConnectedGradients(self): // with ops.Graph().as_default(): // x = constant(1.0) @@ -716,9 +804,19 @@ public void testUnconnectedGradientsZeroConnectedGradients() // [y], [x], unconnected_gradients="zero") // with self.cached_session() as sess: // self.assertEquals(3.0, self.evaluate(grad)[0]) + + // tf.Graph().as_default(); + + // var x = tf.constant(1.0f); + // var y = x * 3.0f; + // var grad = tf.gradients(new [] { y }, new [] { x }, unconnected_gradients: "zero"); + // using (var sess = tf.Session()) + // { + // self.assertEquals(3.0, self.evaluate(grad)[0]); + // } } - [Ignore("TODO")] + [Ignore("TODO: Unconnected gradients are not implemented")] [TestMethod] public void testUnknownUnconnectedGradientsValueGiven() { @@ -729,15 +827,6 @@ public void testUnknownUnconnectedGradientsValueGiven() // with self.assertRaisesRegexp( // ValueError, "Unknown value for unconnected_gradients: 'nonsense'"): // gradients.gradients([y], [x], unconnected_gradients="nonsense") - } - - - - /* - - - - */ } } From a9dad3ce1114aa0b140472782d2ea4e36331107d Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Thu, 14 Sep 2023 15:47:39 +0000 Subject: [PATCH 38/98] fixme labels --- test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs b/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs index b0827f2ab..3ce6661cc 100644 --- a/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs +++ b/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs @@ -488,6 +488,7 @@ public void testNoGradientForStringOutputs() // self.assertTrue(isinstance(grads[0], ops.Tensor)) } + [Ignore("FIXME")] [TestMethod] public void testSingletonIndexedSlices() { From 628b2ce7366329f03390c4fffb9a8c779bb75663 Mon Sep 17 00:00:00 2001 From: lingbai-kong Date: Fri, 15 Sep 2023 20:36:52 +0800 Subject: [PATCH 39/98] optimize temporal complexity of Imdb dataset loader --- src/TensorFlowNET.Keras/Datasets/Imdb.cs | 48 +++++++++------------ src/TensorFlowNET.Keras/Utils/data_utils.cs | 14 +++--- 2 files changed, 27 insertions(+), 35 deletions(-) diff --git a/src/TensorFlowNET.Keras/Datasets/Imdb.cs b/src/TensorFlowNET.Keras/Datasets/Imdb.cs index 49fc79251..081c26cb9 100644 --- a/src/TensorFlowNET.Keras/Datasets/Imdb.cs +++ b/src/TensorFlowNET.Keras/Datasets/Imdb.cs @@ -116,23 +116,13 @@ public DatasetPass load_data( for (var i = 0; i < x_train_array.GetLength(0); i++) { new_x_train_array[i, 0] = (int)start_char; - for (var j = 0; j < x_train_array.GetLength(1); j++) - { - if (x_train_array[i, j] == 0) - break; - new_x_train_array[i, j + 1] = x_train_array[i, j]; - } + Array.Copy(x_train_array, i * x_train_array.GetLength(1), new_x_train_array, i * new_x_train_array.GetLength(1) + 1, x_train_array.GetLength(1)); } int[,] new_x_test_array = new int[x_test_array.GetLength(0), x_test_array.GetLength(1) + 1]; for (var i = 0; i < x_test_array.GetLength(0); i++) { new_x_test_array[i, 0] = (int)start_char; - for (var j = 0; j < x_test_array.GetLength(1); j++) - { - if (x_test_array[i, j] == 0) - break; - new_x_test_array[i, j + 1] = x_test_array[i, j]; - } + Array.Copy(x_test_array, i * x_test_array.GetLength(1), new_x_test_array, i * new_x_test_array.GetLength(1) + 1, x_test_array.GetLength(1)); } x_train_array = new_x_train_array; x_test_array = new_x_test_array; @@ -163,15 +153,19 @@ public DatasetPass load_data( { maxlen = max(x_train_array.GetLength(1), x_test_array.GetLength(1)); } - (x_train, labels_train) = data_utils._remove_long_seq((int)maxlen, x_train_array, labels_train_array); - (x_test, labels_test) = data_utils._remove_long_seq((int)maxlen, x_test_array, labels_test_array); - if (x_train.size == 0 || x_test.size == 0) + (x_train_array, labels_train_array) = data_utils._remove_long_seq((int)maxlen, x_train_array, labels_train_array); + (x_test_array, labels_test_array) = data_utils._remove_long_seq((int)maxlen, x_test_array, labels_test_array); + if (x_train_array.Length == 0 || x_test_array.Length == 0) throw new ValueError("After filtering for sequences shorter than maxlen=" + $"{maxlen}, no sequence was kept. Increase maxlen."); - var xs = np.concatenate(new[] { x_train, x_test }); - var labels = np.concatenate(new[] { labels_train, labels_test }); - var xs_array = (int[,])xs.ToMultiDimArray(); + int[,] xs_array = new int[x_train_array.GetLength(0) + x_test_array.GetLength(0), (int)maxlen]; + Array.Copy(x_train_array, xs_array, x_train_array.Length); + Array.Copy(x_test_array, 0, xs_array, x_train_array.Length, x_train_array.Length); + + long[] labels_array = new long[labels_train_array.Length + labels_test_array.Length]; + Array.Copy(labels_train_array, labels_array, labels_train_array.Length); + Array.Copy(labels_test_array, 0, labels_array, labels_train_array.Length, labels_test_array.Length); if (num_words == null) { @@ -197,7 +191,7 @@ public DatasetPass load_data( new_xs_array[i, j] = (int)oov_char; } } - xs = new NDArray(new_xs_array); + xs_array = new_xs_array; } else { @@ -211,19 +205,19 @@ public DatasetPass load_data( new_xs_array[i, k++] = xs_array[i, j]; } } - xs = new NDArray(new_xs_array); + xs_array = new_xs_array; } - var idx = len(x_train); - x_train = xs[$"0:{idx}"]; - x_test = xs[$"{idx}:"]; - var y_train = labels[$"0:{idx}"]; - var y_test = labels[$"{idx}:"]; + Array.Copy(xs_array, x_train_array, x_train_array.Length); + Array.Copy(xs_array, x_train_array.Length, x_test_array, 0, x_train_array.Length); + + Array.Copy(labels_array, labels_train_array, labels_train_array.Length); + Array.Copy(labels_array, labels_train_array.Length, labels_test_array, 0, labels_test_array.Length); return new DatasetPass { - Train = (x_train, y_train), - Test = (x_test, y_test) + Train = (x_train_array, labels_train_array), + Test = (x_test_array, labels_test_array) }; } diff --git a/src/TensorFlowNET.Keras/Utils/data_utils.cs b/src/TensorFlowNET.Keras/Utils/data_utils.cs index 57ae76695..e6db0ef72 100644 --- a/src/TensorFlowNET.Keras/Utils/data_utils.cs +++ b/src/TensorFlowNET.Keras/Utils/data_utils.cs @@ -40,7 +40,7 @@ public static string get_file(string fname, string origin, return datadir; } - public static (NDArray, NDArray) _remove_long_seq(int maxlen, NDArray seq, NDArray label) + public static (int[,], long[]) _remove_long_seq(int maxlen, int[,] seq, long[] label) { /*Removes sequences that exceed the maximum length. @@ -56,19 +56,17 @@ public static (NDArray, NDArray) _remove_long_seq(int maxlen, NDArray seq, NDArr List new_seq = new List(); List new_label = new List(); - var seq_array = (int[,])seq.ToMultiDimArray(); - var label_array = (long[])label.ToArray(); - for (var i = 0; i < seq_array.GetLength(0); i++) + for (var i = 0; i < seq.GetLength(0); i++) { - if (maxlen < seq_array.GetLength(1) && seq_array[i,maxlen] != 0) + if (maxlen < seq.GetLength(1) && seq[i, maxlen] != 0) continue; int[] sentence = new int[maxlen]; - for (var j = 0; j < maxlen && j < seq_array.GetLength(1); j++) + for (var j = 0; j < maxlen && j < seq.GetLength(1); j++) { - sentence[j] = seq_array[i, j]; + sentence[j] = seq[i, j]; } new_seq.Add(sentence); - new_label.Add(label_array[i]); + new_label.Add(label[i]); } int[,] new_seq_array = new int[new_seq.Count, maxlen]; From 57feb65dbc96fbe383d3dec1cee05bd3f34bb292 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Fri, 15 Sep 2023 14:57:48 +0000 Subject: [PATCH 40/98] comment IndexedSlices test --- .../GradientTest/GradientTest.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs b/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs index 3ce6661cc..fc2280051 100644 --- a/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs +++ b/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs @@ -488,17 +488,20 @@ public void testNoGradientForStringOutputs() // self.assertTrue(isinstance(grads[0], ops.Tensor)) } - [Ignore("FIXME")] + [Ignore("TODO: CompositeTensors are not supported yet.")] [TestMethod] public void testSingletonIndexedSlices() { tf.Graph().as_default(); + // TODO: uncomment when CompositeTensors are supported. + /* var x = tf.placeholder(TF_DataType.TF_FLOAT); var y = tf.identity(x); var dy_indices = tf.placeholder(TF_DataType.TF_INT32); var dy_values = tf.placeholder(TF_DataType.TF_FLOAT); - Tensor dy = new IndexedSlices(dy_values, dy_indices); + var dy = new IndexedSlices(dy_values, dy_indices); + var dx = tf.gradients(new[] { y }, new[] { x }, grad_ys: new[] { dy })[0]; // The IndexedSlices gradient of tf.identity is the identity map. using (var sess = self.cached_session()) @@ -514,6 +517,7 @@ public void testSingletonIndexedSlices() var vdy = result[1]; self.assertEqual(vdx, vdy); } + */ } From 56e389154cc3252888761b7bb7c931e4dbe88064 Mon Sep 17 00:00:00 2001 From: lingbai-kong Date: Mon, 18 Sep 2023 14:21:09 +0800 Subject: [PATCH 41/98] improve unpickler speed with BufferedStream --- .../NumPy/Implementation/NumPyImpl.Creation.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs b/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs index fa4ef0191..c0f9e695d 100644 --- a/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs +++ b/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs @@ -101,9 +101,10 @@ Array ReadValueMatrix(BinaryReader reader, Array matrix, int bytes, Type type, i Array ReadObjectMatrix(BinaryReader reader, Array matrix, int[] shape) { - Stream stream = reader.BaseStream; + Stream deflateStream = reader.BaseStream; + BufferedStream bufferedStream = new BufferedStream(deflateStream); var unpickler = new Unpickler(); - return (MultiArrayPickleWarpper)unpickler.load(stream); + return (MultiArrayPickleWarpper)unpickler.load(bufferedStream); } public (NDArray, NDArray) meshgrid(T[] array, bool copy = true, bool sparse = false) From 725ec1e55f83bae6e4745ddf0605bd15c40fbd92 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 18 Sep 2023 03:05:00 -0500 Subject: [PATCH 42/98] Optimize imdb.load_data --- src/TensorFlowNET.Keras/Datasets/Imdb.cs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/TensorFlowNET.Keras/Datasets/Imdb.cs b/src/TensorFlowNET.Keras/Datasets/Imdb.cs index 081c26cb9..1c9805189 100644 --- a/src/TensorFlowNET.Keras/Datasets/Imdb.cs +++ b/src/TensorFlowNET.Keras/Datasets/Imdb.cs @@ -180,10 +180,11 @@ public DatasetPass load_data( // 0 (padding), 1 (start), 2 (OOV) if (oov_char != null) { - int[,] new_xs_array = new int[xs_array.GetLength(0), xs_array.GetLength(1)]; - for (var i = 0; i < xs_array.GetLength(0); i++) + var (d1, d2) = (xs_array.GetLength(0), xs_array.GetLength(1)); + int[,] new_xs_array = new int[d1, d2]; + for (var i = 0; i < d1; i++) { - for (var j = 0; j < xs_array.GetLength(1); j++) + for (var j = 0; j < d2; j++) { if (xs_array[i, j] == 0 || skip_top <= xs_array[i, j] && xs_array[i, j] < num_words) new_xs_array[i, j] = xs_array[i, j]; @@ -195,11 +196,12 @@ public DatasetPass load_data( } else { - int[,] new_xs_array = new int[xs_array.GetLength(0), xs_array.GetLength(1)]; - for (var i = 0; i < xs_array.GetLength(0); i++) + var (d1, d2) = (xs_array.GetLength(0), xs_array.GetLength(1)); + int[,] new_xs_array = new int[d1, d2]; + for (var i = 0; i < d1; i++) { int k = 0; - for (var j = 0; j < xs_array.GetLength(1); j++) + for (var j = 0; j < d2; j++) { if (xs_array[i, j] == 0 || skip_top <= xs_array[i, j] && xs_array[i, j] < num_words) new_xs_array[i, k++] = xs_array[i, j]; From 9552d4cb7a51ea0081be027e15645dca11ea1239 Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Thu, 21 Sep 2023 21:54:49 +0800 Subject: [PATCH 43/98] feat: add np.less and np.greater binding --- src/TensorFlowNET.Core/NumPy/Numpy.Math.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/TensorFlowNET.Core/NumPy/Numpy.Math.cs b/src/TensorFlowNET.Core/NumPy/Numpy.Math.cs index 5bc97952b..2559638b3 100644 --- a/src/TensorFlowNET.Core/NumPy/Numpy.Math.cs +++ b/src/TensorFlowNET.Core/NumPy/Numpy.Math.cs @@ -85,5 +85,11 @@ public static NDArray dot(NDArray x1, NDArray x2, NDArray? axes = null, string? [AutoNumPy] public static NDArray add(NDArray x, NDArray y) => new NDArray(math_ops.add(x, y)); + + [AutoNumPy] + public static NDArray greater(NDArray x, NDArray y) => new NDArray(tf.greater(x, y)); + + [AutoNumPy] + public static NDArray less(NDArray x, NDArray y) => new NDArray(tf.less(x, y)); } } From f809f6eacee83336ac7971d018686b7ee8999198 Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Thu, 21 Sep 2023 21:56:22 +0800 Subject: [PATCH 44/98] fix: fix EarlyStopping --- .../Callbacks/Earlystopping.cs | 64 ++++++++++++------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/src/TensorFlowNET.Keras/Callbacks/Earlystopping.cs b/src/TensorFlowNET.Keras/Callbacks/Earlystopping.cs index 36993b637..a2a2ecfe2 100644 --- a/src/TensorFlowNET.Keras/Callbacks/Earlystopping.cs +++ b/src/TensorFlowNET.Keras/Callbacks/Earlystopping.cs @@ -19,8 +19,10 @@ public class EarlyStopping: ICallback string _monitor; string _mode; bool _restore_best_weights; - List? _best_weights; + List? _best_weights; CallbackParams _parameters; + Func _monitor_op; + public Dictionary>? history { get; set; } // user need to pass a CallbackParams to EarlyStopping, CallbackParams at least need the model public EarlyStopping(CallbackParams parameters,string monitor = "val_loss", float min_delta = 0f, int patience = 0, @@ -38,17 +40,49 @@ public EarlyStopping(CallbackParams parameters,string monitor = "val_loss", floa _min_delta = Math.Abs(min_delta); _restore_best_weights = restore_best_weights; _mode = mode; - if (mode != "auto" && mode != "min" && mode != "max") + + if (_mode != "auto" && _mode != "min" && _mode != "max") + { + Console.WriteLine($"EarlyStopping mode {_mode} is unknown, fallback to auto mode."); + _mode = "auto"; + } + + if (_mode == "min") + { + _monitor_op = np.less; + } + else if (_mode == "max") + { + _monitor_op = np.greater; + } + else + { + if (_monitor.EndsWith("acc") || _monitor.EndsWith("accuracy") || _monitor.EndsWith("auc")) + { + _monitor_op = np.greater; + } + else + { + _monitor_op = np.less; + } + } + + if (_monitor_op == np.greater) { - Console.WriteLine("EarlyStopping mode %s is unknown, fallback to auto mode.", mode); + _min_delta *= 1; + } + else + { + _min_delta *= -1; } } public void on_train_begin() { _wait = 0; _stopped_epoch = 0; + _best = _monitor_op == np.less ? (float)np.Inf : (float)-np.Inf; + _best_weights = null; _best_epoch = 0; - _best = (float)np.Inf; } public void on_epoch_begin(int epoch) @@ -74,7 +108,7 @@ public void on_epoch_end(int epoch, Dictionary epoch_logs) // Restore the weights after first epoch if no progress is ever made. if (_restore_best_weights && _best_weights == null) { - _best_weights = _parameters.Model.Weights; + _best_weights = _parameters.Model.get_weights(); } _wait += 1; @@ -83,7 +117,7 @@ public void on_epoch_end(int epoch, Dictionary epoch_logs) _best = current; _best_epoch = epoch; if (_restore_best_weights) - _best_weights = _parameters.Model.TrainableWeights; + _best_weights = _parameters.Model.get_weights(); // Only restart wait if we beat both the baseline and our previous best. if (_baseline == 0f || _is_improvement(current, _baseline)) _wait = 0; @@ -99,7 +133,7 @@ public void on_epoch_end(int epoch, Dictionary epoch_logs) { Console.WriteLine($"Restoring model weights from the end of the best epoch: {_best_epoch + 1}"); } - _parameters.Model.Weights = _best_weights; + _parameters.Model.set_weights(_best_weights); } } } @@ -131,21 +165,7 @@ float get_monitor_value(Dictionary logs) } public bool _is_improvement(float monitor_value, float reference_value) { - bool less_op = (monitor_value - _min_delta) < reference_value; - bool greater_op = (monitor_value - _min_delta) >= reference_value; - if (_mode == "min") - return less_op; - else if (_mode == "max") - return greater_op; - else - { - if (_monitor.EndsWith("acc") || _monitor.EndsWith("accuracy") || _monitor.EndsWith("auc")) - { - return greater_op; - } - else - return less_op; - } + return _monitor_op(monitor_value - _min_delta, reference_value); } public void on_test_end(Dictionary logs) From 9fb847991a1e45c0dbf40fd896b36b6d91953a24 Mon Sep 17 00:00:00 2001 From: lingbai-kong Date: Fri, 22 Sep 2023 18:34:08 +0800 Subject: [PATCH 45/98] fix: adjust imdb dataset loader for faster loading speed --- src/TensorFlowNET.Keras/Datasets/Imdb.cs | 29 ++++++++++++--------- src/TensorFlowNET.Keras/Utils/data_utils.cs | 8 +++--- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/TensorFlowNET.Keras/Datasets/Imdb.cs b/src/TensorFlowNET.Keras/Datasets/Imdb.cs index 1c9805189..4d6df913b 100644 --- a/src/TensorFlowNET.Keras/Datasets/Imdb.cs +++ b/src/TensorFlowNET.Keras/Datasets/Imdb.cs @@ -112,35 +112,39 @@ public DatasetPass load_data( if (start_char != null) { - int[,] new_x_train_array = new int[x_train_array.GetLength(0), x_train_array.GetLength(1) + 1]; - for (var i = 0; i < x_train_array.GetLength(0); i++) + var (d1, d2) = (x_train_array.GetLength(0), x_train_array.GetLength(1)); + int[,] new_x_train_array = new int[d1, d2 + 1]; + for (var i = 0; i < d1; i++) { new_x_train_array[i, 0] = (int)start_char; - Array.Copy(x_train_array, i * x_train_array.GetLength(1), new_x_train_array, i * new_x_train_array.GetLength(1) + 1, x_train_array.GetLength(1)); + Array.Copy(x_train_array, i * d2, new_x_train_array, i * (d2 + 1) + 1, d2); } - int[,] new_x_test_array = new int[x_test_array.GetLength(0), x_test_array.GetLength(1) + 1]; - for (var i = 0; i < x_test_array.GetLength(0); i++) + (d1, d2) = (x_test_array.GetLength(0), x_test_array.GetLength(1)); + int[,] new_x_test_array = new int[d1, d2 + 1]; + for (var i = 0; i < d1; i++) { new_x_test_array[i, 0] = (int)start_char; - Array.Copy(x_test_array, i * x_test_array.GetLength(1), new_x_test_array, i * new_x_test_array.GetLength(1) + 1, x_test_array.GetLength(1)); + Array.Copy(x_test_array, i * d2, new_x_test_array, i * (d2 + 1) + 1, d2); } x_train_array = new_x_train_array; x_test_array = new_x_test_array; } else if (index_from != 0) { - for (var i = 0; i < x_train_array.GetLength(0); i++) + var (d1, d2) = (x_train_array.GetLength(0), x_train_array.GetLength(1)); + for (var i = 0; i < d1; i++) { - for (var j = 0; j < x_train_array.GetLength(1); j++) + for (var j = 0; j < d2; j++) { if (x_train_array[i, j] == 0) break; x_train_array[i, j] += index_from; } } - for (var i = 0; i < x_test_array.GetLength(0); i++) + (d1, d2) = (x_test_array.GetLength(0), x_test_array.GetLength(1)); + for (var i = 0; i < d1; i++) { - for (var j = 0; j < x_test_array.GetLength(1); j++) + for (var j = 0; j < d2; j++) { if (x_test_array[i, j] == 0) break; @@ -169,9 +173,10 @@ public DatasetPass load_data( if (num_words == null) { + var (d1, d2) = (xs_array.GetLength(0), xs_array.GetLength(1)); num_words = 0; - for (var i = 0; i < xs_array.GetLength(0); i++) - for (var j = 0; j < xs_array.GetLength(1); j++) + for (var i = 0; i < d1; i++) + for (var j = 0; j < d2; j++) num_words = max((int)num_words, (int)xs_array[i, j]); } diff --git a/src/TensorFlowNET.Keras/Utils/data_utils.cs b/src/TensorFlowNET.Keras/Utils/data_utils.cs index e6db0ef72..b0bc15540 100644 --- a/src/TensorFlowNET.Keras/Utils/data_utils.cs +++ b/src/TensorFlowNET.Keras/Utils/data_utils.cs @@ -53,15 +53,17 @@ public static (int[,], long[]) _remove_long_seq(int maxlen, int[,] seq, long[] l new_seq, new_label: shortened lists for `seq` and `label`. */ + var nRow = seq.GetLength(0); + var nCol = seq.GetLength(1); List new_seq = new List(); List new_label = new List(); - for (var i = 0; i < seq.GetLength(0); i++) + for (var i = 0; i < nRow; i++) { - if (maxlen < seq.GetLength(1) && seq[i, maxlen] != 0) + if (maxlen < nCol && seq[i, maxlen] != 0) continue; int[] sentence = new int[maxlen]; - for (var j = 0; j < maxlen && j < seq.GetLength(1); j++) + for (var j = 0; j < maxlen && j < nCol; j++) { sentence[j] = seq[i, j]; } From eb4c1f4fb01bb02b7c7f87d5bee958bd9d4b0e42 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 23 Sep 2023 20:57:48 -0500 Subject: [PATCH 46/98] Release v0.110.4. --- src/TensorFlowNET.Core/Tensorflow.Binding.csproj | 9 +++++---- src/TensorFlowNET.Keras/Tensorflow.Keras.csproj | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index be714618d..85c41bd2a 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -5,7 +5,7 @@ Tensorflow.Binding Tensorflow 2.11.0 - 0.110.3 + 0.110.4 10.0 enable Haiping Chen, Eli Belash, Yaohui Liu, Meinrad Recheis @@ -25,7 +25,8 @@ https://tensorflownet.readthedocs.io tf.net 0.110.x and above are based on tensorflow native 2.11.0 * Support RNN, LSTM model. * Support Transformer model. - + * Added IMDB dataset. + tf.net 0.100.x and above are based on tensorflow native 2.10.0 * Eager Mode is added finally. @@ -43,7 +44,7 @@ https://tensorflownet.readthedocs.io tf.net 0.10x.x aligns with TensorFlow v2.10.x native library. tf.net 0.11x.x aligns with TensorFlow v2.11.x native library. - 0.110.3.0 + 0.110.4.0 LICENSE true packages @@ -174,7 +175,7 @@ https://tensorflownet.readthedocs.io - + diff --git a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj index 36d1bc1d4..a0ee22284 100644 --- a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj +++ b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj @@ -7,7 +7,7 @@ enable Tensorflow.Keras AnyCPU;x64 - 0.11.3 + 0.11.4 Haiping Chen Keras for .NET Apache 2.0, Haiping Chen since 2018 @@ -42,8 +42,8 @@ Keras is an API designed for human beings, not machines. Keras follows best prac Git False Open.snk - 0.11.3.0 - 0.11.3.0 + 0.11.4.0 + 0.11.4.0 LICENSE Debug;Release;GPU From 21210795d0fb7963c13fb99604b7e7e46df2443d Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 27 Sep 2023 13:16:28 +0000 Subject: [PATCH 47/98] gradient descent tests --- .../Variables/variables.py.cs | 7 +- .../GradientTest/GradientTest.cs | 2 - test/TensorFlowNET.UnitTest/PythonTest.cs | 178 +++++++++++++++++- .../Training/GradientDescentOptimizerTests.cs | 68 +++++++ 4 files changed, 250 insertions(+), 5 deletions(-) create mode 100644 test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs diff --git a/src/TensorFlowNET.Core/Variables/variables.py.cs b/src/TensorFlowNET.Core/Variables/variables.py.cs index 0c07e0243..f3ae248e6 100644 --- a/src/TensorFlowNET.Core/Variables/variables.py.cs +++ b/src/TensorFlowNET.Core/Variables/variables.py.cs @@ -72,7 +72,9 @@ public static List global_variables(string scope = null) public static Operation variables_initializer(IVariableV1[] var_list, string name = "init") { if (var_list.Length > 0) + { return control_flow_ops.group(var_list.Select(x => x.Initializer).ToArray(), name); + } else return gen_control_flow_ops.no_op(name: name); } @@ -155,7 +157,10 @@ public static Operation _safe_initial_value_from_op(string name, Operation op, D public static Tensor global_variables_initializer() { - throw new NotImplementedException(); + // if context.executing_eagerly(): + // return control_flow_ops.no_op(name = "global_variables_initializer") + var group = variables_initializer(global_variables().ToArray()); + return group; } } } diff --git a/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs b/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs index fc2280051..e2d6db912 100644 --- a/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs +++ b/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs @@ -776,8 +776,6 @@ public void testUnconnectedGradientsNoneUnconnectedGradients() [TestMethod] public void testUnconnectedGradientsZerosUnconnectedGradients() { - - //def testUnconnectedGradientsZerosUnconnectedGradients(self): // with ops.Graph().as_default(): // x = constant(1.0, shape=[2, 2]) diff --git a/test/TensorFlowNET.UnitTest/PythonTest.cs b/test/TensorFlowNET.UnitTest/PythonTest.cs index 50cc2b328..12fd72360 100644 --- a/test/TensorFlowNET.UnitTest/PythonTest.cs +++ b/test/TensorFlowNET.UnitTest/PythonTest.cs @@ -144,6 +144,37 @@ public void assertAllClose(double value, NDArray array2, double eps = 1e-5) Assert.IsTrue(np.allclose(array1, array2, rtol: eps)); } + private class CollectionComparer : System.Collections.IComparer + { + private readonly double _epsilon; + + public CollectionComparer(double eps = 1e-06) { + _epsilon = eps; + } + public int Compare(object x, object y) + { + var a = (double)x; + var b = (double)y; + + double delta = Math.Abs(a - b); + if (delta < _epsilon) + { + return 0; + } + return a.CompareTo(b); + } + } + + public void assertAllCloseAccordingToType( + T[] expected, + T[] given, + double eps = 1e-6, + float float_eps = 1e-6f) + { + // TODO: check if any of arguments is not double and change toletance + CollectionAssert.AreEqual(expected, given, new CollectionComparer(eps)); + } + public void assertProtoEquals(object toProto, object o) { throw new NotImplementedException(); @@ -153,6 +184,20 @@ public void assertProtoEquals(object toProto, object o) #region tensor evaluation and test session + private Session _cached_session = null; + private Graph _cached_graph = null; + private object _cached_config = null; + private bool _cached_force_gpu = false; + + private void _ClearCachedSession() + { + if (self._cached_session != null) + { + self._cached_session.Dispose(); + self._cached_session = null; + } + } + //protected object _eval_helper(Tensor[] tensors) //{ // if (tensors == null) @@ -218,9 +263,56 @@ public T evaluate(Tensor tensor) } - public Session cached_session() + ///Returns a TensorFlow Session for use in executing tests. + public Session cached_session( + Graph graph = null, object config = null, bool use_gpu = false, bool force_gpu = false) { - throw new NotImplementedException(); + // This method behaves differently than self.session(): for performance reasons + // `cached_session` will by default reuse the same session within the same + // test.The session returned by this function will only be closed at the end + // of the test(in the TearDown function). + + // Use the `use_gpu` and `force_gpu` options to control where ops are run.If + // `force_gpu` is True, all ops are pinned to `/ device:GPU:0`. Otherwise, if + // `use_gpu` is True, TensorFlow tries to run as many ops on the GPU as + // possible.If both `force_gpu and `use_gpu` are False, all ops are pinned to + // the CPU. + + // Example: + // python + // class MyOperatorTest(test_util.TensorFlowTestCase) : + // def testMyOperator(self): + // with self.cached_session() as sess: + // valid_input = [1.0, 2.0, 3.0, 4.0, 5.0] + // result = MyOperator(valid_input).eval() + // self.assertEqual(result, [1.0, 2.0, 3.0, 5.0, 8.0] + // invalid_input = [-1.0, 2.0, 7.0] + // with self.assertRaisesOpError("negative input not supported"): + // MyOperator(invalid_input).eval() + + + // Args: + // graph: Optional graph to use during the returned session. + // config: An optional config_pb2.ConfigProto to use to configure the + // session. + // use_gpu: If True, attempt to run as many ops as possible on GPU. + // force_gpu: If True, pin all ops to `/device:GPU:0`. + + // Yields: + // A Session object that should be used as a context manager to surround + // the graph building and execution code in a test case. + + + // TODO: + // if context.executing_eagerly(): + // return self._eval_helper(tensors) + // else: + { + var sess = self._get_cached_session( + graph, config, force_gpu, crash_if_inconsistent_args: true); + using var cached = self._constrain_devices_and_set_default(sess, use_gpu, force_gpu); + return cached; + } } //Returns a TensorFlow Session for use in executing tests. @@ -268,6 +360,40 @@ public Session session(Graph graph = null, object config = null, bool use_gpu = return s.as_default(); } + private Session _constrain_devices_and_set_default(Session sess, bool use_gpu, bool force_gpu) + { + // Set the session and its graph to global default and constrain devices.""" + if (tf.executing_eagerly()) + return null; + else + { + sess.graph.as_default(); + sess.as_default(); + { + if (force_gpu) + { + // TODO: + + // Use the name of an actual device if one is detected, or + // '/device:GPU:0' otherwise + /* var gpu_name = gpu_device_name(); + if (!gpu_name) + gpu_name = "/device:GPU:0" + using (sess.graph.device(gpu_name)) { + yield return sess; + }*/ + return sess; + } + else if (use_gpu) + return sess; + else + using (sess.graph.device("/device:CPU:0")) + return sess; + } + + } + } + // See session() for details. private Session _create_session(Graph graph, object cfg, bool forceGpu) { @@ -312,6 +438,54 @@ private Session _create_session(Graph graph, object cfg, bool forceGpu) return new Session(graph);//, config = prepare_config(config)) } + private Session _get_cached_session( + Graph graph = null, + object config = null, + bool force_gpu = false, + bool crash_if_inconsistent_args = true) + { + // See cached_session() for documentation. + if (self._cached_session == null) + { + var sess = self._create_session(graph, config, force_gpu); + self._cached_session = sess; + self._cached_graph = graph; + self._cached_config = config; + self._cached_force_gpu = force_gpu; + return sess; + } + else + { + + if (crash_if_inconsistent_args && !self._cached_graph.Equals(graph)) + throw new ValueError(@"The graph used to get the cached session is + different than the one that was used to create the + session. Maybe create a new session with + self.session()"); + if (crash_if_inconsistent_args && !self._cached_config.Equals(config)) + { + throw new ValueError(@"The config used to get the cached session is + different than the one that was used to create the + session. Maybe create a new session with + self.session()"); + } + if (crash_if_inconsistent_args && !self._cached_force_gpu.Equals(force_gpu)) + { + throw new ValueError(@"The force_gpu value used to get the cached session is + different than the one that was used to create the + session. Maybe create a new session with + self.session()"); + } + return _cached_session; + } + } + + [TestCleanup] + public void Cleanup() + { + _ClearCachedSession(); + } + #endregion public void AssetSequenceEqual(T[] a, T[] b) diff --git a/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs b/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs new file mode 100644 index 000000000..977544ae9 --- /dev/null +++ b/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs @@ -0,0 +1,68 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Linq; +using System.Runtime.Intrinsics.X86; +using System.Security.AccessControl; +using Tensorflow.NumPy; +using TensorFlowNET.UnitTest; +using static Tensorflow.Binding; + +namespace Tensorflow.Keras.UnitTest.Optimizers +{ + [TestClass] + public class GradientDescentOptimizerTest : PythonTest + { + private void TestBasicGeneric() where T : struct + { + var dtype = Type.GetTypeCode(typeof(T)) switch + { + TypeCode.Single => np.float32, + TypeCode.Double => np.float64, + _ => throw new NotImplementedException(), + }; + + // train.GradientDescentOptimizer is V1 only API. + tf.Graph().as_default(); + using (self.cached_session()) + { + var var0 = tf.Variable(new[] { 1.0, 2.0 }, dtype: dtype); + var var1 = tf.Variable(new[] { 3.0, 4.0 }, dtype: dtype); + var grads0 = tf.constant(new[] { 0.1, 0.1 }, dtype: dtype); + var grads1 = tf.constant(new[] { 0.01, 0.01 }, dtype: dtype); + var optimizer = tf.train.GradientDescentOptimizer(3.0f); + var grads_and_vars = new[] { + Tuple.Create(grads0, var0 as IVariableV1), + Tuple.Create(grads1, var1 as IVariableV1) + }; + var sgd_op = optimizer.apply_gradients(grads_and_vars); + + var global_variables = variables.global_variables_initializer(); + self.evaluate(global_variables); + // Fetch params to validate initial values + // TODO: use self.evaluate instead of self.evaluate + self.assertAllCloseAccordingToType(new double[] { 1.0, 2.0 }, self.evaluate(var0)); + self.assertAllCloseAccordingToType(new double[] { 3.0, 4.0 }, self.evaluate(var1)); + // Run 1 step of sgd + sgd_op.run(); + // Validate updated params + self.assertAllCloseAccordingToType( + new double[] { 1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1 }, + self.evaluate(var0)); + self.assertAllCloseAccordingToType( + new double[] { 3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01 }, + self.evaluate(var1)); + // TODO: self.assertEqual(0, len(optimizer.variables())); + } + } + + [TestMethod] + public void TestBasic() + { + //TODO: add np.half + TestBasicGeneric(); + TestBasicGeneric(); + } + + + } +} From 02bfb9af176c13e8c37fe42ce600f4600ab8938d Mon Sep 17 00:00:00 2001 From: Beacontownfc <19636977267@qq.com> Date: Thu, 28 Sep 2023 15:22:13 +0000 Subject: [PATCH 48/98] improve raggedtensor --- .../Operations/array_ops.cs | 13 +++++ .../Tensors/Ragged/RaggedTensor.cs | 33 +++++++++++ .../Tensors/Ragged/RowPartition.cs | 55 +++++++++++++++++++ .../ManagedAPI/RaggedTensorTest.cs | 26 +++++++++ 4 files changed, 127 insertions(+) create mode 100644 test/TensorFlowNET.UnitTest/ManagedAPI/RaggedTensorTest.cs diff --git a/src/TensorFlowNET.Core/Operations/array_ops.cs b/src/TensorFlowNET.Core/Operations/array_ops.cs index f80dcd2c4..fdc53cd7e 100644 --- a/src/TensorFlowNET.Core/Operations/array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/array_ops.cs @@ -1139,5 +1139,18 @@ public static Tensor placeholder(TF_DataType dtype, Shape shape = null, string n var _op = tf.OpDefLib._apply_op_helper("Placeholder", name: name, args: new { dtype, shape }); return _op.output; } + + public static int get_positive_axis(int axis, int ndims=-100, string axis_name="axis", string ndims_name= "ndims") + { + if(ndims != -100) + { + if (axis >= 0 && axis < ndims) return axis; + else if (-ndims <= axis && axis < 0) return axis + ndims; + else throw new ValueError($"{axis_name}={axis} out of bounds:expected {-ndims}<={axis_name}<{ndims}"); + + } else if(axis < 0) throw new ValueError($"{axis_name}={axis} may only be negative if {ndims_name} is statically known."); + return axis; + } + } } diff --git a/src/TensorFlowNET.Core/Tensors/Ragged/RaggedTensor.cs b/src/TensorFlowNET.Core/Tensors/Ragged/RaggedTensor.cs index 4f85e1081..0f09d4128 100644 --- a/src/TensorFlowNET.Core/Tensors/Ragged/RaggedTensor.cs +++ b/src/TensorFlowNET.Core/Tensors/Ragged/RaggedTensor.cs @@ -163,5 +163,38 @@ public static implicit operator RaggedTensor(Tensor tensor) { return tensor.Tag as RaggedTensor; } + public Tensor nrows(TF_DataType out_type, string name = null) + { + tf_with(ops.name_scope(name, "RaggedNRows"), scope => + { + return math_ops.cast(this._row_partition.nrows(), dtype: out_type); + }); + return null; + } + public RaggedTensor row_lengths(int axis=-1, string name=null) + { + if (axis == 0) return this._row_partition.nrows(); + if (axis == 1) return this._row_partition.row_lengths(); + var values = (RaggedTensor)this._values; + axis = array_ops.get_positive_axis( + axis, this.shape.rank, ndims_name: "rank(this)"); + if (axis == 0) return this.nrows(this._row_partition.GetDataType()); + else if (axis == 1) + { + var splits = this._row_partition.row_splits; + return splits[new Slice(start: 1)] - splits[new Slice(stop: -1)]; + + } + else if (this._values is RaggedTensor) + { + return values.row_lengths(axis - 1); + } + else + { + var shape = array_ops.shape(values, out_type: this._row_partition.GetDataType()); + return array_ops.ones(shape[new Slice(stop:axis - 1)], this._row_partition.GetDataType()) * + shape[axis - 1]; + } + } } } diff --git a/src/TensorFlowNET.Core/Tensors/Ragged/RowPartition.cs b/src/TensorFlowNET.Core/Tensors/Ragged/RowPartition.cs index 29dc525df..9e242ff38 100644 --- a/src/TensorFlowNET.Core/Tensors/Ragged/RowPartition.cs +++ b/src/TensorFlowNET.Core/Tensors/Ragged/RowPartition.cs @@ -14,10 +14,15 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using Serilog.Debugging; using System; +using System.Collections.Concurrent; using System.Collections.Generic; +//using System.ComponentModel.DataAnnotations; using System.Text; +using System.Xml.Linq; using Tensorflow.Framework; +using Tensorflow.NumPy; using static Tensorflow.Binding; namespace Tensorflow @@ -99,5 +104,55 @@ public static RowPartition from_row_splits(Tensor row_splits, return new RowPartition(row_splits); }); } + + public static RowPartition from_row_lengths(Tensor row_lengths, + bool validate=true, + TF_DataType dtype = TF_DataType.TF_INT32, + TF_DataType dtype_hint= TF_DataType.TF_INT32) + { + row_lengths = _convert_row_partition( + row_lengths, "row_lengths", dtype_hint: dtype_hint, dtype: dtype); + Tensor row_limits = math_ops.cumsum(row_lengths, tf.constant(-1)); + Tensor row_splits = array_ops.concat(new Tensor[] { tf.convert_to_tensor(np.array(new int[] { 0 }, TF_DataType.TF_INT64)), row_limits }, axis:0); + return new RowPartition(row_splits: row_splits, row_lengths: row_lengths); + } + + public static Tensor _convert_row_partition(Tensor partition, string name, TF_DataType dtype, + TF_DataType dtype_hint= TF_DataType.TF_INT64) + { + if (partition is NDArray && partition.GetDataType() == np.int32) partition = ops.convert_to_tensor(partition, name: name); + if (partition.GetDataType() != np.int32 && partition.GetDataType() != np.int64) throw new ValueError($"{name} must have dtype int32 or int64"); + return partition; + } + + public Tensor nrows() + { + /*Returns the number of rows created by this `RowPartition*/ + if (this._nrows != null) return this._nrows; + var nsplits = tensor_shape.dimension_at_index(this._row_splits.shape, 0); + if (nsplits == null) return array_ops.shape(this._row_splits, out_type: this.row_splits.dtype)[0] - 1; + else return constant_op.constant(nsplits.value - 1, dtype: this.row_splits.dtype); + } + + public Tensor row_lengths() + { + + if (this._row_splits != null) + { + int nrows_plus_one = tensor_shape.dimension_value(this._row_splits.shape[0]); + return tf.constant(nrows_plus_one - 1); + + } + if (this._row_lengths != null) + { + var nrows = tensor_shape.dimension_value(this._row_lengths.shape[0]); + return tf.constant(nrows); + } + if(this._nrows != null) + { + return tensor_util.constant_value(this._nrows); + } + return tf.constant(-1); + } } } diff --git a/test/TensorFlowNET.UnitTest/ManagedAPI/RaggedTensorTest.cs b/test/TensorFlowNET.UnitTest/ManagedAPI/RaggedTensorTest.cs new file mode 100644 index 000000000..7a3de882e --- /dev/null +++ b/test/TensorFlowNET.UnitTest/ManagedAPI/RaggedTensorTest.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Tensorflow; +using Tensorflow.NumPy; +using static Tensorflow.Binding; + +namespace TensorFlowNET.UnitTest.ManagedAPI +{ + public class RaggedTensorTest :EagerModeTestBase + { + [TestMethod] + public void Test_from_row_lengths() + { + var row_lengths = tf.convert_to_tensor(np.array(new int[] { 2, 0, 3, 1, 1 }, TF_DataType.TF_INT64)); + var rp = RowPartition.from_row_lengths(row_lengths, validate: false); + var rp_row_lengths = rp.row_lengths(); + var rp_nrows = rp.nrows(); + Assert.IsTrue(rp_nrows.ToArray()[0] == rp.nrows().ToArray()[0]); + + } + } +} From f5af07ce5efc938686c897db57f0a33ec371adec Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Mon, 2 Oct 2023 00:23:56 +0800 Subject: [PATCH 49/98] feat: add the implementation of sample_weight in model.fit --- .../Keras/ArgsDefinition/DataAdapterArgs.cs | 3 + .../Keras/ArgsDefinition/DataHandlerArgs.cs | 3 + src/TensorFlowNET.Core/Keras/Engine/IModel.cs | 11 +- src/TensorFlowNET.Core/Util/Data.cs | 66 +++++++++ .../Engine/DataAdapters/DataAdapter.cs | 59 ++++++++ .../Engine/DataAdapters/DataHandler.cs | 3 + .../Engine/DataAdapters/IDataAdapter.cs | 2 + .../DataAdapters/TensorLikeDataAdapter.cs | 7 +- .../Engine/LossesContainer.cs | 4 +- .../Engine/Model.Evaluate.cs | 19 ++- src/TensorFlowNET.Keras/Engine/Model.Fit.cs | 129 ++++++------------ src/TensorFlowNET.Keras/Engine/Model.Train.cs | 40 +++++- .../Layers/Rnn.Test.cs | 4 +- 13 files changed, 250 insertions(+), 100 deletions(-) create mode 100644 src/TensorFlowNET.Core/Util/Data.cs diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/DataAdapterArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/DataAdapterArgs.cs index 78882e82d..ba0332836 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/DataAdapterArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/DataAdapterArgs.cs @@ -1,5 +1,6 @@ using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; +using Tensorflow.NumPy; namespace Tensorflow.Keras.ArgsDefinition { @@ -16,5 +17,7 @@ public class DataAdapterArgs: IKerasConfig public int Worker { get; set; } public bool UseMultiprocessing { get; set; } public IModel Model { get; set; } + public Dictionary ClassWeight = null; + public NDArray SampleWeight = null; } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/DataHandlerArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/DataHandlerArgs.cs index 82530e950..72d0bb811 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/DataHandlerArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/DataHandlerArgs.cs @@ -1,5 +1,6 @@ using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; +using Tensorflow.NumPy; namespace Tensorflow.Keras.ArgsDefinition { @@ -18,5 +19,7 @@ public class DataHandlerArgs: IKerasConfig public bool UseMultiprocessing { get; set; } = false; public IModel Model { get; set; } public IVariableV1 StepsPerExecution { get; set; } + public Dictionary ClassWeight = null; + public NDArray SampleWeight = null; } } diff --git a/src/TensorFlowNET.Core/Keras/Engine/IModel.cs b/src/TensorFlowNET.Core/Keras/Engine/IModel.cs index 19f3df9ba..1840f88b9 100644 --- a/src/TensorFlowNET.Core/Keras/Engine/IModel.cs +++ b/src/TensorFlowNET.Core/Keras/Engine/IModel.cs @@ -3,6 +3,7 @@ using Tensorflow.Keras.Metrics; using Tensorflow.Keras.Saving; using Tensorflow.NumPy; +using Tensorflow.Util; namespace Tensorflow.Keras.Engine; @@ -22,8 +23,10 @@ ICallback fit(NDArray x, NDArray y, int verbose = 1, List callbacks = null, float validation_split = 0f, - (NDArray val_x, NDArray val_y)? validation_data = null, + ValidationDataPack validation_data = null, bool shuffle = true, + Dictionary class_weight = null, + NDArray sample_weight = null, int initial_epoch = 0, int max_queue_size = 10, int workers = 1, @@ -35,8 +38,10 @@ ICallback fit(IEnumerable x, NDArray y, int verbose = 1, List callbacks = null, float validation_split = 0f, - (IEnumerable val_x, NDArray val_y)? validation_data = null, + ValidationDataPack validation_data = null, bool shuffle = true, + Dictionary class_weight = null, + NDArray sample_weight = null, int initial_epoch = 0, int max_queue_size = 10, int workers = 1, @@ -63,6 +68,8 @@ void load_weights(string filepath, Dictionary evaluate(NDArray x, NDArray y, int batch_size = -1, int verbose = 1, + NDArray sample_weight = null, + int steps = -1, int max_queue_size = 10, int workers = 1, diff --git a/src/TensorFlowNET.Core/Util/Data.cs b/src/TensorFlowNET.Core/Util/Data.cs new file mode 100644 index 000000000..a14c69b18 --- /dev/null +++ b/src/TensorFlowNET.Core/Util/Data.cs @@ -0,0 +1,66 @@ +using Tensorflow.NumPy; + +namespace Tensorflow.Util +{ + /// + /// ValidationDataPack is used to pass validation data to fit method. + /// It can recive data which could be A tuple `(x_val, xy_val)` or `(x_val, y_val, sample_weight_val)` of Numpy arrays. + /// + public class ValidationDataPack + { + public NDArray val_x; + public NDArray val_y; + public NDArray val_sample_weight = null; + + public ValidationDataPack((NDArray, NDArray) validation_data) + { + this.val_x = validation_data.Item1; + this.val_y = validation_data.Item2; + } + + public ValidationDataPack((NDArray, NDArray, NDArray) validation_data) + { + this.val_x = validation_data.Item1; + this.val_y = validation_data.Item2; + this.val_sample_weight = validation_data.Item3; + } + + public ValidationDataPack((IEnumerable, NDArray) validation_data) + { + this.val_x = validation_data.Item1.ToArray()[0]; + this.val_y = validation_data.Item2; + } + + public ValidationDataPack((IEnumerable, NDArray, NDArray) validation_data) + { + this.val_x = validation_data.Item1.ToArray()[0]; + this.val_y = validation_data.Item2; + this.val_sample_weight = validation_data.Item3; + } + + public static implicit operator ValidationDataPack((NDArray, NDArray) validation_data) + => new ValidationDataPack(validation_data); + + public static implicit operator ValidationDataPack((NDArray, NDArray, NDArray) validation_data) + => new ValidationDataPack(validation_data); + + public static implicit operator ValidationDataPack((IEnumerable, NDArray) validation_data) + => new ValidationDataPack(validation_data); + + public static implicit operator ValidationDataPack((IEnumerable, NDArray, NDArray) validation_data) + => new ValidationDataPack(validation_data); + + public void Deconstruct(out NDArray val_x, out NDArray val_y) + { + val_x = this.val_x; + val_y = this.val_y; + } + + public void Deconstruct(out NDArray val_x, out NDArray val_y, out NDArray val_sample_weight) + { + val_x = this.val_x; + val_y = this.val_y; + val_sample_weight = this.val_sample_weight; + } + } +} diff --git a/src/TensorFlowNET.Keras/Engine/DataAdapters/DataAdapter.cs b/src/TensorFlowNET.Keras/Engine/DataAdapters/DataAdapter.cs index 6c7d53b2f..b2750496a 100644 --- a/src/TensorFlowNET.Keras/Engine/DataAdapters/DataAdapter.cs +++ b/src/TensorFlowNET.Keras/Engine/DataAdapters/DataAdapter.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Text; using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Util; namespace Tensorflow.Keras.Engine.DataAdapters { @@ -34,9 +35,67 @@ public virtual (Tensors, Tensors) Expand1d(Tensors x, Tensors y) return (x, y); } + public virtual (Tensors, Tensors, Tensors) Expand1d(Tensors x, Tensors y, Tensors sample_weight) + { + for (int i = 0; i < x.Length; i++) + { + if (x[i].shape.ndim == 1) + x[i] = array_ops.expand_dims(x[i], axis: -1); + } + for (int i = 0; i < y.Length; i++) + { + if (y[i].shape.ndim == 1) + y[i] = array_ops.expand_dims(y[i], axis: -1); + } + for (int i = 0; i < sample_weight.Length; i++) + { + if (sample_weight[i].shape.ndim == 1) + sample_weight[i] = array_ops.expand_dims(sample_weight[i], axis: -1); + } + return (x, y, sample_weight); + } + public virtual bool ShouldRecreateIterator() { return true; } + + public static ((NDArray, NDArray, NDArray),ValidationDataPack) train_validation_split((NDArray, NDArray, NDArray) x_y_sample_weight, float validation_split) + { + var x = x_y_sample_weight.Item1; + var y = x_y_sample_weight.Item2; + var sample_weight = x_y_sample_weight.Item3; + int train_count = Convert.ToInt32(x.dims[0] * (1 - validation_split)); + var train_x = x[new Slice(0, train_count)]; + var train_y = y[new Slice(0, train_count)]; + ValidationDataPack validation_data; + if (sample_weight != null) + { + validation_data = (x[new Slice(train_count)], y[new Slice(train_count)], sample_weight[new Slice(train_count)]); + sample_weight = sample_weight[new Slice(0, train_count)]; + } + else + { + validation_data = (x[new Slice(train_count)], y[new Slice(train_count)]); + } + + return ((train_x, train_y, sample_weight), validation_data); + } + + public static ((IEnumerable, NDArray, NDArray), ValidationDataPack) train_validation_split((IEnumerable, NDArray, NDArray) x_y_sample_weight, float validation_split) + { + var x = x_y_sample_weight.Item1; + var y = x_y_sample_weight.Item2; + var sample_weight = x_y_sample_weight.Item3; + int train_count = Convert.ToInt32(y.dims[0] * (1 - validation_split)); + var train_x = x.Select(x => x[new Slice(0, train_count)] as NDArray); + var train_y = y[new Slice(0, train_count)]; + var val_x = x.Select(x => x[new Slice(train_count)] as NDArray); + var val_y = y[new Slice(train_count)]; + NDArray tmp_sample_weight = sample_weight; + sample_weight = sample_weight[new Slice(0, train_count)]; + ValidationDataPack validation_data = (val_x, val_y, tmp_sample_weight[new Slice(train_count)]); + return ((train_x, train_y, sample_weight), validation_data); + } } } diff --git a/src/TensorFlowNET.Keras/Engine/DataAdapters/DataHandler.cs b/src/TensorFlowNET.Keras/Engine/DataAdapters/DataHandler.cs index 4723222f2..a5ee75c93 100644 --- a/src/TensorFlowNET.Keras/Engine/DataAdapters/DataHandler.cs +++ b/src/TensorFlowNET.Keras/Engine/DataAdapters/DataHandler.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using Tensorflow.Keras.ArgsDefinition; using static Tensorflow.Binding; +using Tensorflow.Keras.Utils; namespace Tensorflow.Keras.Engine.DataAdapters { @@ -28,6 +29,7 @@ public class DataHandler public DataHandler(DataHandlerArgs args) { this.args = args; + if (args.StepsPerExecution == null) { _steps_per_execution = tf.Variable(1L); @@ -48,6 +50,7 @@ public DataHandler(DataHandlerArgs args) BatchSize = args.BatchSize, Steps = args.StepsPerEpoch, Epochs = args.Epochs - args.InitialEpoch, + SampleWeight = args.SampleWeight, Shuffle = args.Shuffle, MaxQueueSize = args.MaxQueueSize, Worker = args.Workers, diff --git a/src/TensorFlowNET.Keras/Engine/DataAdapters/IDataAdapter.cs b/src/TensorFlowNET.Keras/Engine/DataAdapters/IDataAdapter.cs index 4bdc49795..bb71b0a2d 100644 --- a/src/TensorFlowNET.Keras/Engine/DataAdapters/IDataAdapter.cs +++ b/src/TensorFlowNET.Keras/Engine/DataAdapters/IDataAdapter.cs @@ -17,6 +17,8 @@ public interface IDataAdapter IDatasetV2 GetDataset(); int GetSize(); (Tensors, Tensors) Expand1d(Tensors x, Tensors y); + (Tensors, Tensors, Tensors) Expand1d(Tensors x, Tensors y, Tensors sample_weight); + bool ShouldRecreateIterator(); } } diff --git a/src/TensorFlowNET.Keras/Engine/DataAdapters/TensorLikeDataAdapter.cs b/src/TensorFlowNET.Keras/Engine/DataAdapters/TensorLikeDataAdapter.cs index 16e646a35..978a3f51c 100644 --- a/src/TensorFlowNET.Keras/Engine/DataAdapters/TensorLikeDataAdapter.cs +++ b/src/TensorFlowNET.Keras/Engine/DataAdapters/TensorLikeDataAdapter.cs @@ -20,7 +20,7 @@ public class TensorLikeDataAdapter : DataAdapter, IDataAdapter public TensorLikeDataAdapter(DataAdapterArgs args) { this.args = args; - _process_tensorlike(); + Tensor sample_weight_tensor = args.SampleWeight != null ? _process_tensorlike(args.SampleWeight) : null; num_samples = (int)args.X.shape[0]; var batch_size = args.BatchSize == -1 ? 32 : args.BatchSize; _batch_size = batch_size; @@ -37,6 +37,8 @@ public TensorLikeDataAdapter(DataAdapterArgs args) inputs.AddRange(args.X); if (args.Y != null) inputs.AddRange(args.Y); + if (sample_weight_tensor != null) + inputs.Add(sample_weight_tensor); dataset = slice_inputs(indices_dataset, inputs); dataset.FirstInputTensorCount = args.X.Length; } @@ -94,8 +96,9 @@ IDatasetV2 slice_inputs(IDatasetV2 indices_dataset, Tensors elements) public override bool ShouldRecreateIterator() => false; - void _process_tensorlike() + Tensor _process_tensorlike(NDArray sample_weights) { + return tf.convert_to_tensor(sample_weights); } } } diff --git a/src/TensorFlowNET.Keras/Engine/LossesContainer.cs b/src/TensorFlowNET.Keras/Engine/LossesContainer.cs index 6a91450de..c06fca593 100644 --- a/src/TensorFlowNET.Keras/Engine/LossesContainer.cs +++ b/src/TensorFlowNET.Keras/Engine/LossesContainer.cs @@ -26,11 +26,11 @@ public LossesContainer(ILossFunc losses, string[] output_names = null) /// /// /// - public Tensor Call(Tensor y_true, Tensor y_pred) + public Tensor Call(Tensor y_true, Tensor y_pred, Tensor sample_weight = null) { if (!_built) Build(y_pred); - var loss_value = _losses.Call(y_true, y_pred); + var loss_value = _losses.Call(y_true, y_pred, sample_weight:sample_weight); var loss_metric_value = loss_value; var batch_dim = array_ops.shape(y_true)[0]; diff --git a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs index a74a77f18..626d7fcad 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs @@ -30,6 +30,7 @@ public partial class Model public Dictionary evaluate(NDArray x, NDArray y, int batch_size = -1, int verbose = 1, + NDArray sample_weight = null, int steps = -1, int max_queue_size = 10, int workers = 1, @@ -51,6 +52,7 @@ public Dictionary evaluate(NDArray x, NDArray y, StepsPerEpoch = steps, InitialEpoch = 0, Epochs = 1, + SampleWeight = sample_weight, MaxQueueSize = max_queue_size, Workers = workers, UseMultiprocessing = use_multiprocessing, @@ -140,7 +142,8 @@ Dictionary evaluate(DataHandler data_handler, CallbackList callba Dictionary test_function(DataHandler data_handler, OwnedIterator iterator) { var data = iterator.next(); - var outputs = test_step(data_handler, data[0], data[1]); + var outputs = data.Length == 2 ? test_step(data_handler, data[0], data[1]) : + test_step(data_handler, data[0], data[1], data[2]); tf_with(ops.control_dependencies(new object[0]), ctl => _test_counter.assign_add(1)); return outputs; } @@ -149,17 +152,23 @@ Dictionary test_step_multi_inputs_function(DataHandler data_handl { var data = iterator.next(); var x_size = data_handler.DataAdapter.GetDataset().FirstInputTensorCount; - var outputs = test_step(data_handler, data.Take(x_size).ToArray(), data.Skip(x_size).ToArray()); + var outputs = data.Length == 2 ? + test_step(data_handler, new Tensors(data.Take(x_size).ToArray()), new Tensors(data.Skip(x_size).ToArray())) : + test_step( + data_handler, + new Tensors(data.Take(x_size).ToArray()), + new Tensors(data.Skip(x_size).Take(x_size).ToArray()), + new Tensors(data.Skip(2 * x_size).ToArray())); tf_with(ops.control_dependencies(new object[0]), ctl => _test_counter.assign_add(1)); return outputs; } - Dictionary test_step(DataHandler data_handler, Tensors x, Tensors y) + Dictionary test_step(DataHandler data_handler, Tensors x, Tensors y, Tensors sample_weight = null) { - (x, y) = data_handler.DataAdapter.Expand1d(x, y); + (x, y, sample_weight) = data_handler.DataAdapter.Expand1d(x, y, sample_weight); var y_pred = Apply(x, training: false); - var loss = compiled_loss.Call(y, y_pred); + var loss = compiled_loss.Call(y, y_pred, sample_weight:sample_weight); compiled_metrics.update_state(y, y_pred); return metrics.Select(x => (x.Name, x.result())).ToDictionary(x => x.Item1, x => (float)x.Item2); } diff --git a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs index d6f89d8be..23c53b707 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs @@ -6,10 +6,12 @@ using Tensorflow.Keras.Engine.DataAdapters; using System.Diagnostics; using Tensorflow.Keras.Callbacks; -using System.Data; +using Tensorflow.Util; namespace Tensorflow.Keras.Engine { + + public partial class Model { /// @@ -19,19 +21,29 @@ public partial class Model /// /// /// - /// /// + /// /// /// /// + /// + /// + /// + /// + /// + /// + /// + /// public ICallback fit(NDArray x, NDArray y, int batch_size = -1, int epochs = 1, int verbose = 1, List callbacks = null, float validation_split = 0f, - (NDArray val_x, NDArray val_y)? validation_data = null, + ValidationDataPack validation_data = null, bool shuffle = true, + Dictionary class_weight = null, + NDArray sample_weight = null, int initial_epoch = 0, int max_queue_size = 10, int workers = 1, @@ -43,21 +55,25 @@ public ICallback fit(NDArray x, NDArray y, $"The array x and y should have same value at dim 0, but got {x.dims[0]} and {y.dims[0]}"); } - var train_x = x; - var train_y = y; + // The default dtype in NDArray is double, so we need to cast sample_weight to float to mul with loss which's dtype is float. + sample_weight = sample_weight?.astype(TF_DataType.TF_FLOAT); if (validation_split != 0f && validation_data == null) { - int train_count = Convert.ToInt32(x.dims[0] * (1 - validation_split)); - train_x = x[new Slice(0, train_count)]; - train_y = y[new Slice(0, train_count)]; - validation_data = (val_x: x[new Slice(train_count)], val_y: y[new Slice(train_count)]); + ((x, y, sample_weight), validation_data) = DataAdapter.train_validation_split((x, y, sample_weight), validation_split); + } + + // TODO(Wanglongzhi2001) + if (class_weight != null) + { + throw new NotImplementedException("class_weight is not implemented"); } var data_handler = new DataHandler(new DataHandlerArgs { - X = train_x, - Y = train_y, + X = x, + Y = y, + SampleWeight = sample_weight, BatchSize = batch_size, InitialEpoch = initial_epoch, Epochs = epochs, @@ -73,14 +89,17 @@ public ICallback fit(NDArray x, NDArray y, train_step_func: train_step_function); } + public ICallback fit(IEnumerable x, NDArray y, int batch_size = -1, int epochs = 1, int verbose = 1, List callbacks = null, float validation_split = 0f, - (IEnumerable val_x, NDArray val_y)? validation_data = null, + ValidationDataPack validation_data = null, bool shuffle = true, + Dictionary class_weight = null, + NDArray sample_weight = null, int initial_epoch = 0, int max_queue_size = 10, int workers = 1, @@ -95,27 +114,23 @@ public ICallback fit(IEnumerable x, NDArray y, } } - var train_x = x; - var train_y = y; + sample_weight = sample_weight?.astype(TF_DataType.TF_FLOAT); + if (validation_split != 0f && validation_data == null) { - int train_count = Convert.ToInt32(y.dims[0] * (1 - validation_split)); - train_x = x.Select(x => x[new Slice(0, train_count)] as NDArray); - train_y = y[new Slice(0, train_count)]; - var val_x = x.Select(x => x[new Slice(train_count)] as NDArray); - var val_y = y[new Slice(train_count)]; - validation_data = (val_x, val_y); + ((x, y, sample_weight), validation_data) = DataAdapter.train_validation_split((x, y, sample_weight), validation_split); } var data_handler = new DataHandler(new DataHandlerArgs { - X = new Tensors(train_x.ToArray()), - Y = train_y, + X = new Tensors(x.ToArray()), + Y = y, BatchSize = batch_size, InitialEpoch = initial_epoch, Epochs = epochs, Shuffle = shuffle, + SampleWeight = sample_weight, MaxQueueSize = max_queue_size, Workers = workers, UseMultiprocessing = use_multiprocessing, @@ -142,8 +157,10 @@ public History fit(IDatasetV2 dataset, int verbose = 1, List callbacks = null, IDatasetV2 validation_data = null, - int validation_step = 10, // 间隔多少次会进行一次验证 + int validation_step = 10, bool shuffle = true, + Dictionary class_weight = null, + NDArray sample_weight = null, int initial_epoch = 0, int max_queue_size = 10, int workers = 1, @@ -210,7 +227,7 @@ History FitInternal(DataHandler data_handler, int epochs, int validation_step, i { if (validation_step > 0 && epoch ==0 || (epoch) % validation_step != 0) continue; - + var val_logs = evaluate(validation_data); foreach(var log in val_logs) { @@ -233,7 +250,7 @@ History FitInternal(DataHandler data_handler, int epochs, int validation_step, i return callbacks.History; } - History FitInternal(DataHandler data_handler, int epochs, int verbose, List callbackList, (NDArray, NDArray)? validation_data, + History FitInternal(DataHandler data_handler, int epochs, int verbose, List callbackList, ValidationDataPack validation_data, Func> train_step_func) { stop_training = false; @@ -274,7 +291,8 @@ History FitInternal(DataHandler data_handler, int epochs, int verbose, List callbackList, (IEnumerable, NDArray)? validation_data, - Func> train_step_func) - { - stop_training = false; - _train_counter.assign(0); - var callbacks = new CallbackList(new CallbackParams - { - Model = this, - Verbose = verbose, - Epochs = epochs, - Steps = data_handler.Inferredsteps - }); - - if (callbackList != null) - { - foreach (var callback in callbackList) - callbacks.callbacks.add(callback); - } - - callbacks.on_train_begin(); - - foreach (var (epoch, iterator) in data_handler.enumerate_epochs()) - { - reset_metrics(); - callbacks.on_epoch_begin(epoch); - // data_handler.catch_stop_iteration(); - var logs = new Dictionary(); - long End_step = 0; - foreach (var step in data_handler.steps()) - { - callbacks.on_train_batch_begin(step); - logs = train_step_func(data_handler, iterator); - var end_step = step + data_handler.StepIncrement; - End_step = end_step; - callbacks.on_train_batch_end(end_step, logs); - } - - if (validation_data != null) - { - var val_logs = evaluate(validation_data.Value.Item1, validation_data.Value.Item2); - foreach (var log in val_logs) - { - logs["val_" + log.Key] = log.Value; - callbacks.on_train_batch_end(End_step, logs); - } - } - - callbacks.on_epoch_end(epoch, logs); - - GC.Collect(); - GC.WaitForPendingFinalizers(); - if (stop_training) - { - break; - } - } - - return callbacks.History; - } } } diff --git a/src/TensorFlowNET.Keras/Engine/Model.Train.cs b/src/TensorFlowNET.Keras/Engine/Model.Train.cs index ad3c70d2d..8f1ec808c 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Train.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Train.cs @@ -12,7 +12,9 @@ public partial class Model Dictionary train_step_function(DataHandler data_handler, OwnedIterator iterator) { var data = iterator.next(); - var outputs = train_step(data_handler, data[0], data[1]); + // whether have sample_weight + var outputs = data.Length == 2 ? train_step(data_handler, data[0], data[1]) : + train_step(data_handler, data[0], data[1], data[2]); tf_with(ops.control_dependencies(new object[0]), ctl => _train_counter.assign_add(1)); return outputs; } @@ -21,7 +23,13 @@ Dictionary train_step_multi_inputs_function(DataHandler data_hand { var data = iterator.next(); var x_size = data_handler.DataAdapter.GetDataset().FirstInputTensorCount; - var outputs = train_step(data_handler, new Tensors(data.Take(x_size).ToArray()), new Tensors(data.Skip(x_size).ToArray())); + var outputs = data.Length == 2 ? + train_step(data_handler, new Tensors(data.Take(x_size).ToArray()), new Tensors(data.Skip(x_size).ToArray())) : + train_step( + data_handler, + new Tensors(data.Take(x_size).ToArray()), + new Tensors(data.Skip(x_size).Take(x_size).ToArray()), + new Tensors(data.Skip(2 * x_size).ToArray())); tf_with(ops.control_dependencies(new object[0]), ctl => _train_counter.assign_add(1)); return outputs; } @@ -61,6 +69,34 @@ Dictionary train_step(DataHandler data_handler, Tensors x, Tensor }); return dict; } + Dictionary train_step(DataHandler data_handler, Tensors x, Tensors y, Tensors sample_weight = null) + { + (x, y, sample_weight) = data_handler.DataAdapter.Expand1d(x, y, sample_weight); + using var tape = tf.GradientTape(); + var y_pred = Apply(x, training: true); + var loss = compiled_loss.Call(y, y_pred, sample_weight:sample_weight); + + // For custom training steps, users can just write: + // trainable_variables = self.trainable_variables + // gradients = tape.gradient(loss, trainable_variables) + // self.optimizer.apply_gradients(zip(gradients, trainable_variables)) + // The _minimize call does a few extra steps unnecessary in most cases, + // such as loss scaling and gradient clipping. + _minimize(tape, optimizer, loss, TrainableVariables); + compiled_metrics.update_state(y, y_pred); + + var dict = new Dictionary(); + metrics.ToList().ForEach(x => + { + var r = x.result(); + if (r.ndim > 0) + { + r = tf.reduce_mean(r); + } + dict[x.Name] = (float)r; + }); + return dict; + } void _minimize(GradientTape tape, IOptimizer optimizer, Tensor loss, List trainable_variables) { diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs index dbf5cae1e..67e2b0464 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs @@ -74,8 +74,8 @@ public void TrainLSTMWithMnist() OneHot = true, ValidationSize = 55000, }).Result; - - model.fit(dataset.Train.Data, dataset.Train.Labels, batch_size: 16, epochs: 1); + var sample_weight = np.ones(((int)dataset.Train.Data.shape[0])); + model.fit(dataset.Train.Data, dataset.Train.Labels, batch_size: 16, epochs: 1, sample_weight:sample_weight); } [TestMethod] From 0f02885dfb3647ae1b2bfae51491b4f119da4be9 Mon Sep 17 00:00:00 2001 From: hchen Date: Mon, 2 Oct 2023 18:57:17 -0500 Subject: [PATCH 50/98] Allow Model to cache weights. --- .../Engine/Model.Training.cs | 35 ++++++++++++++++++- src/TensorFlowNET.Keras/Saving/hdf5_format.cs | 4 +-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/TensorFlowNET.Keras/Engine/Model.Training.cs b/src/TensorFlowNET.Keras/Engine/Model.Training.cs index 50d934d9d..457b3d694 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Training.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Training.cs @@ -10,8 +10,38 @@ namespace Tensorflow.Keras.Engine { public partial class Model { + static Dictionary> weightsCache + = new Dictionary>(); + public void load_weights(string filepath, bool by_name = false, bool skip_mismatch = false, object options = null) { + // Get from cache + if (weightsCache.ContainsKey(filepath)) + { + var filtered_layers = new List(); + foreach (var layer in Layers) + { + var weights = hdf5_format._legacy_weights(layer); + if (weights.Count > 0) + filtered_layers.append(layer); + } + + var weight_value_tuples = new List<(IVariableV1, NDArray)>(); + filtered_layers.Select((layer, i) => + { + var symbolic_weights = hdf5_format._legacy_weights(layer); + foreach(var weight in symbolic_weights) + { + var weight_value = weightsCache[filepath].First(x => x.Item1 == weight.Name).Item2; + weight_value_tuples.Add((weight, weight_value)); + } + return layer; + }).ToList(); + + keras.backend.batch_set_value(weight_value_tuples); + return; + } + long fileId = Hdf5.OpenFile(filepath, true); if(fileId < 0) { @@ -29,8 +59,11 @@ public void load_weights(string filepath, bool by_name = false, bool skip_mismat throw new NotImplementedException(""); else { - hdf5_format.load_weights_from_hdf5_group(fileId, Layers); + var weight_value_tuples = hdf5_format.load_weights_from_hdf5_group(fileId, Layers); Hdf5.CloseFile(fileId); + + weightsCache[filepath] = weight_value_tuples.Select(x => (x.Item1.Name, x.Item2)).ToList(); + keras.backend.batch_set_value(weight_value_tuples); } } diff --git a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs index bab0efecf..68b73953d 100644 --- a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs +++ b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs @@ -82,7 +82,7 @@ public static void load_optimizer_weights_from_hdf5_group(long filepath = -1, Di } - public static void load_weights_from_hdf5_group(long f, List layers) + public static List<(IVariableV1, NDArray)> load_weights_from_hdf5_group(long f, List layers) { string original_keras_version = "2.5.0"; string original_backend = null; @@ -152,7 +152,7 @@ public static void load_weights_from_hdf5_group(long f, List layers) weight_value_tuples.AddRange(zip(symbolic_weights, weight_values)); } - keras.backend.batch_set_value(weight_value_tuples); + return weight_value_tuples; } public static void toarrayf4(long filepath = -1, Dictionary custom_objects = null, bool compile = false) From a1c64effcfe7976b6cb0f3fbbd268cee203b4874 Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Thu, 5 Oct 2023 20:49:22 +0800 Subject: [PATCH 51/98] feat: add the implementation of class_weight in model.fit --- .../Engine/DataAdapters/DataHandler.cs | 70 ++++++++++++++++++- .../Engine/Model.Evaluate.cs | 13 +++- src/TensorFlowNET.Keras/Engine/Model.Fit.cs | 11 ++- 3 files changed, 84 insertions(+), 10 deletions(-) diff --git a/src/TensorFlowNET.Keras/Engine/DataAdapters/DataHandler.cs b/src/TensorFlowNET.Keras/Engine/DataAdapters/DataHandler.cs index a5ee75c93..a305e5033 100644 --- a/src/TensorFlowNET.Keras/Engine/DataAdapters/DataHandler.cs +++ b/src/TensorFlowNET.Keras/Engine/DataAdapters/DataHandler.cs @@ -3,6 +3,8 @@ using Tensorflow.Keras.ArgsDefinition; using static Tensorflow.Binding; using Tensorflow.Keras.Utils; +using Tensorflow.Util; +using Tensorflow.Framework; namespace Tensorflow.Keras.Engine.DataAdapters { @@ -24,6 +26,7 @@ public class DataHandler long _steps_per_execution_value; int _initial_epoch => args.InitialEpoch; int _epochs => args.Epochs; + NDArray _sample_weight => args.SampleWeight; IVariableV1 _steps_per_execution; public DataHandler(DataHandlerArgs args) @@ -75,10 +78,75 @@ public DataHandler(DataHandlerArgs args) } _dataset = _adapter.GetDataset(); - _inferred_steps = _infer_steps(args.StepsPerEpoch, _dataset); _current_step = 0; _step_increment = _steps_per_execution_value - 1; _insufficient_data = false; + _configure_dataset_and_inferred_steps(args.X, args.ClassWeight); + } + + void _configure_dataset_and_inferred_steps(Tensors x, Dictionary class_weight) + { + if (_dataset == null) + { + _dataset = _adapter.GetDataset(); + _inferred_steps = _infer_steps(args.StepsPerEpoch, _dataset); + } + + if (class_weight != null) + { + _dataset = _dataset.map(_make_class_weight_map_fn(class_weight)); + } + _inferred_steps = _infer_steps(args.StepsPerEpoch, _dataset); + } + + + Func _make_class_weight_map_fn(Dictionary class_weight) + { + var class_ids = class_weight.Keys.OrderBy(key => key).ToList(); + var expected_class_ids = range(class_ids[0], class_ids[class_ids.Count - 1] + 1); + if (!class_ids.SequenceEqual(expected_class_ids)) + { + throw new ValueError("Expected `class_weight` to be a dict with keys from 0 to one less "+ + $"than the number of classes, found {class_weight}"); + } + + var class_weight_list = new List(); + foreach (var class_id in class_ids) + { + class_weight_list.Add(class_weight[class_id]); + } + var class_weight_tensor = tf.convert_to_tensor(class_weight_list.ToArray()); + + Func _class_weight_map_fn = (Tensors data) => + { + var x = data[0]; + var y = data[1]; + var sw = _sample_weight == null ? null : ops.convert_to_tensor(_sample_weight); + + if (y.shape.rank > 2) + { + throw new ValueError("`class_weight` not supported for 3+ dimensional targets."); + } + + var y_classes = smart_module.smart_cond( + y.shape.rank == 2 && y.shape[1] > 1, + () => math_ops.argmax(y, dimension: 1), + () => math_ops.cast(tf.reshape(y, (-1)), TF_DataType.TF_INT64)); + + var cw = array_ops.gather(class_weight_tensor, y_classes); + if (sw != null) + { + cw = tf.cast(cw, sw.dtype); + cw *= sw; + } + else + { + sw = cw; + } + return new Tensors { x, y, sw }; + }; + + return _class_weight_map_fn; } long _infer_steps(int steps_per_epoch, IDatasetV2 dataset) diff --git a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs index 626d7fcad..94a2e6646 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs @@ -164,11 +164,20 @@ Dictionary test_step_multi_inputs_function(DataHandler data_handl } - Dictionary test_step(DataHandler data_handler, Tensors x, Tensors y, Tensors sample_weight = null) + Dictionary test_step(DataHandler data_handler, Tensors x, Tensors y) + { + (x,y) = data_handler.DataAdapter.Expand1d(x, y); + var y_pred = Apply(x, training: false); + var loss = compiled_loss.Call(y, y_pred); + compiled_metrics.update_state(y, y_pred); + return metrics.Select(x => (x.Name, x.result())).ToDictionary(x => x.Item1, x => (float)x.Item2); + } + + Dictionary test_step(DataHandler data_handler, Tensors x, Tensors y, Tensors sample_weight) { (x, y, sample_weight) = data_handler.DataAdapter.Expand1d(x, y, sample_weight); var y_pred = Apply(x, training: false); - var loss = compiled_loss.Call(y, y_pred, sample_weight:sample_weight); + var loss = compiled_loss.Call(y, y_pred, sample_weight: sample_weight); compiled_metrics.update_state(y, y_pred); return metrics.Select(x => (x.Name, x.result())).ToDictionary(x => x.Item1, x => (float)x.Item2); } diff --git a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs index 23c53b707..689fc9fb8 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs @@ -63,12 +63,6 @@ public ICallback fit(NDArray x, NDArray y, ((x, y, sample_weight), validation_data) = DataAdapter.train_validation_split((x, y, sample_weight), validation_split); } - // TODO(Wanglongzhi2001) - if (class_weight != null) - { - throw new NotImplementedException("class_weight is not implemented"); - } - var data_handler = new DataHandler(new DataHandlerArgs { X = x, @@ -78,6 +72,7 @@ public ICallback fit(NDArray x, NDArray y, InitialEpoch = initial_epoch, Epochs = epochs, Shuffle = shuffle, + ClassWeight = class_weight, MaxQueueSize = max_queue_size, Workers = workers, UseMultiprocessing = use_multiprocessing, @@ -126,11 +121,12 @@ public ICallback fit(IEnumerable x, NDArray y, { X = new Tensors(x.ToArray()), Y = y, + SampleWeight = sample_weight, BatchSize = batch_size, InitialEpoch = initial_epoch, Epochs = epochs, Shuffle = shuffle, - SampleWeight = sample_weight, + ClassWeight = class_weight, MaxQueueSize = max_queue_size, Workers = workers, UseMultiprocessing = use_multiprocessing, @@ -174,6 +170,7 @@ public History fit(IDatasetV2 dataset, InitialEpoch = initial_epoch, Epochs = epochs, Shuffle = shuffle, + SampleWeight = sample_weight, MaxQueueSize = max_queue_size, Workers = workers, UseMultiprocessing = use_multiprocessing, From ba8f0b084fe30868f091a168d2afa4ff274971d1 Mon Sep 17 00:00:00 2001 From: dogvane Date: Sun, 8 Oct 2023 21:45:26 +0800 Subject: [PATCH 52/98] =?UTF-8?q?add=20DepthwiseConv2D=20(=E6=B7=B1?= =?UTF-8?q?=E5=BA=A6=E5=8F=AF=E5=88=86=E7=A6=BB=E5=8D=B7=E7=A7=AF)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Eager/EagerRunner.RecordGradient.cs | 5 + src/TensorFlowNET.Core/Gradients/nn_grad.cs | 31 ++++ .../Keras/Layers/ILayersApi.cs | 13 ++ src/TensorFlowNET.Core/Tensors/tensor_util.cs | 5 +- .../Layers/Convolution/DepthwiseConv2D.cs | 167 ++++++++++++++++++ src/TensorFlowNET.Keras/Layers/LayersApi.cs | 32 ++++ .../EagerModeTestBase.cs | 34 ++++ .../Layers/Layers.Convolution.Test.cs | 125 +++++++++++++ .../EagerModeTestBase.cs | 14 ++ 9 files changed, 425 insertions(+), 1 deletion(-) create mode 100644 src/TensorFlowNET.Keras/Layers/Convolution/DepthwiseConv2D.cs diff --git a/src/TensorFlowNET.Core/Eager/EagerRunner.RecordGradient.cs b/src/TensorFlowNET.Core/Eager/EagerRunner.RecordGradient.cs index 59d5fd030..2bdd65f5b 100644 --- a/src/TensorFlowNET.Core/Eager/EagerRunner.RecordGradient.cs +++ b/src/TensorFlowNET.Core/Eager/EagerRunner.RecordGradient.cs @@ -80,6 +80,11 @@ BackwardFunction GetGradientFunction(string op_name, Tensor[] op_outputs) => (out_grads, unneeded_gradients) => { + if(!ops.gradientFunctions.ContainsKey(op_name)) + { + throw new Exception($"gradientFunctions not find op_name: {op_name}"); + } + if (ops.gradientFunctions[op_name] == null) return new Tensor[op_inputs.Length]; diff --git a/src/TensorFlowNET.Core/Gradients/nn_grad.cs b/src/TensorFlowNET.Core/Gradients/nn_grad.cs index a43a91b9a..87646a9ea 100644 --- a/src/TensorFlowNET.Core/Gradients/nn_grad.cs +++ b/src/TensorFlowNET.Core/Gradients/nn_grad.cs @@ -229,6 +229,37 @@ public static Tensor[] _Conv2DGrad(Operation op, Tensor[] grads) }; } + /// + /// Gradient function for Conv2D. + /// + /// + /// + /// + [RegisterGradient("DepthwiseConv2dNative")] + public static Tensor[] _DepthwiseConv2DGrad(Operation op, Tensor[] grads) + { + var dilations = op.get_attr_list("dilations"); + var strides = op.get_attr_list("strides"); + var padding = op.get_attr("padding"); + var explicit_paddings = op.get_attr_list("explicit_paddings"); + var data_format = op.get_attr("data_format"); + var shape = gen_array_ops.shape_n(new Tensor[] { op.inputs[0], op.inputs[1] }); + + return new Tensor[] + { + gen_nn_ops.depthwise_conv2d_native_backprop_input( + shape[0], op.inputs[1], grads[0], + strides, padding, explicit_paddings, + dilations: dilations, + data_format: data_format), + gen_nn_ops.depthwise_conv2d_native_backprop_filter(op.inputs[0], shape[1], grads[0], + strides, padding, + dilations: dilations, + explicit_paddings: explicit_paddings, + data_format: data_format) + }; + } + [RegisterGradient("FusedBatchNorm")] public static Tensor[] _FusedBatchNormGrad(Operation op, Tensor[] grads) => _BaseFusedBatchNormGrad(op, 0, grads); diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index 5e08eadc4..a8141d354 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -95,6 +95,19 @@ public ILayer Conv2D(int filters, bool use_bias = true, string kernel_initializer = "glorot_uniform", string bias_initializer = "zeros"); + public ILayer DepthwiseConv2D(Shape kernel_size = null, + Shape strides = null, + string padding = "valid", + string data_format = null, + Shape dilation_rate = null, + int groups = 1, + int depth_multiplier = 1, + string activation = null, + bool use_bias = false, + string kernel_initializer = "glorot_uniform", + string bias_initializer = "zeros", + string depthwise_initializer = "glorot_uniform" + ); public ILayer Dense(int units); public ILayer Dense(int units, diff --git a/src/TensorFlowNET.Core/Tensors/tensor_util.cs b/src/TensorFlowNET.Core/Tensors/tensor_util.cs index e65c4850d..f688d4d5d 100644 --- a/src/TensorFlowNET.Core/Tensors/tensor_util.cs +++ b/src/TensorFlowNET.Core/Tensors/tensor_util.cs @@ -249,6 +249,9 @@ public static TensorProto make_tensor_proto(object values, TF_DataType dtype = T case sbyte val: tensor_proto.IntVal.AddRange(new[] { (int)val }); break; + case byte val: + tensor_proto.IntVal.AddRange(new[] { (int)val }); + break; case int val: tensor_proto.IntVal.AddRange(new[] { val }); break; @@ -262,7 +265,7 @@ public static TensorProto make_tensor_proto(object values, TF_DataType dtype = T tensor_proto.DoubleVal.AddRange(new[] { val }); break; default: - throw new Exception("make_tensor_proto Not Implemented"); + throw new Exception($"make_tensor_proto Not Implemented {values.GetType().Name}"); } } diff --git a/src/TensorFlowNET.Keras/Layers/Convolution/DepthwiseConv2D.cs b/src/TensorFlowNET.Keras/Layers/Convolution/DepthwiseConv2D.cs new file mode 100644 index 000000000..dae4a4036 --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Convolution/DepthwiseConv2D.cs @@ -0,0 +1,167 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.Saving; +using Tensorflow.Common.Types; +using Tensorflow.Keras.Utils; +using Tensorflow.Operations; +using Newtonsoft.Json; +using System.Security.Cryptography; + +namespace Tensorflow.Keras.Layers +{ + public class DepthwiseConv2DArgs: Conv2DArgs + { + /// + /// depth_multiplier: The number of depthwise convolution output channels for + /// each input channel.The total number of depthwise convolution output + /// channels will be equal to `filters_in* depth_multiplier`. + /// + [JsonProperty("depth_multiplier")] + public int DepthMultiplier { get; set; } = 1; + + [JsonProperty("depthwise_initializer")] + public IInitializer DepthwiseInitializer { get; set; } + } + + public class DepthwiseConv2D : Conv2D + { + /// + /// depth_multiplier: The number of depthwise convolution output channels for + /// each input channel.The total number of depthwise convolution output + /// channels will be equal to `filters_in* depth_multiplier`. + /// + int DepthMultiplier = 1; + + IInitializer DepthwiseInitializer; + + int[] strides; + + int[] dilation_rate; + + string getDataFormat() + { + return data_format == "channels_first" ? "NCHW" : "NHWC"; + } + + static int _id = 1; + + public DepthwiseConv2D(DepthwiseConv2DArgs args):base(args) + { + args.Padding = args.Padding.ToUpper(); + + if(string.IsNullOrEmpty(args.Name)) + name = "DepthwiseConv2D_" + _id; + + this.DepthMultiplier = args.DepthMultiplier; + this.DepthwiseInitializer = args.DepthwiseInitializer; + + } + + public override void build(KerasShapesWrapper input_shape) + { + //base.build(input_shape); + + var shape = input_shape.ToSingleShape(); + + int channel_axis = data_format == "channels_first" ? 1 : -1; + var input_channel = channel_axis < 0 ? + shape.dims[shape.ndim + channel_axis] : + shape.dims[channel_axis]; + + var arg = args as DepthwiseConv2DArgs; + + if (arg.Strides.ndim != shape.ndim) + { + if (arg.Strides.ndim == 2) + { + this.strides = new int[] { 1, (int)arg.Strides[0], (int)arg.Strides[1], 1 }; + } + else + { + this.strides = conv_utils.normalize_tuple(new int[] { (int)arg.Strides[0] }, shape.ndim, "strides"); + } + } + else + { + this.strides = arg.Strides.dims.Select(o=>(int)(o)).ToArray(); + } + + if (arg.DilationRate.ndim != shape.ndim) + { + this.dilation_rate = conv_utils.normalize_tuple(new int[] { (int)arg.DilationRate[0] }, shape.ndim, "dilation_rate"); + } + + long channel_data = data_format == "channels_first" ? shape[0] : shape[shape.Length - 1]; + + var depthwise_kernel_shape = this.kernel_size.dims.concat(new long[] { + channel_data, + this.DepthMultiplier + }); + + this.kernel = this.add_weight( + shape: depthwise_kernel_shape, + initializer: this.DepthwiseInitializer != null ? this.DepthwiseInitializer : this.kernel_initializer, + name: "depthwise_kernel", + trainable: true, + dtype: DType, + regularizer: this.kernel_regularizer + ); + + var axes = new Dictionary(); + axes.Add(-1, (int)input_channel); + inputSpec = new InputSpec(min_ndim: rank + 2, axes: axes); + + + if (use_bias) + { + bias = add_weight(name: "bias", + shape: ((int)channel_data), + initializer: bias_initializer, + trainable: true, + dtype: DType); + } + + built = true; + _buildInputShape = input_shape; + } + + protected override Tensors Call(Tensors inputs, Tensors state = null, + bool? training = false, IOptionalArgs? optional_args = null) + { + Tensor outputs = null; + + outputs = gen_nn_ops.depthwise_conv2d_native( + inputs, + filter: this.kernel.AsTensor(), + strides: this.strides, + padding: this.padding, + dilations: this.dilation_rate, + data_format: this.getDataFormat(), + name: name + ); + + if (use_bias) + { + if (data_format == "channels_first") + { + throw new NotImplementedException("call channels_first"); + } + else + { + outputs = gen_nn_ops.bias_add(outputs, ops.convert_to_tensor(bias), + data_format: this.getDataFormat(), name: name); + } + } + + if (activation != null) + outputs = activation.Apply(outputs); + + + return outputs; + } + + } +} \ No newline at end of file diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index 928e7e337..95828fbf7 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -210,6 +210,38 @@ public ILayer Conv2D(int filters, Activation = keras.activations.GetActivationFromName(activation) }); + public ILayer DepthwiseConv2D(Shape kernel_size = null, + Shape strides = null, + string padding = "valid", + string data_format = null, + Shape dilation_rate = null, + int groups = 1, + int depth_multiplier = 1, + string activation = null, + bool use_bias = false, + string kernel_initializer = "glorot_uniform", + string bias_initializer = "zeros", + string depthwise_initializer = "glorot_uniform" + ) + => new DepthwiseConv2D(new DepthwiseConv2DArgs + { + Rank = 2, + Filters = 1, + KernelSize = (kernel_size == null) ? (5, 5) : kernel_size, + Strides = strides == null ? (1) : strides, + Padding = padding, + DepthMultiplier = depth_multiplier, + DataFormat = data_format, + DilationRate = dilation_rate == null ? (1) : dilation_rate, + Groups = groups, + UseBias = use_bias, + KernelInitializer = GetInitializerByName(kernel_initializer), + DepthwiseInitializer = GetInitializerByName(depthwise_initializer == null ? kernel_initializer : depthwise_initializer), + BiasInitializer = GetInitializerByName(bias_initializer), + Activation = keras.activations.GetActivationFromName(activation), + }); + + /// /// Transposed convolution layer (sometimes called Deconvolution). /// diff --git a/test/TensorFlowNET.Keras.UnitTest/EagerModeTestBase.cs b/test/TensorFlowNET.Keras.UnitTest/EagerModeTestBase.cs index c7eab364c..635f13a54 100644 --- a/test/TensorFlowNET.Keras.UnitTest/EagerModeTestBase.cs +++ b/test/TensorFlowNET.Keras.UnitTest/EagerModeTestBase.cs @@ -33,6 +33,40 @@ public bool Equal(float[] f1, float[] f2) return ret; } + + public void AssertArray(int[] f1, int[] f2) + { + bool ret = false; + for (var i = 0; i < f1.Length; i++) + { + ret = f1[i] == f2[i]; + if (!ret) + break; + } + + if (!ret) + { + Assert.Fail($"Array not Equal:[{string.Join(",", f1)}] [{string.Join(",", f2)}]"); + } + } + + public void AssertArray(float[] f1, float[] f2) + { + bool ret = false; + var tolerance = .00001f; + for (var i = 0; i < f1.Length; i++) + { + ret = Math.Abs(f1[i] - f2[i]) <= tolerance; + if (!ret) + break; + } + + if (!ret) + { + Assert.Fail($"Array float not Equal:[{string.Join(",", f1)}] [{string.Join(",", f2)}]"); + } + } + public bool Equal(double[] d1, double[] d2) { bool ret = false; diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/Layers.Convolution.Test.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/Layers.Convolution.Test.cs index 997dcb4f6..15c6e80fe 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/Layers.Convolution.Test.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/Layers.Convolution.Test.cs @@ -1,6 +1,8 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Linq; using Tensorflow.NumPy; using static Tensorflow.KerasApi; +using static Tensorflow.Binding; namespace Tensorflow.Keras.UnitTest.Layers { @@ -193,5 +195,128 @@ public void BasicConv2D_ksize_dilation_same() Assert.AreEqual(x.dims[2], y.shape[2]); Assert.AreEqual(filters, y.shape[3]); } + + + [TestMethod] + public void BasicDepthwiseConv2D() + { + var conv = keras.layers.DepthwiseConv2D(kernel_size:3, strides:1, activation: null, + padding:"same", depthwise_initializer: "ones"); + + var x = np.arange(2 * 9* 9* 3).reshape((2, 9, 9, 3)); + var x2 = ops.convert_to_tensor(x, TF_DataType.TF_FLOAT); + + var y = conv.Apply(x2); + + print($"input:{x2.shape} DepthwiseConv2D.out: {y.shape}"); + + + Assert.AreEqual(4, y.shape.ndim); + var arr = y.numpy().reshape((2, 9, 9, 3)); + + AssertArray(x[new int[] { 1, 1, 1 }].ToArray(), new int[] { 273, 274, 275 }); + AssertArray(arr[new int[] { 1, 1, 1 }].ToArray(), new float[] { 2457f, 2466f, 2475f }); + + var bn = keras.layers.BatchNormalization(); + var y2 = bn.Apply(y); + arr = y2.numpy().ToArray(); + + double delta = 0.0001; // 误差范围 + + Assert.AreEqual(arr[0], 59.97002f, delta); + Assert.AreEqual(arr[1], 63.96802f, delta); + } + + + [TestMethod] + public void BasicDepthwiseConv2D_strides_2() + { + var conv = keras.layers.DepthwiseConv2D(kernel_size: 3, strides: (1, 2, 2, 1), activation: null, + padding: "same", depthwise_initializer: "ones"); + + var x = np.arange(2 * 9 * 9 * 3).reshape((2, 9, 9, 3)); + var x2 = ops.convert_to_tensor(x, TF_DataType.TF_FLOAT); + + var y = conv.Apply(x2); + + print($"input:{x2.shape} DepthwiseConv2D.out: {y.shape}"); + + Assert.AreEqual(4, y.shape.ndim); + var arr = y.numpy().reshape((2, 5, 5, 3)); + + AssertArray(x[new int[] { 1, 1, 1 }].ToArray(), new int[] { 273, 274, 275 }); + AssertArray(arr[new int[] { 1, 1, 1 }].ToArray(), new float[] { 2727f, 2736f, 2745f }); + + var bn = keras.layers.BatchNormalization(); + var y2 = bn.Apply(y); + arr = y2.numpy().ToArray(); + + double delta = 0.0001; // 误差范围 + + Assert.AreEqual(arr[0], 59.97002f, delta); + Assert.AreEqual(arr[1], 63.96802f, delta); + } + + + + [TestMethod] + public void BasicDepthwiseConv2D_strides_3() + { + var conv = keras.layers.DepthwiseConv2D(kernel_size: 3, strides: 3, activation: null, + padding: "same", depthwise_initializer: "ones"); + + var x = np.arange(2 * 9 * 9 * 3).reshape((2, 9, 9, 3)); + var x2 = ops.convert_to_tensor(x, TF_DataType.TF_FLOAT); + + var y = conv.Apply(x2); + + print($"input:{x2.shape} DepthwiseConv2D.out: {y.shape}"); + + Assert.AreEqual(4, y.shape.ndim); + var arr = y.numpy().reshape((2, 3, 3, 3)); + + AssertArray(x[new int[] { 1, 1, 1 }].ToArray(), new int[] { 273, 274, 275 }); + AssertArray(arr[new int[] { 1, 1, 1 }].ToArray(), new float[] { 3267f, 3276f, 3285f }); + + var bn = keras.layers.BatchNormalization(); + var y2 = bn.Apply(y); + arr = y2.numpy().ToArray(); + + double delta = 0.0001; // 误差范围 + + Assert.AreEqual(arr[0], 269.86508f, delta); + Assert.AreEqual(arr[1], 278.8606f, delta); + + } + [TestMethod] + public void BasicDepthwiseConv2D_UseBias() + { + var conv = keras.layers.DepthwiseConv2D(kernel_size: 3, strides: 1, activation: null, + use_bias: true, padding: "same", + depthwise_initializer: "ones", + bias_initializer:"ones" + ); + + var weight = conv.get_weights(); + + var x = np.arange(9 * 9 * 3).reshape((1, 9, 9, 3)); + var x2 = ops.convert_to_tensor(x, TF_DataType.TF_FLOAT); + var y = conv.Apply(x2); + + Assert.AreEqual(4, y.shape.ndim); + var arr = y.numpy().ToArray(); + + Assert.AreEqual(arr[0], 61f); + Assert.AreEqual(arr[1], 65f); + + var bn = keras.layers.BatchNormalization(); + var y2 = bn.Apply(y); + arr = y2.numpy().ToArray(); + + double delta = 0.0001; // 误差范围 + + Assert.AreEqual(arr[0], 60.96952f, delta); + Assert.AreEqual(arr[1], 64.96752f, delta); + } } } diff --git a/test/TensorFlowNET.UnitTest/EagerModeTestBase.cs b/test/TensorFlowNET.UnitTest/EagerModeTestBase.cs index d08f4e505..b7b9ae128 100644 --- a/test/TensorFlowNET.UnitTest/EagerModeTestBase.cs +++ b/test/TensorFlowNET.UnitTest/EagerModeTestBase.cs @@ -20,6 +20,20 @@ public bool Equal(float f1, float f2) return Math.Abs(f1 - f2) <= tolerance; } + public bool Equal(long[] l1, long[] l2) + { + if (l1.Length != l2.Length) + return false; + + for (var i = 0; i < l1.Length; i++) + { + if (l1[i] != l2[i]) + return false; + } + + return true; + } + public bool Equal(float[] f1, float[] f2) { bool ret = false; From 5e4f53077f94ddf8513dd925f18eeb05b81a9482 Mon Sep 17 00:00:00 2001 From: dogvane Date: Sun, 8 Oct 2023 21:52:55 +0800 Subject: [PATCH 53/98] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E5=B7=A6=E5=8F=B3=E5=92=8C=E4=B8=8A=E4=B8=8B=E7=BF=BB=E8=BD=AC?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98=EF=BC=8C=E5=B9=B6=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E5=AF=B9=E5=BA=94=E6=B5=8B=E8=AF=95=E7=94=A8=E4=BE=8B=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data/img001.bmp | Bin 0 -> 178662 bytes src/TensorFlowNET.Core/APIs/tf.image.cs | 7 + src/TensorFlowNET.Core/APIs/tf.io.cs | 7 + .../Keras/Layers/ILayersApi.cs | 6 + .../Operations/image_ops_impl.cs | 43 ++- src/TensorFlowNET.Keras/Layers/LayersApi.cs | 23 +- .../TensorFlowNET.Graph.UnitTest/ImageTest.cs | 90 +++++ .../ManagedAPI/ArrayOpsTest.cs | 317 ++++++++++++++++++ .../TensorFlowNET.UnitTest/NumPy/ShapeTest.cs | 44 +++ 9 files changed, 525 insertions(+), 12 deletions(-) create mode 100644 data/img001.bmp create mode 100644 test/TensorFlowNET.UnitTest/NumPy/ShapeTest.cs diff --git a/data/img001.bmp b/data/img001.bmp new file mode 100644 index 0000000000000000000000000000000000000000..d149d76f1ac11b4f5f6700560f9bba04868f8ab4 GIT binary patch literal 178662 zcmeI5G14wM4MiWUf?&c4SOE(lBVd!j0!Y~qMKT#V2t4KLvfOT2mSnm6zIrp&-Jjdm zJ@@?Io0+1DKmPfj|M=~X|NZ&{{q=kL>)*fr^_w5RqpKf3{{HLd|G)Y5Z~wtB5C8!X z009sH0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009sH z0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI z5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X z009sH0T2Lz&k(r&|61buBLx2SmmlSK$@^B>Khh>*RsvP*syE!OaBy)hfkte)rC)IS z+(I#VA;86k+i>Fr1n$J9Xdn6BerwE+=NgC&w?FR2(ecM(!|kKX{qpdSV#Dn(>v3-O zuGkcZe7^oVH{f{kYeu>rFUF?_5x5hZB7CIukHkF4Nt00rV#93{!q{XBfw9FtGIoe3AQ2mG$M=F07YPW(hTDsr@p>=;x!7&*!`i4C{So5F@u1pLH?+fzkgu`K~t zvEjCDS=dcZz*}s%E$;#=4G9E@4Yv*Z!d5~8F=E4QVI5dzMIcOUxNTJ#_K^^X6dP_! z8o?S90>NU#ZIjlp$s>VyvElY(DOj*hV2jvrd%d{KJN*39FOuJ`5q60Ux39JS?N5Kj z?{^R=wSMG(*!&pt9i1XYC4lek`n_G5;Nl$w;5OWThsitz+^%lZ1?Ph&hY~2YKK_H~ zwf%FiB{6xAdGbpBa|f+48F=NaE&o;It~g2QCdBq`B8+)QY2)w`)11beJvx=x#6(!S z6TPr)>1o@9({V#95&M;^BaM)9!&o($&MA~r=HmnHu~3!vwB#8k{NF~1?N zJKu?PU9;ty50+DGf-_b67MLc7&cN5V(QN54=*&-< zjoEOcTl0~+8&+qUi|*pR4{5?YVs{IgEqyOf_34V3@p?Bt*IRXJ*B85DmziD9rOk!a zC`$L9++%-5Od&w4X{*W2h1LGH`wYYm&|6rIqI6qIF}vwv3K_0PXnYRPV`b(?v0dkv zm!d0uh0K>br@tmc>eVE-LZ2eg5G6s)BPpl@KEn=l~tC<7#S&Z3GwHlb4V70UPJ`%CZ zp?;xeTUadJoT1Z(i!mFjbh|#Mp9AZ&H9N?~E-!jSQ&g~6dIKTt?RrcF+cAwsfbrs8 zpLVULhmZ3#>FD%h+PJ-imAqKQo0FD*G;TqsyScNS(Sd>)Ao8@Y?{X;<_d7D(rwwA z?`9Qq1$7B!(zrRF>xNyiuM6bY>tR=bTa_-)yJe+e%oW;{y{UT(jlrRYVwY1^@-Ct_ zEPX(1`$x7h8|LzKP0@5Ke>Y&Ix!5%GN@O+X!pgSv0RFloY-2Xe=~g&ZQ?&4{m}MSf zm&tXh)sQPL+tTZ{*)Ggu>d5O_4uI(=HVxmTrrXCvI=s!c=A}3LC%H9`serGjIEJ39 z*d;=f{V|sUFW=L=bbdsJCmu0p`Oo$n!!1B;3Nn>_7VI4;kJ9hX+W6cl=2Qz){l>70 z5u38evri!%Q*tVOd}#cqPBG<0H09E5JcQ=NFtN+I4S{BudMig-U%9(*D*f)bUC;eu z3cCDf(jUk0UJw)WuBL9M z-jVD6NyMgkx*YDH)%n#IF{P`=vwATsX1A$sr&iI%`{WUu=Hwi@xWk)>bH z4ZaSJ$y>YD8jwAq*p&JTfg>`<5?s3Z{K5@`WAfszwF+leFUu)5#koS@h}3!SUJNe% zZVd0|@iBRKwOQA&X~m}0l>mq1!6d$Pj}(u4Zi&eoEag5)iBjXWb69JCJg9h!nrF5q(gKc-k9<)zddg&_yN)yC5KFn&+=#^jw^ z>3xdW)GUYKS9jTYZ|UCY>i&CU@>)&YK1l5NFKw!RZ|T)N?~Cm*x!#MB4-%V_n#Y&W z+g`e4$n`ZDVjA`fu_a1mC|xq=;+hOGt=v9KY%0zw_^bmNN}rusu|7%6^_C&Fy!|Al z%Lmr2OcIm-E7eq-Qpg#>4WTticN{8Vh#wXKT1`)I>qCQ zsbb2%y5A&r@Rv3zk*f4ZMb!(k#gym`vDI~#7qgYV80YTwgfYi|WOv2BOPB5WI9k}W z2}}3Lo-pR%tNUeQ$N$qvQ7B>QA90&va=q!{(PGp5&6oBrv1t;xO%OKzYD}P8?)Jv< zW{>7^W_K`L54R0rDz@TZ-HqOgJ$Jpr#oqX(O%UMrE)(jxT#5SCUDxEl=>CxI9UE@% zYZUQ;S34Irpe*9us*+K@rAu_3T9YtlBX(Es>QA|titM`hI#}~3j_bKon-zrJWN3qM~+?htI=9cF* zdD5KH-0~q!4mI2)N$G}(x?5$4DHOZR&&_Gn>fUy<&pzz#+&tvt&7KmlK11nWh&d%T z0qM%q2=;g@1pBy8v*2z#;c;Qh;C2A&S+UDVUA^g}#uThd|Fq@`Einw_*+|^z->S5i(;3-rd%@d zd;nx>n87|VAHi)W&L(2hY%4kmRhJW38Ag7J)k?T+Me@3jmDpv^Bt|6}Ix+o>#fi=DQ;JI?Pp+?KOVC3YFx+(YvX<#^3k!fgrC zuGREV@~-O|a}~`ubd5jW9Bz;K(5Gp|E|VMDXt>oQ)qn+Xdki_b*oJz)Ol%?pw+C)_ z#U8BR^^u8|CY$Uc&^;G!cd^jJQ^YRwFPgdd`W?f)&*65L|5;*p_3q}nd)T;rx9qIv zaGUpaqS%z8@snnQYb7{ywVU$dkeR()f_E01yP8qc;!kg z^EKSA?VK)l2{PsC)Zz;2lnc0BntF%WQ+mH!Y(fFI3BsGirZ_@rh0Ds%%ZY!7+b)m5SzO0YTSLu#dyJ!54!ykv5TDC9KMV2HMc({_M_f+R=nf+ zr`+y}-4J-+ZSzHay?tNUL85p3!15Fn`I};wuCG4V>wyNZ{D@oejwfBWU)kVYsH$|i zxLfw(=kG9?r=ZH3@9h_t*o_9axfSntQtGaHciVsc!>ae|lXe5X!(^lY0T2KI5C8!X z009sH0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009svKkpSF2u{F#r sApo}}l(6PR0&x4p)-bb#0Nj>P!kQBa!0i)T!^{!_a9ctNYfdEaUtXM_NB{r; literal 0 HcmV?d00001 diff --git a/src/TensorFlowNET.Core/APIs/tf.image.cs b/src/TensorFlowNET.Core/APIs/tf.image.cs index ac9cbc60d..41ef52967 100644 --- a/src/TensorFlowNET.Core/APIs/tf.image.cs +++ b/src/TensorFlowNET.Core/APIs/tf.image.cs @@ -339,6 +339,13 @@ public Tensor decode_image(Tensor contents, int channels = 0, TF_DataType dtype => image_ops_impl.decode_image(contents, channels: channels, dtype: dtype, name: name, expand_animations: expand_animations); + public Tensor encode_png(Tensor contents, string name = null) + => image_ops_impl.encode_png(contents, name: name); + + public Tensor encode_jpeg(Tensor contents, string name = null) + => image_ops_impl.encode_jpeg(contents, name: name); + + /// /// Convenience function to check if the 'contents' encodes a JPEG image. /// diff --git a/src/TensorFlowNET.Core/APIs/tf.io.cs b/src/TensorFlowNET.Core/APIs/tf.io.cs index be1e86e6c..ea1e44b28 100644 --- a/src/TensorFlowNET.Core/APIs/tf.io.cs +++ b/src/TensorFlowNET.Core/APIs/tf.io.cs @@ -16,6 +16,7 @@ limitations under the License. using System.Collections.Generic; using Tensorflow.IO; +using Tensorflow.Operations; namespace Tensorflow { @@ -46,6 +47,12 @@ public Operation save_v2(Tensor prefix, string[] tensor_names, public Tensor[] restore_v2(Tensor prefix, string[] tensor_names, string[] shape_and_slices, TF_DataType[] dtypes, string name = null) => ops.restore_v2(prefix, tensor_names, shape_and_slices, dtypes, name: name); + + public Operation write_file(string filename, Tensor conentes, string name = null) + => write_file(Tensorflow.ops.convert_to_tensor(filename, TF_DataType.TF_STRING), conentes, name); + + public Operation write_file(Tensor filename, Tensor conentes, string name = null) + => gen_ops.write_file(filename, conentes, name); } public GFile gfile = new GFile(); diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index a8141d354..3fd98e7a8 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -55,6 +55,12 @@ public ILayer Conv1D(int filters, string kernel_initializer = "glorot_uniform", string bias_initializer = "zeros"); + public ILayer Conv2D(int filters, + Shape kernel_size = null, + Shape strides = null, + string padding = "valid" + ); + public ILayer Conv2D(int filters, Shape kernel_size = null, Shape strides = null, diff --git a/src/TensorFlowNET.Core/Operations/image_ops_impl.cs b/src/TensorFlowNET.Core/Operations/image_ops_impl.cs index 318b8b142..f1aff28ee 100644 --- a/src/TensorFlowNET.Core/Operations/image_ops_impl.cs +++ b/src/TensorFlowNET.Core/Operations/image_ops_impl.cs @@ -102,7 +102,10 @@ internal static Operation[] _CheckAtLeast3DImage(Tensor image, bool require_stat { throw new ValueError("\'image\' must be fully defined."); } - var dims = image_shape["-3:"]; + var dims = new Shape(new[] { + image_shape.dims[image_shape.dims.Length - 3], + image_shape.dims[image_shape.dims.Length - 2], + image_shape.dims[image_shape.dims.Length - 1]}); foreach (var dim in dims.dims) { if (dim == 0) @@ -112,16 +115,18 @@ internal static Operation[] _CheckAtLeast3DImage(Tensor image, bool require_stat } var image_shape_last_three_elements = new Shape(new[] { - image_shape.dims[image_shape.dims.Length - 1], + image_shape.dims[image_shape.dims.Length - 3], image_shape.dims[image_shape.dims.Length - 2], - image_shape.dims[image_shape.dims.Length - 3]}); + image_shape.dims[image_shape.dims.Length - 1]}); if (!image_shape_last_three_elements.IsFullyDefined) { Tensor image_shape_ = array_ops.shape(image); - var image_shape_return = tf.constant(new[] { - image_shape_.dims[image_shape.dims.Length - 1], - image_shape_.dims[image_shape.dims.Length - 2], - image_shape_.dims[image_shape.dims.Length - 3]}); + var image_shape_return = tf.slice(image_shape_, new[] { Math.Max(image_shape.dims.Length - 3, 0) }, new[] { 3 }); + + //var image_shape_return = tf.constant(new[] { + // image_shape_.dims[image_shape_.dims.Length - 3], + // image_shape_.dims[image_shape_.dims.Length - 2], + // image_shape_.dims[image_shape_.dims.Length - 1]}); return new Operation[] { check_ops.assert_positive( @@ -209,10 +214,10 @@ internal static Tensor _random_flip(Tensor image, int flip_index, int seed, stri } public static Tensor flip_left_right(Tensor image) - => _flip(image, 0, "flip_left_right"); + => _flip(image, 1, "flip_left_right"); public static Tensor flip_up_down(Tensor image) - => _flip(image, 1, "flip_up_down"); + => _flip(image, 0, "flip_up_down"); internal static Tensor _flip(Tensor image, int flip_index, string scope_name) { @@ -223,11 +228,11 @@ internal static Tensor _flip(Tensor image, int flip_index, string scope_name) Shape shape = image.shape; if (shape.ndim == 3 || shape.ndim == Unknown) { - return fix_image_flip_shape(image, gen_array_ops.reverse(image, ops.convert_to_tensor(new int[] { flip_index }))); + return fix_image_flip_shape(image, gen_array_ops.reverse_v2(image, ops.convert_to_tensor(new int[] { flip_index }))); } else if (shape.ndim == 4) { - return gen_array_ops.reverse_v2(image, ops.convert_to_tensor(new[] { (flip_index + 1) % 2 })); + return gen_array_ops.reverse_v2(image, ops.convert_to_tensor(new[] { flip_index + 1 })); } else { @@ -2047,6 +2052,22 @@ internal static (Tensor, Tensor) non_max_suppression_padded_v1(Tensor boxes, Ten }); } + public static Tensor encode_jpeg(Tensor contents, string name = null) + { + return tf_with(ops.name_scope(name, "encode_jpeg"), scope => + { + return gen_ops.encode_jpeg(contents, name:name); + }); + } + + public static Tensor encode_png(Tensor contents, string name = null) + { + return tf_with(ops.name_scope(name, "encode_png"), scope => + { + return gen_ops.encode_png(contents, name: name); + }); + } + public static Tensor is_jpeg(Tensor contents, string name = null) { return tf_with(ops.name_scope(name, "is_jpeg"), scope => diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index 95828fbf7..bcc19dc22 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -112,7 +112,28 @@ public ILayer Conv1D(int filters, KernelInitializer = GetInitializerByName(kernel_initializer), BiasInitializer = GetInitializerByName(bias_initializer) }); - + public ILayer Conv2D(int filters, + Shape kernel_size = null, + Shape strides = null, + string padding = "valid") + => new Conv2D(new Conv2DArgs + { + Rank = 2, + Filters = filters, + KernelSize = (kernel_size == null) ? (5, 5) : kernel_size, + Strides = strides == null ? (1, 1) : strides, + Padding = padding, + DataFormat = null, + DilationRate = (1, 1), + Groups = 1, + UseBias = false, + KernelRegularizer = null, + KernelInitializer =tf.glorot_uniform_initializer, + BiasInitializer = tf.zeros_initializer, + BiasRegularizer = null, + ActivityRegularizer = null, + Activation = keras.activations.Linear, + }); /// /// 2D convolution layer (e.g. spatial convolution over images). /// This layer creates a convolution kernel that is convolved with the layer input to produce a tensor of outputs. diff --git a/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs b/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs index d671b6096..127b65bf6 100644 --- a/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs +++ b/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs @@ -4,6 +4,7 @@ using Tensorflow; using static Tensorflow.Binding; using System; +using System.IO; namespace TensorFlowNET.UnitTest { @@ -164,5 +165,94 @@ public void TestCropAndResize() Assert.AreEqual(result.size, 16ul); Assert.AreEqual(result[0, 0, 0, 0], 12f); } + + [TestMethod] + public void ImageSaveTest() + { + var imgPath = TestHelper.GetFullPathFromDataDir("img001.bmp"); + var jpegImgPath = TestHelper.GetFullPathFromDataDir("img001.jpeg"); + var pngImgPath = TestHelper.GetFullPathFromDataDir("img001.png"); + + File.Delete(jpegImgPath); + File.Delete(pngImgPath); + + var contents = tf.io.read_file(imgPath); + var bmp = tf.image.decode_image(contents); + Assert.AreEqual(bmp.name, "decode_image/DecodeImage:0"); + + var jpeg = tf.image.encode_jpeg(bmp); + var op1 = tf.io.write_file(jpegImgPath, jpeg); + + var png = tf.image.encode_png(bmp); + var op2 = tf.io.write_file(pngImgPath, png); + + this.session().run(op1); + this.session().run(op2); + + Assert.IsTrue(File.Exists(jpegImgPath), "not find file:" + jpegImgPath); + Assert.IsTrue(File.Exists(pngImgPath), "not find file:" + pngImgPath); + + // 如果要测试图片正确性，需要注释下面两行代码 + File.Delete(jpegImgPath); + File.Delete(pngImgPath); + } + + [TestMethod] + public void ImageFlipTest() + { + var imgPath = TestHelper.GetFullPathFromDataDir("img001.bmp"); + + var contents = tf.io.read_file(imgPath); + var bmp = tf.image.decode_image(contents); + + // 左右翻转 + var lrImgPath = TestHelper.GetFullPathFromDataDir("img001_lr.png"); + File.Delete(lrImgPath); + + var lr = tf.image.flip_left_right(bmp); + var png = tf.image.encode_png(lr); + var op = tf.io.write_file(lrImgPath, png); + this.session().run(op); + + Assert.IsTrue(File.Exists(lrImgPath), "not find file:" + lrImgPath); + + // 上下翻转 + var updownImgPath = TestHelper.GetFullPathFromDataDir("img001_updown.png"); + File.Delete(updownImgPath); + + var updown = tf.image.flip_up_down(bmp); + var pngupdown = tf.image.encode_png(updown); + var op2 = tf.io.write_file(updownImgPath, pngupdown); + this.session().run(op2); + Assert.IsTrue(File.Exists(updownImgPath)); + + + // 暂时先人工观测图片是否翻转，观测时需要删除下面这两行代码 + File.Delete(lrImgPath); + File.Delete(updownImgPath); + + // 多图翻转 + // 目前直接通过 bmp 拿到 shape ，这里先用默认定义图片大小来构建了 + var mImg = tf.stack(new[] { bmp, lr }, axis:0); + print(mImg.shape); + + var up2 = tf.image.flip_up_down(mImg); + + var updownImgPath_m1 = TestHelper.GetFullPathFromDataDir("img001_m_ud.png"); // 直接上下翻转 + File.Delete(updownImgPath_m1); + + var img001_updown_m2 = TestHelper.GetFullPathFromDataDir("img001_m_lr_ud.png"); // 先左右再上下 + File.Delete(img001_updown_m2); + + var png2 = tf.image.encode_png(up2[0]); + tf.io.write_file(updownImgPath_m1, png2); + + png2 = tf.image.encode_png(up2[1]); + tf.io.write_file(img001_updown_m2, png2); + + // 如果要测试图片正确性，需要注释下面两行代码 + File.Delete(updownImgPath_m1); + File.Delete(img001_updown_m2); + } } } diff --git a/test/TensorFlowNET.UnitTest/ManagedAPI/ArrayOpsTest.cs b/test/TensorFlowNET.UnitTest/ManagedAPI/ArrayOpsTest.cs index 675689bb1..e25c9779d 100644 --- a/test/TensorFlowNET.UnitTest/ManagedAPI/ArrayOpsTest.cs +++ b/test/TensorFlowNET.UnitTest/ManagedAPI/ArrayOpsTest.cs @@ -3,6 +3,7 @@ using Tensorflow; using static Tensorflow.Binding; using System.Linq; +using Tensorflow.Operations; namespace TensorFlowNET.UnitTest.ManagedAPI { @@ -105,5 +106,321 @@ public void ReverseArray() Assert.IsTrue(Equal(a[0].ToArray().Reverse().ToArray(), b[0].ToArray())); Assert.IsTrue(Equal(a[1].ToArray().Reverse().ToArray(), b[1].ToArray())); } + + [TestMethod] + public void ReverseImgArray3D() + { + // 创建 sourceImg 数组 + var sourceImgArray = new float[,,] { + { + { 237, 28, 36 }, + { 255, 255, 255 }, + { 255, 255, 255 } + }, + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + } + }; + var sourceImg = ops.convert_to_tensor(sourceImgArray); + + // 创建 lrImg 数组 + var lrImgArray = new float[,,] { + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 237, 28, 36 } + }, + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + } + }; + var lrImg = ops.convert_to_tensor(lrImgArray); + + var lr = tf.image.flip_left_right(sourceImg); + Assert.IsTrue(Equal(lrImg.numpy().ToArray(), lr.numpy().ToArray()), "tf.image.flip_left_right fail."); + + var lr2 = tf.reverse(sourceImg, 1); + Assert.IsTrue(Equal(lrImg.numpy().ToArray(), lr2.numpy().ToArray()), "tf.reverse (axis=1) fail."); + + var lr3 = gen_array_ops.reverse_v2(sourceImg, ops.convert_to_tensor(new[] { 1 })); + Assert.IsTrue(Equal(lrImg.numpy().ToArray(), lr3.numpy().ToArray()), "gen_array_ops.reverse_v2 axis=1 fail."); + + // 创建 udImg 数组 + var udImgArray = new float[,,] { + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + }, + { + { 237, 28, 36 }, + { 255, 255, 255 }, + { 255, 255, 255 } + } + }; + var udImg = ops.convert_to_tensor(udImgArray); + + var ud = tf.image.flip_up_down(sourceImg); + Assert.IsTrue(Equal(udImg.numpy().ToArray(), ud.numpy().ToArray()), "tf.image.flip_up_down fail."); + + var ud2 = tf.reverse(sourceImg, new Axis(0)); + Assert.IsTrue(Equal(udImg.numpy().ToArray(), ud2.numpy().ToArray()), "tf.reverse (axis=0) fail."); + + var ud3 = gen_array_ops.reverse_v2(sourceImg, ops.convert_to_tensor(new[] { 0 })); + Assert.IsTrue(Equal(udImg.numpy().ToArray(), ud3.numpy().ToArray()), "gen_array_ops.reverse_v2 axis=0 fail."); + } + + [TestMethod] + public void ReverseImgArray4D() + { + // 原图左上角，加一张左右翻转后的图片 + var m = new float[,,,] { + { + { + { 237, 28, 36 }, + { 255, 255, 255 }, + { 255, 255, 255 } + }, + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + } + }, + { + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 237, 28, 36 } + }, + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + } + } + }; + var sourceImg = ops.convert_to_tensor(m); + + var lrArray = new float[,,,] { + { + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 237, 28, 36 }, + }, + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + } + }, + { + { + { 237, 28, 36 }, + { 255, 255, 255 }, + { 255, 255, 255 }, + }, + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + } + } + }; + var lrImg = ops.convert_to_tensor(lrArray); + + // 创建 ud 数组 + var udArray = new float[,,,] { + { + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + }, + { + { 237, 28, 36 }, + { 255, 255, 255 }, + { 255, 255, 255 } + } + }, + { + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + }, + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 237, 28, 36 } + } + } + }; + var udImg = ops.convert_to_tensor(udArray); + + var ud3 = gen_array_ops.reverse_v2(sourceImg, ops.convert_to_tensor(new[] { 1 })); + Assert.IsTrue(Equal(udImg.numpy().ToArray(), ud3.numpy().ToArray()), "gen_array_ops.reverse_v2 axis=1 fail."); + + var ud2 = tf.reverse(sourceImg, new Axis(1)); + Assert.IsTrue(Equal(udImg.numpy().ToArray(), ud2.numpy().ToArray()), "tf.reverse (axis=1) fail."); + + var ud = tf.image.flip_up_down(sourceImg); + Assert.IsTrue(Equal(udImg.numpy().ToArray(), ud.numpy().ToArray()), "tf.image.flip_up_down fail."); + + // 左右翻转 + var lr = tf.image.flip_left_right(sourceImg); + Assert.IsTrue(Equal(lrImg.numpy().ToArray(), lr.numpy().ToArray()), "tf.image.flip_left_right fail."); + + var lr2 = tf.reverse(sourceImg, 0); + Assert.IsTrue(Equal(lrImg.numpy().ToArray(), lr2.numpy().ToArray()), "tf.reverse (axis=1) fail."); + + var lr3 = gen_array_ops.reverse_v2(sourceImg, ops.convert_to_tensor(new[] { 0 })); + Assert.IsTrue(Equal(lrImg.numpy().ToArray(), lr3.numpy().ToArray()), "gen_array_ops.reverse_v2 axis=1 fail."); + + } + + [TestMethod] + public void ReverseImgArray4D_3x3() + { + // 原图左上角，加一张左右翻转后的图片 + var m = new float[,,,] { + { + { + { 237, 28, 36 }, + { 255, 255, 255 }, + { 255, 255, 255 } + }, + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + }, + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + } + }, + { + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 237, 28, 36 } + }, + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + }, + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + } + } + }; + var sourceImg = ops.convert_to_tensor(m); + + var lrArray = new float[,,,] { + { + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 237, 28, 36 }, + }, + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + }, + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + } + }, + { + { + { 237, 28, 36 }, + { 255, 255, 255 }, + { 255, 255, 255 }, + }, + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + }, + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + } + } + }; + var lrImg = ops.convert_to_tensor(lrArray); + + // 创建 ud 数组 + var udArray = new float[,,,] { + { + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + }, + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + }, + { + { 237, 28, 36 }, + { 255, 255, 255 }, + { 255, 255, 255 } + } + }, + { { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + }, + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 255, 255, 255 } + }, + { + { 255, 255, 255 }, + { 255, 255, 255 }, + { 237, 28, 36 } + } + } + }; + var udImg = ops.convert_to_tensor(udArray); + + var ud3 = gen_array_ops.reverse_v2(sourceImg, ops.convert_to_tensor(new[] { 1 })); + Assert.IsTrue(Equal(udImg.numpy().ToArray(), ud3.numpy().ToArray()), "gen_array_ops.reverse_v2 axis=1 fail."); + + var ud2 = tf.reverse(sourceImg, new Axis(1)); + Assert.IsTrue(Equal(udImg.numpy().ToArray(), ud2.numpy().ToArray()), "tf.reverse (axis=1) fail."); + + var ud = tf.image.flip_up_down(sourceImg); + Assert.IsTrue(Equal(udImg.numpy().ToArray(), ud.numpy().ToArray()), "tf.image.flip_up_down fail."); + + // 左右翻转 + var lr = tf.image.flip_left_right(sourceImg); + Assert.IsTrue(Equal(lrImg.numpy().ToArray(), lr.numpy().ToArray()), "tf.image.flip_left_right fail."); + + var lr2 = tf.reverse(sourceImg, 0); + Assert.IsTrue(Equal(lrImg.numpy().ToArray(), lr2.numpy().ToArray()), "tf.reverse (axis=1) fail."); + + var lr3 = gen_array_ops.reverse_v2(sourceImg, ops.convert_to_tensor(new[] { 0 })); + Assert.IsTrue(Equal(lrImg.numpy().ToArray(), lr3.numpy().ToArray()), "gen_array_ops.reverse_v2 axis=1 fail."); + + } } } diff --git a/test/TensorFlowNET.UnitTest/NumPy/ShapeTest.cs b/test/TensorFlowNET.UnitTest/NumPy/ShapeTest.cs new file mode 100644 index 000000000..f5a8685be --- /dev/null +++ b/test/TensorFlowNET.UnitTest/NumPy/ShapeTest.cs @@ -0,0 +1,44 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Tensorflow.NumPy; +using System; +using System.Linq; +using static Tensorflow.Binding; +using Tensorflow; + +namespace TensorFlowNET.UnitTest.NumPy +{ + [TestClass] + public class ShapeTest : EagerModeTestBase + { + [Ignore] + [TestMethod] + public unsafe void ShapeGetLastElements() + { + // test code from function _CheckAtLeast3DImage + // 之前的 _CheckAtLeast3DImage 有bug，现在通过测试，下面的代码是正确的 + // todo: shape["-3:"] 的写法，目前有bug，需要修复，单元测试等修复后再放开，暂时先忽略测试 + + var image_shape = new Shape(new[] { 32, 64, 3 }); + var image_shape_4d = new Shape(new[] { 4, 64, 32, 3 }); + + var image_shape_last_three_elements = new Shape(new[] { + image_shape.dims[image_shape.dims.Length - 3], + image_shape.dims[image_shape.dims.Length - 2], + image_shape.dims[image_shape.dims.Length - 1]}); + + var image_shape_last_three_elements2 = image_shape["-3:"]; + + Assert.IsTrue(Equal(image_shape_last_three_elements.dims, image_shape_last_three_elements2.dims), "3dims get fail."); + + var image_shape_last_three_elements_4d = new Shape(new[] { + image_shape_4d.dims[image_shape_4d.dims.Length - 3], + image_shape_4d.dims[image_shape_4d.dims.Length - 2], + image_shape_4d.dims[image_shape_4d.dims.Length - 1]}); + + var image_shape_last_three_elements2_4d = image_shape_4d["-3:"]; + + Assert.IsTrue(Equals(image_shape_last_three_elements_4d.dims, image_shape_last_three_elements2_4d.dims), "4dims get fail."); + } + + } +} \ No newline at end of file From baf620a3e875e7cf6cfa82eb3c56392e2b7fab9a Mon Sep 17 00:00:00 2001 From: dogvane Date: Sun, 8 Oct 2023 22:06:15 +0800 Subject: [PATCH 54/98] =?UTF-8?q?=E8=A7=A3=E5=86=B3keras=E6=A8=A1=E5=BC=8F?= =?UTF-8?q?=E4=B8=8B=EF=BC=8C=E4=BD=BF=E7=94=A8GPU=E8=AE=AD=E7=BB=83?= =?UTF-8?q?=E6=97=B6=E4=BC=9A=E7=88=86=E6=98=BE=E5=AD=98=E7=9A=84bug?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 观察到的现象是，一些模型增大batchsize后，会在首个epoch的中途爆显存不足，只要过了一个epoch后，就能完整训练。同样的batchsize在python下能设置大得多的值。 最后使用最小训练代码分析出，是每个step之后，图片加载到显存里的数据没有释放导致的。 在寻找释放显存接口没有结果的时候，直接使用了GC.Collect();可以让显存主动回收。 因此当前的修复方案是在每个step里，都执行一次 GC.Collect(); 用来释放显存资源。 --- src/TensorFlowNET.Core/Keras/Engine/IModel.cs | 23 +++++++++++++++++++ .../Engine/Model.Evaluate.cs | 3 +++ src/TensorFlowNET.Keras/Engine/Model.Fit.cs | 12 +++++----- .../Engine/Model.Predict.cs | 2 +- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/TensorFlowNET.Core/Keras/Engine/IModel.cs b/src/TensorFlowNET.Core/Keras/Engine/IModel.cs index 1840f88b9..889c76d91 100644 --- a/src/TensorFlowNET.Core/Keras/Engine/IModel.cs +++ b/src/TensorFlowNET.Core/Keras/Engine/IModel.cs @@ -24,6 +24,7 @@ ICallback fit(NDArray x, NDArray y, List callbacks = null, float validation_split = 0f, ValidationDataPack validation_data = null, + int validation_step = 10, bool shuffle = true, Dictionary class_weight = null, NDArray sample_weight = null, @@ -47,6 +48,20 @@ ICallback fit(IEnumerable x, NDArray y, int workers = 1, bool use_multiprocessing = false); + public ICallback fit(IDatasetV2 dataset, + int batch_size = -1, + int epochs = 1, + int verbose = 1, + List callbacks = null, + IDatasetV2 validation_data = null, + int validation_step = 10, // 间隔多少次会进行一次验证 + bool shuffle = true, + Dictionary class_weight = null, + int initial_epoch = 0, + int max_queue_size = 10, + int workers = 1, + bool use_multiprocessing = false); + void save(string filepath, bool overwrite = true, bool include_optimizer = true, @@ -85,6 +100,14 @@ Tensors predict(Tensors x, int workers = 1, bool use_multiprocessing = false); + public Tensors predict(IDatasetV2 dataset, + int batch_size = -1, + int verbose = 0, + int steps = -1, + int max_queue_size = 10, + int workers = 1, + bool use_multiprocessing = false); + void summary(int line_length = -1, float[] positions = null); IKerasConfig get_config(); diff --git a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs index 94a2e6646..474d5e5a5 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs @@ -132,6 +132,7 @@ Dictionary evaluate(DataHandler data_handler, CallbackList callba var end_step = step + data_handler.StepIncrement; if (!is_val) callbacks.on_test_batch_end(end_step, logs); + GC.Collect(); } } callbacks.on_test_end(logs); @@ -167,7 +168,9 @@ Dictionary test_step_multi_inputs_function(DataHandler data_handl Dictionary test_step(DataHandler data_handler, Tensors x, Tensors y) { (x,y) = data_handler.DataAdapter.Expand1d(x, y); + var y_pred = Apply(x, training: false); + var loss = compiled_loss.Call(y, y_pred); compiled_metrics.update_state(y, y_pred); return metrics.Select(x => (x.Name, x.result())).ToDictionary(x => x.Item1, x => (float)x.Item2); diff --git a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs index 689fc9fb8..d61211c71 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs @@ -41,6 +41,7 @@ public ICallback fit(NDArray x, NDArray y, List callbacks = null, float validation_split = 0f, ValidationDataPack validation_data = null, + int validation_step = 10, bool shuffle = true, Dictionary class_weight = null, NDArray sample_weight = null, @@ -147,7 +148,7 @@ public ICallback fit(IEnumerable x, NDArray y, } } - public History fit(IDatasetV2 dataset, + public ICallback fit(IDatasetV2 dataset, int batch_size = -1, int epochs = 1, int verbose = 1, @@ -156,7 +157,6 @@ public History fit(IDatasetV2 dataset, int validation_step = 10, bool shuffle = true, Dictionary class_weight = null, - NDArray sample_weight = null, int initial_epoch = 0, int max_queue_size = 10, int workers = 1, @@ -170,7 +170,7 @@ public History fit(IDatasetV2 dataset, InitialEpoch = initial_epoch, Epochs = epochs, Shuffle = shuffle, - SampleWeight = sample_weight, + ClassWeight = class_weight, MaxQueueSize = max_queue_size, Workers = workers, UseMultiprocessing = use_multiprocessing, @@ -218,6 +218,7 @@ History FitInternal(DataHandler data_handler, int epochs, int validation_step, i var end_step = step + data_handler.StepIncrement; End_step = end_step; callbacks.on_train_batch_end(end_step, logs); + GC.Collect(); } if (validation_data != null) @@ -233,11 +234,10 @@ History FitInternal(DataHandler data_handler, int epochs, int validation_step, i callbacks.on_train_batch_end(End_step, logs); } + GC.Collect(); callbacks.on_epoch_end(epoch, logs); - GC.Collect(); - GC.WaitForPendingFinalizers(); if (stop_training) { break; @@ -282,6 +282,7 @@ History FitInternal(DataHandler data_handler, int epochs, int verbose, List { { "outputs", batch_outputs } }); + GC.Collect(); } } From 93a242c08a330399328c8a1190f6b0d46308a226 Mon Sep 17 00:00:00 2001 From: Jucko13 Date: Tue, 10 Oct 2023 16:53:04 +0200 Subject: [PATCH 55/98] Implemented support for loading Concatenate layers model.load_model now supports loading of concatenate layers. python tensorflow exports concatenate layers in an extra nested array in the manifest so added a check for that in generic_utils.cs. Concatenate was missing the build=true, this fix prevents the layer being build multiple times. Concatenate has 2 or more input nodes so List was required instead of just NodeConfig in Functional.FromConfig.cs. Added missing axis JsonProperty attribute for MergeArgs (used by Concatenate) --- .../Keras/ArgsDefinition/Merging/MergeArgs.cs | 6 ++-- .../Engine/Functional.FromConfig.cs | 30 +++++++++++-------- .../Layers/Merging/Concatenate.cs | 1 + .../Utils/generic_utils.cs | 13 +++++++- 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/MergeArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/MergeArgs.cs index 0140b3dd0..9bcf1908e 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/MergeArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/MergeArgs.cs @@ -1,13 +1,15 @@ -using System; +using Newtonsoft.Json; +using System; using System.Collections.Generic; using System.Text; namespace Tensorflow.Keras.ArgsDefinition { // TODO: complete the implementation - public class MergeArgs : LayerArgs + public class MergeArgs : AutoSerializeLayerArgs { public Tensors Inputs { get; set; } + [JsonProperty("axis")] public int Axis { get; set; } } } diff --git a/src/TensorFlowNET.Keras/Engine/Functional.FromConfig.cs b/src/TensorFlowNET.Keras/Engine/Functional.FromConfig.cs index 7b826af8e..375fc9106 100644 --- a/src/TensorFlowNET.Keras/Engine/Functional.FromConfig.cs +++ b/src/TensorFlowNET.Keras/Engine/Functional.FromConfig.cs @@ -30,7 +30,7 @@ public static (Tensors, Tensors, Dictionary) reconstruct_from_co created_layers = created_layers ?? new Dictionary(); var node_index_map = new Dictionary<(string, int), int>(); var node_count_by_layer = new Dictionary(); - var unprocessed_nodes = new Dictionary(); + var unprocessed_nodes = new Dictionary>(); // First, we create all layers and enqueue nodes to be processed foreach (var layer_data in config.Layers) process_layer(created_layers, layer_data, unprocessed_nodes, node_count_by_layer); @@ -79,7 +79,7 @@ public static (Tensors, Tensors, Dictionary) reconstruct_from_co static void process_layer(Dictionary created_layers, LayerConfig layer_data, - Dictionary unprocessed_nodes, + Dictionary> unprocessed_nodes, Dictionary node_count_by_layer) { ILayer layer = null; @@ -92,32 +92,38 @@ static void process_layer(Dictionary created_layers, created_layers[layer_name] = layer; } - node_count_by_layer[layer] = _should_skip_first_node(layer) ? 1 : 0; + node_count_by_layer[layer] = layer_data.InboundNodes.Count - (_should_skip_first_node(layer) ? 1 : 0); var inbound_nodes_data = layer_data.InboundNodes; foreach (var node_data in inbound_nodes_data) { if (!unprocessed_nodes.ContainsKey(layer)) - unprocessed_nodes[layer] = node_data; + unprocessed_nodes[layer] = new List() { node_data }; else - unprocessed_nodes.Add(layer, node_data); + unprocessed_nodes[layer].Add(node_data); } } static void process_node(ILayer layer, - NodeConfig node_data, + List nodes_data, Dictionary created_layers, Dictionary node_count_by_layer, Dictionary<(string, int), int> node_index_map) { + var input_tensors = new List(); - var inbound_layer_name = node_data.Name; - var inbound_node_index = node_data.NodeIndex; - var inbound_tensor_index = node_data.TensorIndex; - var inbound_layer = created_layers[inbound_layer_name]; - var inbound_node = inbound_layer.InboundNodes[inbound_node_index]; - input_tensors.Add(inbound_node.Outputs[inbound_node_index]); + for (int i = 0; i < nodes_data.Count; i++) + { + var node_data = nodes_data[i]; + var inbound_layer_name = node_data.Name; + var inbound_node_index = node_data.NodeIndex; + var inbound_tensor_index = node_data.TensorIndex; + + var inbound_layer = created_layers[inbound_layer_name]; + var inbound_node = inbound_layer.InboundNodes[inbound_node_index]; + input_tensors.Add(inbound_node.Outputs[inbound_node_index]); + } var output_tensors = layer.Apply(input_tensors); diff --git a/src/TensorFlowNET.Keras/Layers/Merging/Concatenate.cs b/src/TensorFlowNET.Keras/Layers/Merging/Concatenate.cs index a2a8286ba..fa82426ce 100644 --- a/src/TensorFlowNET.Keras/Layers/Merging/Concatenate.cs +++ b/src/TensorFlowNET.Keras/Layers/Merging/Concatenate.cs @@ -39,6 +39,7 @@ public override void build(KerasShapesWrapper input_shape) shape_set.Add(shape); }*/ _buildInputShape = input_shape; + built = true; } protected override Tensors _merge_function(Tensors inputs) diff --git a/src/TensorFlowNET.Keras/Utils/generic_utils.cs b/src/TensorFlowNET.Keras/Utils/generic_utils.cs index 5402f4995..20937e2e5 100644 --- a/src/TensorFlowNET.Keras/Utils/generic_utils.cs +++ b/src/TensorFlowNET.Keras/Utils/generic_utils.cs @@ -112,12 +112,23 @@ public static FunctionalConfig deserialize_model_config(JToken json) foreach (var token in layersToken) { var args = deserialize_layer_args(token["class_name"].ToObject(), token["config"]); + + List nodeConfig = null; //python tensorflow sometimes exports inbound nodes in an extra nested array + if (token["inbound_nodes"].Count() > 0 && token["inbound_nodes"][0].Count() > 0 && token["inbound_nodes"][0][0].Count() > 0) + { + nodeConfig = token["inbound_nodes"].ToObject>>().FirstOrDefault() ?? new List(); + } + else + { + nodeConfig = token["inbound_nodes"].ToObject>(); + } + config.Layers.Add(new LayerConfig() { Config = args, Name = token["name"].ToObject(), ClassName = token["class_name"].ToObject(), - InboundNodes = token["inbound_nodes"].ToObject>() + InboundNodes = nodeConfig, }); } config.InputLayers = json["input_layers"].ToObject>(); From 9f0ffa4bc83b181ddd525cf1b90d77a32e073fa3 Mon Sep 17 00:00:00 2001 From: Jucko13 Date: Tue, 10 Oct 2023 17:02:22 +0200 Subject: [PATCH 56/98] Implemented unittests for Concatenate layers and calls The loading and saving of a simple model with a Concatenate layer is tested to check if the model is the same after reloading. Implemented missing axis parameter for np.stack (added some handy tuple calls too like the np.concatenate example). --- .../NumPy/Numpy.Manipulation.cs | 9 ++++ .../Layers/Layers.Merging.Test.cs | 15 ++++--- .../Model/ModelLoadTest.cs | 43 +++++++++++++++++++ 3 files changed, 62 insertions(+), 5 deletions(-) diff --git a/src/TensorFlowNET.Core/NumPy/Numpy.Manipulation.cs b/src/TensorFlowNET.Core/NumPy/Numpy.Manipulation.cs index 940856056..5e2574170 100644 --- a/src/TensorFlowNET.Core/NumPy/Numpy.Manipulation.cs +++ b/src/TensorFlowNET.Core/NumPy/Numpy.Manipulation.cs @@ -30,6 +30,15 @@ public static NDArray concatenate((NDArray, NDArray) tuple, int axis = 0) [AutoNumPy] public static NDArray stack(params NDArray[] arrays) => new NDArray(array_ops.stack(arrays)); + [AutoNumPy] + public static NDArray stack(NDArray[] arrays, int axis = 0) => new NDArray(array_ops.stack(arrays, axis)); + + [AutoNumPy] + public static NDArray stack((NDArray, NDArray) tuple, int axis = 0) => new NDArray(array_ops.stack(new[] { tuple.Item1, tuple.Item2 }, axis)); + + [AutoNumPy] + public static NDArray stack((NDArray, NDArray, NDArray) tuple, int axis = 0) => new NDArray(array_ops.stack(new[] { tuple.Item1, tuple.Item2, tuple.Item3 }, axis)); + [AutoNumPy] public static NDArray moveaxis(NDArray array, Axis source, Axis destination) => new NDArray(array_ops.moveaxis(array, source, destination)); } diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/Layers.Merging.Test.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/Layers.Merging.Test.cs index 36e44e482..9bc2fa767 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/Layers.Merging.Test.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/Layers.Merging.Test.cs @@ -1,4 +1,5 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Collections.Generic; using Tensorflow.NumPy; using static Tensorflow.KerasApi; @@ -8,12 +9,16 @@ namespace Tensorflow.Keras.UnitTest.Layers public class LayersMergingTest : EagerModeTestBase { [TestMethod] - public void Concatenate() + [DataRow(1, 4, 1, 5)] + [DataRow(2, 2, 2, 5)] + [DataRow(3, 2, 1, 10)] + public void Concatenate(int axis, int shapeA, int shapeB, int shapeC) { - var x = np.arange(20).reshape((2, 2, 5)); - var y = np.arange(20, 30).reshape((2, 1, 5)); - var z = keras.layers.Concatenate(axis: 1).Apply(new Tensors(x, y)); - Assert.AreEqual((2, 3, 5), z.shape); + var x = np.arange(10).reshape((1, 2, 1, 5)); + var y = np.arange(10, 20).reshape((1, 2, 1, 5)); + var z = keras.layers.Concatenate(axis: axis).Apply(new Tensors(x, y)); + Assert.AreEqual((1, shapeA, shapeB, shapeC), z.shape); } + } } diff --git a/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs b/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs index cb570fc0c..53a67cbfa 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs @@ -1,10 +1,13 @@ using Microsoft.VisualStudio.TestPlatform.Utilities; using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json.Linq; using System.Linq; +using System.Xml.Linq; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Optimizers; using Tensorflow.Keras.UnitTest.Helpers; using Tensorflow.NumPy; +using static HDF.PInvoke.H5Z; using static Tensorflow.Binding; using static Tensorflow.KerasApi; @@ -124,4 +127,44 @@ public void TestModelBeforeTF2_5() var model = tf.saved_model.load(@"D:\development\temp\saved_model") as Tensorflow.Keras.Engine.Model; model.summary(); } + + + + [TestMethod] + public void CreateConcatenateModelSaveAndLoad() + { + // a small demo model that is just here to see if the axis value for the concatenate method is saved and loaded. + var input_layer = tf.keras.layers.Input((8, 8, 5)); + + var conv1 = tf.keras.layers.Conv2D(2, kernel_size: 3, activation: "relu", padding: "same"/*, data_format: "_conv_1"*/).Apply(input_layer); + conv1.Name = "conv1"; + + var conv2 = tf.keras.layers.Conv2D(2, kernel_size: 3, activation: "relu", padding: "same"/*, data_format: "_conv_2"*/).Apply(input_layer); + conv2.Name = "conv2"; + + var concat1 = tf.keras.layers.Concatenate(axis: 3).Apply((conv1, conv2)); + concat1.Name = "concat1"; + + var model = tf.keras.Model(input_layer, concat1); + model.compile(tf.keras.optimizers.Adam(), tf.keras.losses.CategoricalCrossentropy()); + + model.save(@"Assets/concat_axis3_model"); + + + var tensorInput = np.arange(320).reshape((1, 8, 8, 5)).astype(TF_DataType.TF_FLOAT); + + var tensors1 = model.predict(tensorInput); + + Assert.AreEqual((1, 8, 8, 4), tensors1.shape); + + model = null; + keras.backend.clear_session(); + + var model2 = tf.keras.models.load_model(@"Assets/concat_axis3_model"); + + var tensors2 = model2.predict(tensorInput); + + Assert.AreEqual(tensors1.shape, tensors2.shape); + } + } From ec4f372a29b5cbc5fe6c0d6b8414ddb48c22e548 Mon Sep 17 00:00:00 2001 From: dogvane Date: Mon, 16 Oct 2023 11:22:58 +0800 Subject: [PATCH 57/98] add relu6 --- src/TensorFlowNET.Core/APIs/tf.nn.cs | 5 ++++ .../Keras/Activations/Activations.cs | 1 + .../Keras/Layers/ILayersApi.cs | 3 +++ src/TensorFlowNET.Keras/Activations.cs | 7 ++++++ .../Layers/Activation/ReLu6.cs | 25 +++++++++++++++++++ src/TensorFlowNET.Keras/Layers/LayersApi.cs | 9 +++++++ 6 files changed, 50 insertions(+) create mode 100644 src/TensorFlowNET.Keras/Layers/Activation/ReLu6.cs diff --git a/src/TensorFlowNET.Core/APIs/tf.nn.cs b/src/TensorFlowNET.Core/APIs/tf.nn.cs index 397c68c7c..112c48628 100644 --- a/src/TensorFlowNET.Core/APIs/tf.nn.cs +++ b/src/TensorFlowNET.Core/APIs/tf.nn.cs @@ -101,6 +101,8 @@ public Tensor embedding_lookup(Tensor @params, name: name); public IActivation relu() => new relu(); + + public IActivation swish() => new swish(); public IActivation tanh() => new tanh(); @@ -111,6 +113,9 @@ public Tensor tanh(Tensor x, string name = null) public Tensor relu(Tensor features, string name = null) => gen_nn_ops.relu(features, name); + public Tensor relu6(Tensor features, string name = null) + => gen_nn_ops.relu6(features, name); + public Tensor[] fused_batch_norm(Tensor x, Tensor scale, Tensor offset, diff --git a/src/TensorFlowNET.Core/Keras/Activations/Activations.cs b/src/TensorFlowNET.Core/Keras/Activations/Activations.cs index f0d59ed62..37264104a 100644 --- a/src/TensorFlowNET.Core/Keras/Activations/Activations.cs +++ b/src/TensorFlowNET.Core/Keras/Activations/Activations.cs @@ -32,6 +32,7 @@ public interface IActivationsApi Activation Linear { get; } Activation Relu { get; } + Activation Relu6 { get; } Activation Sigmoid { get; } diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index 3fd98e7a8..57273eb08 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -180,6 +180,9 @@ public ILayer LayerNormalization(Axis? axis, public ILayer Normalization(Shape? input_shape = null, int? axis = -1, float? mean = null, float? variance = null, bool invert = false); public ILayer LeakyReLU(float alpha = 0.3f); + public ILayer ReLU6(); + + public IRnnCell LSTMCell(int uints, string activation = "tanh", string recurrent_activation = "sigmoid", diff --git a/src/TensorFlowNET.Keras/Activations.cs b/src/TensorFlowNET.Keras/Activations.cs index ce5b4eb13..d3801902f 100644 --- a/src/TensorFlowNET.Keras/Activations.cs +++ b/src/TensorFlowNET.Keras/Activations.cs @@ -20,6 +20,11 @@ public class Activations: IActivationsApi Name = "relu", ActivationFunction = (features, name) => tf.Context.ExecuteOp("Relu", name, new ExecuteOpArgs(features)) }; + private static Activation _relu6 = new Activation() + { + Name = "relu6", + ActivationFunction = (features, name) => tf.Context.ExecuteOp("Relu6", name, new ExecuteOpArgs(features)) + }; private static Activation _sigmoid = new Activation() { Name = "sigmoid", @@ -55,6 +60,7 @@ static Activations() _nameActivationMap = new Dictionary(); RegisterActivation(_relu); + RegisterActivation(_relu6); RegisterActivation(_linear); RegisterActivation(_sigmoid); RegisterActivation(_softmax); @@ -65,6 +71,7 @@ static Activations() public Activation Linear => _linear; public Activation Relu => _relu; + public Activation Relu6 => _relu6; public Activation Sigmoid => _sigmoid; diff --git a/src/TensorFlowNET.Keras/Layers/Activation/ReLu6.cs b/src/TensorFlowNET.Keras/Layers/Activation/ReLu6.cs new file mode 100644 index 000000000..5af3f7677 --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Activation/ReLu6.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.Engine; +using Tensorflow.Common.Types; +using static Tensorflow.Binding; + +namespace Tensorflow.Keras.Layers +{ + /// + /// Leaky version of a Rectified Linear Unit. + /// + public class ReLu6 : Layer + { + public ReLu6() : base(new LayerArgs { }) + { + } + + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) + { + return tf.nn.relu6(inputs); + } + } +} diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index bcc19dc22..e2adb23d0 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -735,6 +735,15 @@ public ILayer LeakyReLU(float alpha = 0.3f) }); + /// + /// Leaky version of a Rectified Linear Unit. + /// + /// Negative slope coefficient. + /// + public ILayer ReLU6() + => new ReLu6(); + + public IRnnCell SimpleRNNCell( int units, string activation = "tanh", From eb4ff88d39160e6046e43fe5e7453ea3e1abeac4 Mon Sep 17 00:00:00 2001 From: SMURF Date: Wed, 18 Oct 2023 23:34:15 +0100 Subject: [PATCH 58/98] fix: Saving a loaded model --- src/TensorFlowNET.Keras/Engine/Layer.Serialize.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Keras/Engine/Layer.Serialize.cs b/src/TensorFlowNET.Keras/Engine/Layer.Serialize.cs index ed5c2de0a..49811417e 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.Serialize.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.Serialize.cs @@ -27,6 +27,6 @@ public override IDictionary _trackable_children(SaveType save children = new Dictionary(); } - return children.Concat(base._trackable_children(save_type, cache)).ToDictionary(x => x.Key, x => x.Value); + return children.Concat(base._trackable_children(save_type, cache)).GroupBy(x => x.Key).Select(g => g.First()).ToDictionary(x => x.Key, x => x.Value); } } \ No newline at end of file From a73694ab2db42b2a4ea560c6bbb36ed9175fc5fb Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Fri, 20 Oct 2023 11:24:27 +0800 Subject: [PATCH 59/98] fix: add the implementation of the tile's grad --- .../Gradients/array_grad.cs | 24 +++++++++++++++++++ .../Operations/array_ops.cs | 2 +- .../GradientTest/GradientEagerTest.cs | 14 +++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Core/Gradients/array_grad.cs b/src/TensorFlowNET.Core/Gradients/array_grad.cs index 4b7027992..016e4f029 100644 --- a/src/TensorFlowNET.Core/Gradients/array_grad.cs +++ b/src/TensorFlowNET.Core/Gradients/array_grad.cs @@ -381,5 +381,29 @@ public static Tensor[] _ReverseV2Grad(Operation op, Tensor[] grads) var axis = op.inputs[1]; return new Tensor[] { array_ops.reverse(grad, axis), null }; } + + [RegisterGradient("Tile")] + public static Tensor[] _TileGrad(Operation op, Tensor[] grads) + { + var grad = grads[0]; + var input_shape = array_ops.shape(op.inputs[0], out_type: op.inputs[1].dtype); + var split_shape = array_ops.reshape(array_ops.transpose(array_ops.stack(new Tensor[] { op.inputs[1], input_shape })), new Shape(-1)); + var axes = math_ops.range(0, array_ops.size(split_shape), 2); + + //# Sum reduces grad along the first dimension for IndexedSlices + //if isinstance(grad, indexed_slices_lib.IndexedSlices): + //input_shape_0 = math_ops.cast(input_shape[0], grad.indices.dtype) + //grad = math_ops.unsorted_segment_sum( + // grad.values, math_ops.mod(grad.indices, input_shape_0), input_shape_0) + //split_shape = array_ops.concat([[1], split_shape[1:]], axis = 0) + + var input_grad = math_ops.reduce_sum(array_ops.reshape(grad, split_shape), axes); + if (!tf.Context.executing_eagerly()) + { + input_grad.set_shape(op.inputs[0].GetShape()); + } + return new Tensor[] { input_grad, null }; + + } } } diff --git a/src/TensorFlowNET.Core/Operations/array_ops.cs b/src/TensorFlowNET.Core/Operations/array_ops.cs index fdc53cd7e..abf44c643 100644 --- a/src/TensorFlowNET.Core/Operations/array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/array_ops.cs @@ -990,7 +990,7 @@ public static Tensor gather(ResourceVariable @params, Tensor indices, string nam return @params.sparse_read(indices, name); } - public static Tensor transpose(T1 a, Axis perm, string name = "transpose", bool conjugate = false) + public static Tensor transpose(T1 a, Axis perm = null, string name = "transpose", bool conjugate = false) { return tf_with(ops.name_scope(name, "transpose", new { a }), scope => { diff --git a/test/TensorFlowNET.UnitTest/GradientTest/GradientEagerTest.cs b/test/TensorFlowNET.UnitTest/GradientTest/GradientEagerTest.cs index e41e1d617..ed7599045 100644 --- a/test/TensorFlowNET.UnitTest/GradientTest/GradientEagerTest.cs +++ b/test/TensorFlowNET.UnitTest/GradientTest/GradientEagerTest.cs @@ -173,5 +173,19 @@ public void ConditionalMultiply() var result = grad(x, 4); Assert.AreEqual((float)result, 4.0f); } + + [TestMethod] + public void Tile() + { + var a = tf.constant(new int[] { 1 }, TF_DataType.TF_FLOAT); + var b = tf.constant(new int[] { 2 }); + using (var tape = tf.GradientTape()) + { + tape.watch(a); + var y = tf.tile(a, b); + var grad = tape.gradient(y, a); + Assert.AreEqual((float)grad.numpy(), 2.0f); + } + } } } From 3fcc4d8d1540c7c01ce4ca05ea883874abd4e5e5 Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Fri, 20 Oct 2023 11:30:33 +0800 Subject: [PATCH 60/98] fix: add the GRU, LSTM, SimpleRNN's OptionalArgs --- .../Keras/ArgsDefinition/Rnn/GRUOptionalArgs.cs | 4 +--- .../Keras/ArgsDefinition/Rnn/LSTMOptionalArgs.cs | 11 +++++++++++ .../Keras/ArgsDefinition/Rnn/SimpleRNNOptionalArgs.cs | 11 +++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMOptionalArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNOptionalArgs.cs diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUOptionalArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUOptionalArgs.cs index d441dc828..1d215576f 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUOptionalArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUOptionalArgs.cs @@ -4,10 +4,8 @@ namespace Tensorflow.Keras.ArgsDefinition { - public class GRUOptionalArgs + public class GRUOptionalArgs : RnnOptionalArgs { public string Identifier => "GRU"; - - public Tensor Mask { get; set; } = null; } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMOptionalArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMOptionalArgs.cs new file mode 100644 index 000000000..2829927c3 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMOptionalArgs.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition.Rnn +{ + public class LSTMOptionalArgs : RnnOptionalArgs + { + public string Identifier => "LSTM"; + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNOptionalArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNOptionalArgs.cs new file mode 100644 index 000000000..a8b8caf06 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNOptionalArgs.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition.Rnn +{ + public class SimpleRNNOptionalArgs : RnnOptionalArgs + { + public string Identifier => "SimpleRNN"; + } +} From d0ec6591a0cc0ea3325a7fc723435b23eabc757b Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Fri, 20 Oct 2023 15:40:35 +0800 Subject: [PATCH 61/98] fix: add the implementation of GatherND's grad --- src/TensorFlowNET.Core/APIs/tf.array.cs | 10 ++++++++++ .../Gradients/array_grad.cs | 19 +++++++++++++++++++ .../Operations/array_ops.cs | 2 +- .../GradientTest/GradientEagerTest.cs | 17 ++++++++++++++++- 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.array.cs b/src/TensorFlowNET.Core/APIs/tf.array.cs index 4d9c3da58..b529cd319 100644 --- a/src/TensorFlowNET.Core/APIs/tf.array.cs +++ b/src/TensorFlowNET.Core/APIs/tf.array.cs @@ -140,6 +140,16 @@ public Tensor identity(Tensor input, string name = null) public Tensor gather(Tensor @params, Tensor indices, string name = null, int axis = 0) => array_ops.gather(@params, indices, name: name, axis: ops.convert_to_tensor(axis)); + /// + /// Gather slices from `params` into a Tensor with shape specified by `indices`. + /// + /// + /// + /// + /// + public Tensor gather_nd(Tensor @params, Tensor indices, string name = null) + => gen_array_ops.gather_nd(@params, indices, name: name); + /// /// Return the elements, either from `x` or `y`, depending on the `condition`. /// diff --git a/src/TensorFlowNET.Core/Gradients/array_grad.cs b/src/TensorFlowNET.Core/Gradients/array_grad.cs index 016e4f029..a4da60eed 100644 --- a/src/TensorFlowNET.Core/Gradients/array_grad.cs +++ b/src/TensorFlowNET.Core/Gradients/array_grad.cs @@ -403,7 +403,26 @@ public static Tensor[] _TileGrad(Operation op, Tensor[] grads) input_grad.set_shape(op.inputs[0].GetShape()); } return new Tensor[] { input_grad, null }; + } + [RegisterGradient("GatherNd")] + public static Tensor[] _GatherNdGrad(Operation op, Tensor[] grads) + { + var @ref = op.inputs[0]; + var indices = op.inputs[1]; + var grad = grads[0]; + var ref_shape = array_ops.shape(@ref, out_type: indices.dtype); + Tensor ref_grad = null; + if (indices.shape.ndim == 2 && indices.shape.dims[indices.shape.Length - 1] == 1) + { + ref_grad = (Tensor)new IndexedSlices(grad, array_ops.squeeze(indices, axis: -1), ref_shape); + } + else + { + ref_grad = gen_array_ops.scatter_nd(indices, grad, ref_shape); + } + return new Tensor[] { ref_grad, null }; } + } } diff --git a/src/TensorFlowNET.Core/Operations/array_ops.cs b/src/TensorFlowNET.Core/Operations/array_ops.cs index abf44c643..57af3b835 100644 --- a/src/TensorFlowNET.Core/Operations/array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/array_ops.cs @@ -829,7 +829,7 @@ public static Tensor strided_slice_grad(Tensor shape, Tensor begin, Tensor end, /// A `Tensor`. Has the same type as `input`. /// Contains the same data as `input`, but has one or more dimensions of /// size 1 removed. - public static Tensor squeeze(Tensor input, int[] axis = null, string name = null) + public static Tensor squeeze(Tensor input, Axis axis = null, string name = null) => gen_array_ops.squeeze(input, axis, name); public static Tensor identity(Tensor input, string name = null) diff --git a/test/TensorFlowNET.UnitTest/GradientTest/GradientEagerTest.cs b/test/TensorFlowNET.UnitTest/GradientTest/GradientEagerTest.cs index ed7599045..1cfceb3e3 100644 --- a/test/TensorFlowNET.UnitTest/GradientTest/GradientEagerTest.cs +++ b/test/TensorFlowNET.UnitTest/GradientTest/GradientEagerTest.cs @@ -62,7 +62,7 @@ public void SquaredDifference_1D() // Calcute the gradient of (x1-x2)^2 // by Automatic Differentiation in Eager mode // Expected is 2*(abs(x1-x2)) - Tensor x1 = new NDArray( new float[] { 1, 3, 5, 21, 19, 17 }); + Tensor x1 = new NDArray(new float[] { 1, 3, 5, 21, 19, 17 }); Tensor x2 = new NDArray(new float[] { 29, 27, 23, 7, 11, 13 }); float[] expected = new float[] { @@ -187,5 +187,20 @@ public void Tile() Assert.AreEqual((float)grad.numpy(), 2.0f); } } + + [TestMethod] + public void GatherNdTest() + { + var x = tf.constant(new float[,] { { 1.0f, 2.0f, 3.0f }, { 1.0f, 2.0f, 3.0f }, { 1.0f, 2.0f, 3.0f } }, dtype: TF_DataType.TF_FLOAT); + var indices = tf.constant(new int[,] { { 0, 1 }, { 1, 1 }, { 2, 1 } }, dtype: TF_DataType.TF_INT32); + using (var tape = tf.GradientTape()) + { + tape.watch(x); + var res = tf.gather_nd(x, indices); + var grad = tape.gradient(res, x); + var expected = np.array(new float[,] { { 0f, 1f, 0f }, { 0f, 1f, 0f }, { 0f, 1f, 0f } }); + Assert.IsTrue(Enumerable.SequenceEqual(grad.ToArray(), expected.ToArray())); + } + } } } From 4e42d7f3a8ee574caf9c3896bb6438e88cbab211 Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Sat, 4 Nov 2023 10:18:50 +0800 Subject: [PATCH 62/98] fix: fix the bug of boolean_mask --- src/TensorFlowNET.Core/Operations/NnOps/rnn.cs | 4 ++-- src/TensorFlowNET.Core/Operations/array_ops.cs | 13 +++++++++---- src/TensorFlowNET.Core/Operations/nn_ops.cs | 2 +- .../Basics/TensorTest.cs | 7 ++++--- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/TensorFlowNET.Core/Operations/NnOps/rnn.cs b/src/TensorFlowNET.Core/Operations/NnOps/rnn.cs index 6b9f073c1..55f139207 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/rnn.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/rnn.cs @@ -428,9 +428,9 @@ public static Tensor _transpose_batch_time(Tensor x) return x; var x_rank = array_ops.rank(x); - var con1 = new object[] + var con1 = new Tensor[] { - new []{1, 0 }, + new Tensor(new int[]{0, 2}), math_ops.range(2, x_rank) }; var x_t = array_ops.transpose(x, array_ops.concat(con1, 0)); diff --git a/src/TensorFlowNET.Core/Operations/array_ops.cs b/src/TensorFlowNET.Core/Operations/array_ops.cs index 57af3b835..1b424006d 100644 --- a/src/TensorFlowNET.Core/Operations/array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/array_ops.cs @@ -166,6 +166,11 @@ public static Tensor boolean_mask(T1 tensor, T2 mask, string name = "boo throw new ValueError("mask cannot be scalar."); var leading_size = gen_math_ops.prod(shape(tensor_tensor)[$"{axis}:{axis + ndims_mask}"], ops.convert_to_tensor(new[] { 0 })); + if (leading_size.rank == 0) + { + leading_size = expand_dims(leading_size, 0); + } + var shape1 = concat(new[] { shape(tensor_tensor)[$":{axis}"], @@ -185,7 +190,7 @@ public static Tensor boolean_mask(T1 tensor, T2 mask, string name = "boo private static Tensor _apply_mask_1d(Tensor reshaped_tensor, Tensor mask, int axis = 0) { - var indices = squeeze(where(mask), axis: new[] { 1 }); + var indices = squeeze(where_v2(mask), axis: new[] { 1 }); return gather(reshaped_tensor, indices, axis: ops.convert_to_tensor(axis)); } @@ -940,12 +945,12 @@ public static Tensor broadcast_static_shape(Tensor shape_x, Tensor shape_y) /// public static Tensor concat(Tensor[] values, Tensor axis, string name = "concat") { - return tf.Context.ExecuteOp("ConcatV2", name, new ExecuteOpArgs(values, axis)); + return gen_array_ops.concat_v2(values, axis, name: name); } - public static Tensor concat(object[] values, int axis, string name = "concat") + public static Tensor concat(Tensor[] values, Axis axis, string name = "concat") { - return tf.Context.ExecuteOp("ConcatV2", name, new ExecuteOpArgs(values, axis)); + return gen_array_ops.concat_v2(values, axis, name: name); } /// diff --git a/src/TensorFlowNET.Core/Operations/nn_ops.cs b/src/TensorFlowNET.Core/Operations/nn_ops.cs index 00d7d316b..394a591ab 100644 --- a/src/TensorFlowNET.Core/Operations/nn_ops.cs +++ b/src/TensorFlowNET.Core/Operations/nn_ops.cs @@ -287,7 +287,7 @@ private static Tensor _flatten_outer_dims(Tensor logits) new[] { math_ops.subtract(rank, 1) }, new[] { constant_op.constant(1) }); - var ops = array_ops.concat(new[] { new[] { -1 }, (object)last_dim_size }, 0); + var ops = array_ops.concat(new Tensor[] { new Tensor(new int[] {1}), last_dim_size }, 0); var output = array_ops.reshape(logits, ops); // Set output shape if known. diff --git a/test/TensorFlowNET.Graph.UnitTest/Basics/TensorTest.cs b/test/TensorFlowNET.Graph.UnitTest/Basics/TensorTest.cs index 90de78743..8093c1f23 100644 --- a/test/TensorFlowNET.Graph.UnitTest/Basics/TensorTest.cs +++ b/test/TensorFlowNET.Graph.UnitTest/Basics/TensorTest.cs @@ -3,6 +3,7 @@ using System; using System.Linq; using static Tensorflow.Binding; +using Tensorflow; namespace TensorFlowNET.UnitTest.Basics { @@ -60,14 +61,14 @@ public void batch_to_space_nd() Assert.IsTrue(Enumerable.SequenceEqual(new int[] { 15, 21, 16, 22, 17, 23 }, result[0, 3].ToArray())); } - [TestMethod, Ignore] + [TestMethod] public void boolean_mask() { + if (!tf.executing_eagerly()) + tf.enable_eager_execution(); var tensor = new[] { 0, 1, 2, 3 }; var mask = np.array(new[] { true, false, true, false }); var masked = tf.boolean_mask(tensor, mask); - var sess = tf.Session(); - var result = sess.run(masked); Assert.IsTrue(Enumerable.SequenceEqual(new int[] { 0, 2 }, masked.ToArray())); } } From f721baee711cc79a5270e72d73acb475ed4abaf0 Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Sun, 5 Nov 2023 14:05:41 +0800 Subject: [PATCH 63/98] test: add the concat_v2 test --- .../TensorFlow.Kernel.UnitTest.csproj | 24 +++++++ .../array_ops/concat_op_test.cs | 65 +++++++++++++++++++ TensorFlow.NET.sln | 21 ++++++ 3 files changed, 110 insertions(+) create mode 100644 TensorFlow.Kernel.UnitTest/TensorFlow.Kernel.UnitTest.csproj create mode 100644 TensorFlow.Kernel.UnitTest/array_ops/concat_op_test.cs diff --git a/TensorFlow.Kernel.UnitTest/TensorFlow.Kernel.UnitTest.csproj b/TensorFlow.Kernel.UnitTest/TensorFlow.Kernel.UnitTest.csproj new file mode 100644 index 000000000..a52a4cda6 --- /dev/null +++ b/TensorFlow.Kernel.UnitTest/TensorFlow.Kernel.UnitTest.csproj @@ -0,0 +1,24 @@ + + + + net6.0 + enable + enable + + false + true + + + + + + + + + + + + + + + diff --git a/TensorFlow.Kernel.UnitTest/array_ops/concat_op_test.cs b/TensorFlow.Kernel.UnitTest/array_ops/concat_op_test.cs new file mode 100644 index 000000000..cfa8f0fbf --- /dev/null +++ b/TensorFlow.Kernel.UnitTest/array_ops/concat_op_test.cs @@ -0,0 +1,65 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Tensorflow; +using Tensorflow.NumPy; +using TensorFlow; +using static Tensorflow.Binding; +using static Tensorflow.KerasApi; + +namespace TensorFlow.Kernel.UnitTest +{ + [TestClass] + public class concat_op_test + { + [TestMethod] + public void testConcatEmpty() + { + var t1 = tf.constant(new int[] { }); + var t2 = tf.constant(new int[] { }); + var c = array_ops.concat(new[] { t1, t2 }, 0); + var expected = np.array(new int[] { }); + Assert.IsTrue(Enumerable.SequenceEqual(expected.ToArray(), c.numpy().ToArray())); + } + + [TestMethod] + public void testConcatNegativeAxis() + { + var t1 = tf.constant(new int[,] {{ 1, 2, 3 }, { 4, 5, 6 } }); + var t2 = tf.constant(new int[,] { { 7, 8, 9 }, { 10, 11, 12 } }); + var c = array_ops.concat(new[] { t1, t2 }, -2); + var expected = np.array(new int[,,] { { { 1, 2, 3 }, { 4, 5, 6 } }, { { 7, 8, 9 }, { 10, 11, 12 } } }); + Assert.IsTrue(Enumerable.SequenceEqual(expected.ToArray(), c.numpy().ToArray())); + + c = array_ops.concat(new[] { t1, t2 }, -1); + expected = np.array(new int[,] { { 1, 2, 3, 7, 8, 9 }, { 4, 5, 6, 10, 11, 12 } }); + Assert.IsTrue(Enumerable.SequenceEqual(expected.ToArray(), c.numpy().ToArray())); + } + + [TestMethod] + [DataRow(TF_DataType.TF_INT32)] + [DataRow(TF_DataType.TF_INT64)] + [DataRow(TF_DataType.TF_UINT32)] + [DataRow(TF_DataType.TF_UINT64)] + public void testConcatDtype(TF_DataType dtype) + { + var t1 = tf.constant(new int[,] { { 1, 2, 3 }, { 4, 5, 6 } }, dtype: dtype); + var t2 = tf.constant(new int[,] { { 7, 8, 9 }, { 10, 11, 12 } }, dtype: dtype); + var c = array_ops.concat(new[] { t1, t2 }, 1); + var expected = np.array(new int[,] { { 1, 2, 3, 7, 8, 9 }, { 4, 5, 6, 10, 11, 12 } }); + Assert.IsTrue(Enumerable.SequenceEqual(expected.ToArray(), tf.cast(c, TF_DataType.TF_INT32).numpy().ToArray())); + + } + + [TestMethod] + [DataRow(TF_DataType.TF_INT32)] + [DataRow(TF_DataType.TF_INT64)] + public void testConcatAxisType(TF_DataType dtype) + { + var t1 = tf.constant(new int[,] { { 1, 2, 3 }, {4, 5, 6 } }); + var t2 = tf.constant(new int[,] { { 7, 8, 9 }, { 10, 11, 12 } }); + var c = array_ops.concat(new[] { t1, t2 }, tf.constant(1, dtype: dtype)); + var expected = np.array(new int[,] { { 1, 2, 3, 7, 8, 9 }, { 4, 5, 6, 10, 11, 12 } }); + Assert.IsTrue(Enumerable.SequenceEqual(expected.ToArray(), tf.cast(c, TF_DataType.TF_INT32).numpy().ToArray())); + } + + } +} diff --git a/TensorFlow.NET.sln b/TensorFlow.NET.sln index 87729e27d..a246407b0 100644 --- a/TensorFlow.NET.sln +++ b/TensorFlow.NET.sln @@ -39,6 +39,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tensorflow.Benchmark", "too EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tensorflow.Console", "tools\TensorFlowNET.Console\Tensorflow.Console.csproj", "{1DC32255-BA1F-4D6D-A9C9-5BD5ED71CAA0}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TensorFlow.Kernel.UnitTest", "TensorFlow.Kernel.UnitTest\TensorFlow.Kernel.UnitTest.csproj", "{C08C6692-4818-46C1-8462-2F0CC40C9152}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -322,6 +324,24 @@ Global {1DC32255-BA1F-4D6D-A9C9-5BD5ED71CAA0}.Release|x64.Build.0 = Release|x64 {1DC32255-BA1F-4D6D-A9C9-5BD5ED71CAA0}.Release|x86.ActiveCfg = Release|Any CPU {1DC32255-BA1F-4D6D-A9C9-5BD5ED71CAA0}.Release|x86.Build.0 = Release|Any CPU + {C08C6692-4818-46C1-8462-2F0CC40C9152}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C08C6692-4818-46C1-8462-2F0CC40C9152}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C08C6692-4818-46C1-8462-2F0CC40C9152}.Debug|x64.ActiveCfg = Debug|Any CPU + {C08C6692-4818-46C1-8462-2F0CC40C9152}.Debug|x64.Build.0 = Debug|Any CPU + {C08C6692-4818-46C1-8462-2F0CC40C9152}.Debug|x86.ActiveCfg = Debug|Any CPU + {C08C6692-4818-46C1-8462-2F0CC40C9152}.Debug|x86.Build.0 = Debug|Any CPU + {C08C6692-4818-46C1-8462-2F0CC40C9152}.GPU|Any CPU.ActiveCfg = Debug|Any CPU + {C08C6692-4818-46C1-8462-2F0CC40C9152}.GPU|Any CPU.Build.0 = Debug|Any CPU + {C08C6692-4818-46C1-8462-2F0CC40C9152}.GPU|x64.ActiveCfg = Debug|Any CPU + {C08C6692-4818-46C1-8462-2F0CC40C9152}.GPU|x64.Build.0 = Debug|Any CPU + {C08C6692-4818-46C1-8462-2F0CC40C9152}.GPU|x86.ActiveCfg = Debug|Any CPU + {C08C6692-4818-46C1-8462-2F0CC40C9152}.GPU|x86.Build.0 = Debug|Any CPU + {C08C6692-4818-46C1-8462-2F0CC40C9152}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C08C6692-4818-46C1-8462-2F0CC40C9152}.Release|Any CPU.Build.0 = Release|Any CPU + {C08C6692-4818-46C1-8462-2F0CC40C9152}.Release|x64.ActiveCfg = Release|Any CPU + {C08C6692-4818-46C1-8462-2F0CC40C9152}.Release|x64.Build.0 = Release|Any CPU + {C08C6692-4818-46C1-8462-2F0CC40C9152}.Release|x86.ActiveCfg = Release|Any CPU + {C08C6692-4818-46C1-8462-2F0CC40C9152}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -342,6 +362,7 @@ Global {D24FCAA5-548C-4251-B226-A1B6535D0845} = {E1A5D2B7-10AF-4876-85C0-7714EF274214} {C23563DB-FE21-48E7-A411-87A109E4A899} = {E1A5D2B7-10AF-4876-85C0-7714EF274214} {1DC32255-BA1F-4D6D-A9C9-5BD5ED71CAA0} = {E1A5D2B7-10AF-4876-85C0-7714EF274214} + {C08C6692-4818-46C1-8462-2F0CC40C9152} = {1B0918B9-65AD-4F34-A287-AF4597B27DBD} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {2DEAD3CC-486B-4918-A607-50B0DE7B114A} From 8c06bbb0169f4c96c5c17bdd5fcbf07557665d03 Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Sun, 5 Nov 2023 20:47:58 +0800 Subject: [PATCH 64/98] fix: fix the bug caused by concat_v2 --- src/TensorFlowNET.Core/Operations/NnOps/rnn.cs | 4 ++-- src/TensorFlowNET.Core/Operations/array_ops.cs | 6 +++--- src/TensorFlowNET.Core/Operations/nn_ops.cs | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/TensorFlowNET.Core/Operations/NnOps/rnn.cs b/src/TensorFlowNET.Core/Operations/NnOps/rnn.cs index 55f139207..6b9f073c1 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/rnn.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/rnn.cs @@ -428,9 +428,9 @@ public static Tensor _transpose_batch_time(Tensor x) return x; var x_rank = array_ops.rank(x); - var con1 = new Tensor[] + var con1 = new object[] { - new Tensor(new int[]{0, 2}), + new []{1, 0 }, math_ops.range(2, x_rank) }; var x_t = array_ops.transpose(x, array_ops.concat(con1, 0)); diff --git a/src/TensorFlowNET.Core/Operations/array_ops.cs b/src/TensorFlowNET.Core/Operations/array_ops.cs index 1b424006d..548a885ed 100644 --- a/src/TensorFlowNET.Core/Operations/array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/array_ops.cs @@ -945,12 +945,12 @@ public static Tensor broadcast_static_shape(Tensor shape_x, Tensor shape_y) /// public static Tensor concat(Tensor[] values, Tensor axis, string name = "concat") { - return gen_array_ops.concat_v2(values, axis, name: name); + return tf.Context.ExecuteOp("ConcatV2", name, new ExecuteOpArgs(values, axis)); } - public static Tensor concat(Tensor[] values, Axis axis, string name = "concat") + public static Tensor concat(object[] values, int axis, string name = "concat") { - return gen_array_ops.concat_v2(values, axis, name: name); + return tf.Context.ExecuteOp("ConcatV2", name, new ExecuteOpArgs(values, axis)); } /// diff --git a/src/TensorFlowNET.Core/Operations/nn_ops.cs b/src/TensorFlowNET.Core/Operations/nn_ops.cs index 394a591ab..00d7d316b 100644 --- a/src/TensorFlowNET.Core/Operations/nn_ops.cs +++ b/src/TensorFlowNET.Core/Operations/nn_ops.cs @@ -287,7 +287,7 @@ private static Tensor _flatten_outer_dims(Tensor logits) new[] { math_ops.subtract(rank, 1) }, new[] { constant_op.constant(1) }); - var ops = array_ops.concat(new Tensor[] { new Tensor(new int[] {1}), last_dim_size }, 0); + var ops = array_ops.concat(new[] { new[] { -1 }, (object)last_dim_size }, 0); var output = array_ops.reshape(logits, ops); // Set output shape if known. From 7fd455041d85dc4143a4a6e4d876b9c22be51f51 Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Sun, 5 Nov 2023 21:51:33 +0800 Subject: [PATCH 65/98] refactor: refacter the place of the kernel unittest folder --- TensorFlow.NET.sln | 40 +++++++++---------- .../TensorFlow.Kernel.UnitTest.csproj | 4 +- .../array_ops/concat_op_test.cs | 10 ++--- 3 files changed, 26 insertions(+), 28 deletions(-) rename {TensorFlow.Kernel.UnitTest => test/TensorFlow.Kernel.UnitTest}/TensorFlow.Kernel.UnitTest.csproj (74%) rename {TensorFlow.Kernel.UnitTest => test/TensorFlow.Kernel.UnitTest}/array_ops/concat_op_test.cs (89%) diff --git a/TensorFlow.NET.sln b/TensorFlow.NET.sln index a246407b0..214b039d4 100644 --- a/TensorFlow.NET.sln +++ b/TensorFlow.NET.sln @@ -39,7 +39,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tensorflow.Benchmark", "too EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tensorflow.Console", "tools\TensorFlowNET.Console\Tensorflow.Console.csproj", "{1DC32255-BA1F-4D6D-A9C9-5BD5ED71CAA0}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TensorFlow.Kernel.UnitTest", "TensorFlow.Kernel.UnitTest\TensorFlow.Kernel.UnitTest.csproj", "{C08C6692-4818-46C1-8462-2F0CC40C9152}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TensorFlow.Kernel.UnitTest", "test\TensorFlow.Kernel.UnitTest\TensorFlow.Kernel.UnitTest.csproj", "{654A027D-1364-4729-880B-144DFE1FF5BB}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -324,24 +324,24 @@ Global {1DC32255-BA1F-4D6D-A9C9-5BD5ED71CAA0}.Release|x64.Build.0 = Release|x64 {1DC32255-BA1F-4D6D-A9C9-5BD5ED71CAA0}.Release|x86.ActiveCfg = Release|Any CPU {1DC32255-BA1F-4D6D-A9C9-5BD5ED71CAA0}.Release|x86.Build.0 = Release|Any CPU - {C08C6692-4818-46C1-8462-2F0CC40C9152}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {C08C6692-4818-46C1-8462-2F0CC40C9152}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C08C6692-4818-46C1-8462-2F0CC40C9152}.Debug|x64.ActiveCfg = Debug|Any CPU - {C08C6692-4818-46C1-8462-2F0CC40C9152}.Debug|x64.Build.0 = Debug|Any CPU - {C08C6692-4818-46C1-8462-2F0CC40C9152}.Debug|x86.ActiveCfg = Debug|Any CPU - {C08C6692-4818-46C1-8462-2F0CC40C9152}.Debug|x86.Build.0 = Debug|Any CPU - {C08C6692-4818-46C1-8462-2F0CC40C9152}.GPU|Any CPU.ActiveCfg = Debug|Any CPU - {C08C6692-4818-46C1-8462-2F0CC40C9152}.GPU|Any CPU.Build.0 = Debug|Any CPU - {C08C6692-4818-46C1-8462-2F0CC40C9152}.GPU|x64.ActiveCfg = Debug|Any CPU - {C08C6692-4818-46C1-8462-2F0CC40C9152}.GPU|x64.Build.0 = Debug|Any CPU - {C08C6692-4818-46C1-8462-2F0CC40C9152}.GPU|x86.ActiveCfg = Debug|Any CPU - {C08C6692-4818-46C1-8462-2F0CC40C9152}.GPU|x86.Build.0 = Debug|Any CPU - {C08C6692-4818-46C1-8462-2F0CC40C9152}.Release|Any CPU.ActiveCfg = Release|Any CPU - {C08C6692-4818-46C1-8462-2F0CC40C9152}.Release|Any CPU.Build.0 = Release|Any CPU - {C08C6692-4818-46C1-8462-2F0CC40C9152}.Release|x64.ActiveCfg = Release|Any CPU - {C08C6692-4818-46C1-8462-2F0CC40C9152}.Release|x64.Build.0 = Release|Any CPU - {C08C6692-4818-46C1-8462-2F0CC40C9152}.Release|x86.ActiveCfg = Release|Any CPU - {C08C6692-4818-46C1-8462-2F0CC40C9152}.Release|x86.Build.0 = Release|Any CPU + {654A027D-1364-4729-880B-144DFE1FF5BB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {654A027D-1364-4729-880B-144DFE1FF5BB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {654A027D-1364-4729-880B-144DFE1FF5BB}.Debug|x64.ActiveCfg = Debug|Any CPU + {654A027D-1364-4729-880B-144DFE1FF5BB}.Debug|x64.Build.0 = Debug|Any CPU + {654A027D-1364-4729-880B-144DFE1FF5BB}.Debug|x86.ActiveCfg = Debug|Any CPU + {654A027D-1364-4729-880B-144DFE1FF5BB}.Debug|x86.Build.0 = Debug|Any CPU + {654A027D-1364-4729-880B-144DFE1FF5BB}.GPU|Any CPU.ActiveCfg = Debug|Any CPU + {654A027D-1364-4729-880B-144DFE1FF5BB}.GPU|Any CPU.Build.0 = Debug|Any CPU + {654A027D-1364-4729-880B-144DFE1FF5BB}.GPU|x64.ActiveCfg = Debug|Any CPU + {654A027D-1364-4729-880B-144DFE1FF5BB}.GPU|x64.Build.0 = Debug|Any CPU + {654A027D-1364-4729-880B-144DFE1FF5BB}.GPU|x86.ActiveCfg = Debug|Any CPU + {654A027D-1364-4729-880B-144DFE1FF5BB}.GPU|x86.Build.0 = Debug|Any CPU + {654A027D-1364-4729-880B-144DFE1FF5BB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {654A027D-1364-4729-880B-144DFE1FF5BB}.Release|Any CPU.Build.0 = Release|Any CPU + {654A027D-1364-4729-880B-144DFE1FF5BB}.Release|x64.ActiveCfg = Release|Any CPU + {654A027D-1364-4729-880B-144DFE1FF5BB}.Release|x64.Build.0 = Release|Any CPU + {654A027D-1364-4729-880B-144DFE1FF5BB}.Release|x86.ActiveCfg = Release|Any CPU + {654A027D-1364-4729-880B-144DFE1FF5BB}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -362,7 +362,7 @@ Global {D24FCAA5-548C-4251-B226-A1B6535D0845} = {E1A5D2B7-10AF-4876-85C0-7714EF274214} {C23563DB-FE21-48E7-A411-87A109E4A899} = {E1A5D2B7-10AF-4876-85C0-7714EF274214} {1DC32255-BA1F-4D6D-A9C9-5BD5ED71CAA0} = {E1A5D2B7-10AF-4876-85C0-7714EF274214} - {C08C6692-4818-46C1-8462-2F0CC40C9152} = {1B0918B9-65AD-4F34-A287-AF4597B27DBD} + {654A027D-1364-4729-880B-144DFE1FF5BB} = {1B0918B9-65AD-4F34-A287-AF4597B27DBD} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {2DEAD3CC-486B-4918-A607-50B0DE7B114A} diff --git a/TensorFlow.Kernel.UnitTest/TensorFlow.Kernel.UnitTest.csproj b/test/TensorFlow.Kernel.UnitTest/TensorFlow.Kernel.UnitTest.csproj similarity index 74% rename from TensorFlow.Kernel.UnitTest/TensorFlow.Kernel.UnitTest.csproj rename to test/TensorFlow.Kernel.UnitTest/TensorFlow.Kernel.UnitTest.csproj index a52a4cda6..68eb9e9b2 100644 --- a/TensorFlow.Kernel.UnitTest/TensorFlow.Kernel.UnitTest.csproj +++ b/test/TensorFlow.Kernel.UnitTest/TensorFlow.Kernel.UnitTest.csproj @@ -17,8 +17,8 @@ - - + + diff --git a/TensorFlow.Kernel.UnitTest/array_ops/concat_op_test.cs b/test/TensorFlow.Kernel.UnitTest/array_ops/concat_op_test.cs similarity index 89% rename from TensorFlow.Kernel.UnitTest/array_ops/concat_op_test.cs rename to test/TensorFlow.Kernel.UnitTest/array_ops/concat_op_test.cs index cfa8f0fbf..67d0aa602 100644 --- a/TensorFlow.Kernel.UnitTest/array_ops/concat_op_test.cs +++ b/test/TensorFlow.Kernel.UnitTest/array_ops/concat_op_test.cs @@ -1,9 +1,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Tensorflow; using Tensorflow.NumPy; -using TensorFlow; using static Tensorflow.Binding; -using static Tensorflow.KerasApi; namespace TensorFlow.Kernel.UnitTest { @@ -23,14 +21,14 @@ public void testConcatEmpty() [TestMethod] public void testConcatNegativeAxis() { - var t1 = tf.constant(new int[,] {{ 1, 2, 3 }, { 4, 5, 6 } }); + var t1 = tf.constant(new int[,] { { 1, 2, 3 }, { 4, 5, 6 } }); var t2 = tf.constant(new int[,] { { 7, 8, 9 }, { 10, 11, 12 } }); var c = array_ops.concat(new[] { t1, t2 }, -2); var expected = np.array(new int[,,] { { { 1, 2, 3 }, { 4, 5, 6 } }, { { 7, 8, 9 }, { 10, 11, 12 } } }); Assert.IsTrue(Enumerable.SequenceEqual(expected.ToArray(), c.numpy().ToArray())); c = array_ops.concat(new[] { t1, t2 }, -1); - expected = np.array(new int[,] { { 1, 2, 3, 7, 8, 9 }, { 4, 5, 6, 10, 11, 12 } }); + expected = np.array(new int[,] { { 1, 2, 3, 7, 8, 9 }, { 4, 5, 6, 10, 11, 12 } }); Assert.IsTrue(Enumerable.SequenceEqual(expected.ToArray(), c.numpy().ToArray())); } @@ -54,7 +52,7 @@ public void testConcatDtype(TF_DataType dtype) [DataRow(TF_DataType.TF_INT64)] public void testConcatAxisType(TF_DataType dtype) { - var t1 = tf.constant(new int[,] { { 1, 2, 3 }, {4, 5, 6 } }); + var t1 = tf.constant(new int[,] { { 1, 2, 3 }, { 4, 5, 6 } }); var t2 = tf.constant(new int[,] { { 7, 8, 9 }, { 10, 11, 12 } }); var c = array_ops.concat(new[] { t1, t2 }, tf.constant(1, dtype: dtype)); var expected = np.array(new int[,] { { 1, 2, 3, 7, 8, 9 }, { 4, 5, 6, 10, 11, 12 } }); @@ -62,4 +60,4 @@ public void testConcatAxisType(TF_DataType dtype) } } -} +} \ No newline at end of file From 7f0161445d1142f18ca2e18504e25fcad15e1d44 Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Sun, 5 Nov 2023 21:54:56 +0800 Subject: [PATCH 66/98] fix: fix a project reference mistake --- .../TensorFlow.Kernel.UnitTest.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/TensorFlow.Kernel.UnitTest/TensorFlow.Kernel.UnitTest.csproj b/test/TensorFlow.Kernel.UnitTest/TensorFlow.Kernel.UnitTest.csproj index 68eb9e9b2..21b2731b7 100644 --- a/test/TensorFlow.Kernel.UnitTest/TensorFlow.Kernel.UnitTest.csproj +++ b/test/TensorFlow.Kernel.UnitTest/TensorFlow.Kernel.UnitTest.csproj @@ -17,8 +17,8 @@ + - From 94c0bb8796a06a4becb21687141f2a4451c9230e Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 5 Nov 2023 15:02:16 -0600 Subject: [PATCH 67/98] Release v0.150.0 based on tensorflowv v2.15.0. --- README.md | 19 ++++--------------- .../APIs/c_api.customize.cs | 6 +++--- .../Operations/Operation.cs | 2 +- .../Operations/handle_data_util.cs | 2 +- .../Tensorflow.Binding.csproj | 14 +++++++++----- src/TensorFlowNET.Core/ops.cs | 2 +- .../Tensorflow.Keras.csproj | 9 +++++---- src/TensorflowNET.Hub/Tensorflow.Hub.csproj | 2 +- .../Tensorflow.Console.csproj | 5 +---- .../Tensorflow.CodeGen.csproj | 1 - .../Tensorflow.UnitTest.RedistHolder.csproj | 2 +- 11 files changed, 27 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 36ec1660c..0198c873c 100644 --- a/README.md +++ b/README.md @@ -15,20 +15,6 @@ English | [中文](docs/README-CN.md) -**=========================================================** - -### [Voting: Naming Convention Approach of v1.0.0](https://github.com/SciSharp/TensorFlow.NET/issues/1074) - -Dear all, - -We would like to urge you to participate in our upcoming vote regarding the naming convention for TensorFlow.NET version 1.0.0 in [#1074](https://github.com/SciSharp/TensorFlow.NET/issues/1074). Your participation in the vote is essential to help us decide on the best approach for improving the naming convention used in previous versions. - -Thank you, - -TensorFlow.NET Authors - -**=========================================================** - *master branch and v0.100.x is corresponding to tensorflow v2.10, v0.6x branch is from tensorflow v2.6, v0.15-tensorflow1.15 is from tensorflow1.15. Please add `https://www.myget.org/F/scisharp/api/v3/index.json` to nuget source to use nightly release.* @@ -75,9 +61,12 @@ PM> Install-Package TensorFlow.Keras The second part is the computing support part. Only one of the following packages is needed, depending on your device and system. ``` -### CPU version for Windows, Linux and Mac +### CPU version for Windows and Linux PM> Install-Package SciSharp.TensorFlow.Redist +### CPU version for MacOS +PM> Install-Package SciSharp.TensorFlow.Redist-OSX + ### GPU version for Windows (CUDA and cuDNN are required) PM> Install-Package SciSharp.TensorFlow.Redist-Windows-GPU diff --git a/src/TensorFlowNET.Core/APIs/c_api.customize.cs b/src/TensorFlowNET.Core/APIs/c_api.customize.cs index 510e52eb7..bee4897ee 100644 --- a/src/TensorFlowNET.Core/APIs/c_api.customize.cs +++ b/src/TensorFlowNET.Core/APIs/c_api.customize.cs @@ -8,10 +8,10 @@ namespace Tensorflow public partial class c_api { [DllImport(TensorFlowLibName)] - public static extern void TFC_SetAttr(SafeGraphHandle graph, IntPtr op, string attr_name, SafeBufferHandle attr_value_proto, SafeStatusHandle status); + public static extern void TF_SetAttr(SafeGraphHandle graph, IntPtr op, string attr_name, SafeBufferHandle attr_value_proto, SafeStatusHandle status); [DllImport(TensorFlowLibName)] - public static extern SafeBufferHandle TFC_GetHandleShapeAndType(SafeGraphHandle c_graph, TF_Output output); + public static extern SafeBufferHandle TF_GetHandleShapeAndType(SafeGraphHandle c_graph, TF_Output output); [DllImport(TensorFlowLibName)] - public static extern void TFC_SetHandleShapeAndType(SafeGraphHandle c_graph, TF_Output output, byte[] data, long proto_len, SafeStatusHandle status); + public static extern void TF_SetHandleShapeAndType(SafeGraphHandle c_graph, TF_Output output, byte[] data, long proto_len, SafeStatusHandle status); } } diff --git a/src/TensorFlowNET.Core/Operations/Operation.cs b/src/TensorFlowNET.Core/Operations/Operation.cs index e59c381cb..2105c53fa 100644 --- a/src/TensorFlowNET.Core/Operations/Operation.cs +++ b/src/TensorFlowNET.Core/Operations/Operation.cs @@ -437,7 +437,7 @@ internal void _set_attr(string attr_name, AttrValue attr_value) internal void _set_attr_with_buf(string attr_name, Buffer attr_buf) { Status status = new(); - c_api.TFC_SetAttr(graph, _handle, attr_name, attr_buf, status); + c_api.TF_SetAttr(graph, _handle, attr_name, attr_buf, status); status.Check(true); } } diff --git a/src/TensorFlowNET.Core/Operations/handle_data_util.cs b/src/TensorFlowNET.Core/Operations/handle_data_util.cs index a01efc520..363d3144e 100644 --- a/src/TensorFlowNET.Core/Operations/handle_data_util.cs +++ b/src/TensorFlowNET.Core/Operations/handle_data_util.cs @@ -51,7 +51,7 @@ public static void set_handle_data(Tensor target_t, HandleData handle_data) } Status status = new(); var proto = handle_data.ToByteArray(); - c_api.TFC_SetHandleShapeAndType(target_t.graph.c_graph, target_t._as_tf_output(), proto, proto.Length, status); + c_api.TF_SetHandleShapeAndType(target_t.graph.c_graph, target_t._as_tf_output(), proto, proto.Length, status); status.Check(true); } diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index 85c41bd2a..42c0399da 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -4,8 +4,8 @@ netstandard2.0;net6.0 Tensorflow.Binding Tensorflow - 2.11.0 - 0.110.4 + 2.15.0 + 0.150.0 10.0 enable Haiping Chen, Eli Belash, Yaohui Liu, Meinrad Recheis @@ -20,8 +20,11 @@ Google's TensorFlow full binding in .NET Standard. Building, training and infering deep learning models. https://tensorflownet.readthedocs.io - 0.110.3.0 + 0.150.0.0 + tf.net 0.150.x and above are based on tensorflow native 2.15.0 + * Support BERT model. + tf.net 0.110.x and above are based on tensorflow native 2.11.0 * Support RNN, LSTM model. * Support Transformer model. @@ -43,8 +46,9 @@ https://tensorflownet.readthedocs.io tf.net 0.7x.x aligns with TensorFlow v2.7.x native library. tf.net 0.10x.x aligns with TensorFlow v2.10.x native library. tf.net 0.11x.x aligns with TensorFlow v2.11.x native library. + tf.net 0.15x.x aligns with TensorFlow v2.15.x native library. - 0.110.4.0 + 0.150.0.0 LICENSE true packages @@ -176,7 +180,7 @@ https://tensorflownet.readthedocs.io - + diff --git a/src/TensorFlowNET.Core/ops.cs b/src/TensorFlowNET.Core/ops.cs index 351fd18ff..6f51150a2 100644 --- a/src/TensorFlowNET.Core/ops.cs +++ b/src/TensorFlowNET.Core/ops.cs @@ -590,7 +590,7 @@ public static bool inside_function() public static HandleData get_resource_handle_data(Tensor graph_op) { - var handle_data = c_api.TFC_GetHandleShapeAndType(graph_op.graph.c_graph, graph_op._as_tf_output()); + var handle_data = c_api.TF_GetHandleShapeAndType(graph_op.graph.c_graph, graph_op._as_tf_output()); try{ var handle_str = c_api.ByteStringPiece(handle_data.DangerousGetHandle() == IntPtr.Zero ? null : new Buffer(handle_data)); return HandleData.Parser.ParseFrom(handle_str); diff --git a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj index a0ee22284..eb8ebf93c 100644 --- a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj +++ b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj @@ -7,7 +7,7 @@ enable Tensorflow.Keras AnyCPU;x64 - 0.11.4 + 0.15.0 Haiping Chen Keras for .NET Apache 2.0, Haiping Chen since 2018 @@ -30,6 +30,7 @@ * Fixed memory leak for YOLOv3 model. * Support RNN and LSTM models * Support Transformer model + * Support BERT model Keras for .NET @@ -42,8 +43,8 @@ Keras is an API designed for human beings, not machines. Keras follows best prac Git False Open.snk - 0.11.4.0 - 0.11.4.0 + 0.15.0.0 + 0.15.0.0 LICENSE Debug;Release;GPU @@ -143,7 +144,7 @@ Keras is an API designed for human beings, not machines. Keras follows best prac - + diff --git a/src/TensorflowNET.Hub/Tensorflow.Hub.csproj b/src/TensorflowNET.Hub/Tensorflow.Hub.csproj index 3c09f808e..efa37598d 100644 --- a/src/TensorflowNET.Hub/Tensorflow.Hub.csproj +++ b/src/TensorflowNET.Hub/Tensorflow.Hub.csproj @@ -26,7 +26,7 @@ - + diff --git a/tools/TensorFlowNET.Console/Tensorflow.Console.csproj b/tools/TensorFlowNET.Console/Tensorflow.Console.csproj index ecc2d30b5..bb60b6b63 100644 --- a/tools/TensorFlowNET.Console/Tensorflow.Console.csproj +++ b/tools/TensorFlowNET.Console/Tensorflow.Console.csproj @@ -19,13 +19,10 @@ AnyCPU - - - - + diff --git a/tools/Tensorflow.CodeGen/Tensorflow.CodeGen.csproj b/tools/Tensorflow.CodeGen/Tensorflow.CodeGen.csproj index 03195e6ac..2afc68a3c 100644 --- a/tools/Tensorflow.CodeGen/Tensorflow.CodeGen.csproj +++ b/tools/Tensorflow.CodeGen/Tensorflow.CodeGen.csproj @@ -9,7 +9,6 @@ - diff --git a/tools/Tensorflow.UnitTest.RedistHolder/Tensorflow.UnitTest.RedistHolder.csproj b/tools/Tensorflow.UnitTest.RedistHolder/Tensorflow.UnitTest.RedistHolder.csproj index 1ca387dbb..0d1018cab 100644 --- a/tools/Tensorflow.UnitTest.RedistHolder/Tensorflow.UnitTest.RedistHolder.csproj +++ b/tools/Tensorflow.UnitTest.RedistHolder/Tensorflow.UnitTest.RedistHolder.csproj @@ -5,7 +5,7 @@ - + From 53bd70bed3828a81e83bc1a2edbe1b3cbfab197a Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Tue, 7 Nov 2023 22:54:08 +0800 Subject: [PATCH 68/98] fix: fix the validation_pack when multiple input --- src/TensorFlowNET.Core/Util/Data.cs | 26 ++++++++++++++----- .../Engine/DataAdapters/DataAdapter.cs | 14 +++++++--- .../Engine/Model.Evaluate.cs | 8 +++++- src/TensorFlowNET.Keras/Engine/Model.Fit.cs | 23 +++++++++++++--- 4 files changed, 56 insertions(+), 15 deletions(-) diff --git a/src/TensorFlowNET.Core/Util/Data.cs b/src/TensorFlowNET.Core/Util/Data.cs index a14c69b18..4e5a65434 100644 --- a/src/TensorFlowNET.Core/Util/Data.cs +++ b/src/TensorFlowNET.Core/Util/Data.cs @@ -1,4 +1,5 @@ -using Tensorflow.NumPy; +using OneOf; +using Tensorflow.NumPy; namespace Tensorflow.Util { @@ -8,10 +9,10 @@ namespace Tensorflow.Util /// public class ValidationDataPack { - public NDArray val_x; + public OneOf val_x; public NDArray val_y; public NDArray val_sample_weight = null; - + public bool val_x_is_array = false; public ValidationDataPack((NDArray, NDArray) validation_data) { this.val_x = validation_data.Item1; @@ -27,15 +28,17 @@ public ValidationDataPack((NDArray, NDArray, NDArray) validation_data) public ValidationDataPack((IEnumerable, NDArray) validation_data) { - this.val_x = validation_data.Item1.ToArray()[0]; + this.val_x = validation_data.Item1.ToArray(); this.val_y = validation_data.Item2; + val_x_is_array = true; } public ValidationDataPack((IEnumerable, NDArray, NDArray) validation_data) { - this.val_x = validation_data.Item1.ToArray()[0]; + this.val_x = validation_data.Item1.ToArray(); this.val_y = validation_data.Item2; this.val_sample_weight = validation_data.Item3; + val_x_is_array = true; } public static implicit operator ValidationDataPack((NDArray, NDArray) validation_data) @@ -52,15 +55,24 @@ public static implicit operator ValidationDataPack((IEnumerable, NDArra public void Deconstruct(out NDArray val_x, out NDArray val_y) { - val_x = this.val_x; + val_x = this.val_x.AsT0; val_y = this.val_y; } public void Deconstruct(out NDArray val_x, out NDArray val_y, out NDArray val_sample_weight) { - val_x = this.val_x; + val_x = this.val_x.AsT0; + val_y = this.val_y; + val_sample_weight = this.val_sample_weight; + } + + // add a unuse parameter to make it different from Deconstruct(out NDArray val_x, out NDArray val_y, out NDArray val_sample_weight) + public void Deconstruct(out NDArray[] val_x_array, out NDArray val_y, out NDArray val_sample_weight, out NDArray unuse) + { + val_x_array = this.val_x.AsT1; val_y = this.val_y; val_sample_weight = this.val_sample_weight; + unuse = null; } } } diff --git a/src/TensorFlowNET.Keras/Engine/DataAdapters/DataAdapter.cs b/src/TensorFlowNET.Keras/Engine/DataAdapters/DataAdapter.cs index b2750496a..590f30a78 100644 --- a/src/TensorFlowNET.Keras/Engine/DataAdapters/DataAdapter.cs +++ b/src/TensorFlowNET.Keras/Engine/DataAdapters/DataAdapter.cs @@ -92,9 +92,17 @@ public static ((IEnumerable, NDArray, NDArray), ValidationDataPack) tra var train_y = y[new Slice(0, train_count)]; var val_x = x.Select(x => x[new Slice(train_count)] as NDArray); var val_y = y[new Slice(train_count)]; - NDArray tmp_sample_weight = sample_weight; - sample_weight = sample_weight[new Slice(0, train_count)]; - ValidationDataPack validation_data = (val_x, val_y, tmp_sample_weight[new Slice(train_count)]); + + ValidationDataPack validation_data; + if (sample_weight != null) + { + validation_data = (val_x, val_y, sample_weight[new Slice(train_count)]); + sample_weight = sample_weight[new Slice(0, train_count)]; + } + else + { + validation_data = (val_x, val_y); + } return ((train_x, train_y, sample_weight), validation_data); } } diff --git a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs index 474d5e5a5..b3264429e 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs @@ -70,13 +70,19 @@ public Dictionary evaluate(NDArray x, NDArray y, return evaluate(data_handler, callbacks, is_val, test_function); } - public Dictionary evaluate(IEnumerable x, Tensor y, int verbose = 1, bool is_val = false) + public Dictionary evaluate( + IEnumerable x, + Tensor y, + int verbose = 1, + NDArray sample_weight = null, + bool is_val = false) { var data_handler = new DataHandler(new DataHandlerArgs { X = new Tensors(x.ToArray()), Y = y, Model = this, + SampleWeight = sample_weight, StepsPerExecution = _steps_per_execution }); diff --git a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs index d61211c71..13a1b63bc 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs @@ -7,6 +7,7 @@ using System.Diagnostics; using Tensorflow.Keras.Callbacks; using Tensorflow.Util; +using OneOf; namespace Tensorflow.Keras.Engine { @@ -287,10 +288,24 @@ History FitInternal(DataHandler data_handler, int epochs, int verbose, List val_logs; + if (!validation_data.val_x_is_array) + { + (val_x, val_y, val_sample_weight) = validation_data; + // Because evaluate calls call_test_batch_end, this interferes with our output on the screen + // so we need to pass a is_val parameter to stop on_test_batch_end + val_logs = evaluate(val_x, val_y, sample_weight: val_sample_weight, is_val: true); + + } + else + { + (val_x_array, val_y, val_sample_weight, _) = validation_data; + val_logs = evaluate(val_x_array, val_y, sample_weight: val_sample_weight, is_val: true); + } foreach (var log in val_logs) { logs["val_" + log.Key] = log.Value; From d453fb6611f4acb3ab405579ae804279d6e07cbe Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Tue, 7 Nov 2023 23:34:37 +0800 Subject: [PATCH 69/98] refactor: declare some field of ValidationPack as internal --- src/TensorFlowNET.Core/Util/Data.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/TensorFlowNET.Core/Util/Data.cs b/src/TensorFlowNET.Core/Util/Data.cs index 4e5a65434..388efc50f 100644 --- a/src/TensorFlowNET.Core/Util/Data.cs +++ b/src/TensorFlowNET.Core/Util/Data.cs @@ -9,9 +9,9 @@ namespace Tensorflow.Util /// public class ValidationDataPack { - public OneOf val_x; - public NDArray val_y; - public NDArray val_sample_weight = null; + internal OneOf val_x; + internal NDArray val_y; + internal NDArray val_sample_weight = null; public bool val_x_is_array = false; public ValidationDataPack((NDArray, NDArray) validation_data) { @@ -33,7 +33,7 @@ public ValidationDataPack((IEnumerable, NDArray) validation_data) val_x_is_array = true; } - public ValidationDataPack((IEnumerable, NDArray, NDArray) validation_data) + internal ValidationDataPack((IEnumerable, NDArray, NDArray) validation_data) { this.val_x = validation_data.Item1.ToArray(); this.val_y = validation_data.Item2; From 47e9019a187744bf31e315525ffe352dad36a00c Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Tue, 7 Nov 2023 23:36:15 +0800 Subject: [PATCH 70/98] refactor: fix a typo --- src/TensorFlowNET.Core/Util/Data.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Core/Util/Data.cs b/src/TensorFlowNET.Core/Util/Data.cs index 388efc50f..fe3466ed0 100644 --- a/src/TensorFlowNET.Core/Util/Data.cs +++ b/src/TensorFlowNET.Core/Util/Data.cs @@ -33,7 +33,7 @@ public ValidationDataPack((IEnumerable, NDArray) validation_data) val_x_is_array = true; } - internal ValidationDataPack((IEnumerable, NDArray, NDArray) validation_data) + public ValidationDataPack((IEnumerable, NDArray, NDArray) validation_data) { this.val_x = validation_data.Item1.ToArray(); this.val_y = validation_data.Item2; From 2a377e2f91b40083f5de86f01b57b32bad5a5932 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Tue, 7 Nov 2023 19:23:34 +0000 Subject: [PATCH 71/98] tests are passing --- .../Variables/variables.py.cs | 8 ---- test/TensorFlowNET.UnitTest/PythonTest.cs | 40 ++++++++++++------- .../Training/GradientDescentOptimizerTests.cs | 33 +++++++++------ 3 files changed, 46 insertions(+), 35 deletions(-) diff --git a/src/TensorFlowNET.Core/Variables/variables.py.cs b/src/TensorFlowNET.Core/Variables/variables.py.cs index f3ae248e6..91f57e292 100644 --- a/src/TensorFlowNET.Core/Variables/variables.py.cs +++ b/src/TensorFlowNET.Core/Variables/variables.py.cs @@ -154,13 +154,5 @@ public static Operation _safe_initial_value_from_op(string name, Operation op, D return op; } - - public static Tensor global_variables_initializer() - { - // if context.executing_eagerly(): - // return control_flow_ops.no_op(name = "global_variables_initializer") - var group = variables_initializer(global_variables().ToArray()); - return group; - } } } diff --git a/test/TensorFlowNET.UnitTest/PythonTest.cs b/test/TensorFlowNET.UnitTest/PythonTest.cs index 12fd72360..090ef097c 100644 --- a/test/TensorFlowNET.UnitTest/PythonTest.cs +++ b/test/TensorFlowNET.UnitTest/PythonTest.cs @@ -6,6 +6,7 @@ using System.Linq; using Tensorflow; using static Tensorflow.Binding; +using System.Collections.Generic; namespace TensorFlowNET.UnitTest { @@ -144,11 +145,12 @@ public void assertAllClose(double value, NDArray array2, double eps = 1e-5) Assert.IsTrue(np.allclose(array1, array2, rtol: eps)); } - private class CollectionComparer : System.Collections.IComparer + private class CollectionComparer : IComparer { private readonly double _epsilon; - public CollectionComparer(double eps = 1e-06) { + public CollectionComparer(double eps = 1e-06) + { _epsilon = eps; } public int Compare(object x, object y) @@ -166,13 +168,15 @@ public int Compare(object x, object y) } public void assertAllCloseAccordingToType( - T[] expected, - T[] given, + ICollection expected, + ICollection given, double eps = 1e-6, float float_eps = 1e-6f) { // TODO: check if any of arguments is not double and change toletance - CollectionAssert.AreEqual(expected, given, new CollectionComparer(eps)); + // remove givenAsDouble and cast expected instead + var givenAsDouble = given.Select(x => Convert.ToDouble(x)).ToArray(); + CollectionAssert.AreEqual(expected, givenAsDouble, new CollectionComparer(eps)); } public void assertProtoEquals(object toProto, object o) @@ -241,17 +245,25 @@ public T evaluate(Tensor tensor) // return self._eval_helper(tensors) // else: { - var sess = tf.Session(); + var sess = tf.get_default_session(); var ndarray = tensor.eval(sess); - if (typeof(T) == typeof(double)) + if (typeof(T) == typeof(double) + || typeof(T) == typeof(float) + || typeof(T) == typeof(int)) + { + result = Convert.ChangeType(ndarray, typeof(T)); + } + else if (typeof(T) == typeof(double[])) + { + result = ndarray.ToMultiDimArray(); + } + else if (typeof(T) == typeof(float[])) { - double x = ndarray; - result = x; + result = ndarray.ToMultiDimArray(); } - else if (typeof(T) == typeof(int)) + else if (typeof(T) == typeof(int[])) { - int x = ndarray; - result = x; + result = ndarray.ToMultiDimArray(); } else { @@ -457,12 +469,12 @@ private Session _get_cached_session( else { - if (crash_if_inconsistent_args && !self._cached_graph.Equals(graph)) + if (crash_if_inconsistent_args && self._cached_graph != null && !self._cached_graph.Equals(graph)) throw new ValueError(@"The graph used to get the cached session is different than the one that was used to create the session. Maybe create a new session with self.session()"); - if (crash_if_inconsistent_args && !self._cached_config.Equals(config)) + if (crash_if_inconsistent_args && self._cached_config != null && !self._cached_config.Equals(config)) { throw new ValueError(@"The config used to get the cached session is different than the one that was used to create the diff --git a/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs b/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs index 977544ae9..3059068f4 100644 --- a/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs +++ b/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs @@ -1,8 +1,6 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.Linq; -using System.Runtime.Intrinsics.X86; -using System.Security.AccessControl; using Tensorflow.NumPy; using TensorFlowNET.UnitTest; using static Tensorflow.Binding; @@ -12,18 +10,23 @@ namespace Tensorflow.Keras.UnitTest.Optimizers [TestClass] public class GradientDescentOptimizerTest : PythonTest { - private void TestBasicGeneric() where T : struct + private static TF_DataType GetTypeForNumericType() where T : struct { - var dtype = Type.GetTypeCode(typeof(T)) switch + return Type.GetTypeCode(typeof(T)) switch { TypeCode.Single => np.float32, TypeCode.Double => np.float64, _ => throw new NotImplementedException(), }; + } + + private void TestBasicGeneric() where T : struct + { + var dtype = GetTypeForNumericType(); // train.GradientDescentOptimizer is V1 only API. tf.Graph().as_default(); - using (self.cached_session()) + using (var sess = self.cached_session()) { var var0 = tf.Variable(new[] { 1.0, 2.0 }, dtype: dtype); var var1 = tf.Variable(new[] { 3.0, 4.0 }, dtype: dtype); @@ -36,21 +39,25 @@ private void TestBasicGeneric() where T : struct }; var sgd_op = optimizer.apply_gradients(grads_and_vars); - var global_variables = variables.global_variables_initializer(); - self.evaluate(global_variables); + var global_variables = tf.global_variables_initializer(); + sess.run(global_variables); + // Fetch params to validate initial values + var initialVar0 = sess.run(var0); + var valu = var0.eval(sess); + var initialVar1 = sess.run(var1); // TODO: use self.evaluate instead of self.evaluate - self.assertAllCloseAccordingToType(new double[] { 1.0, 2.0 }, self.evaluate(var0)); - self.assertAllCloseAccordingToType(new double[] { 3.0, 4.0 }, self.evaluate(var1)); + self.assertAllCloseAccordingToType(new[] { 1.0, 2.0 }, self.evaluate(var0)); + self.assertAllCloseAccordingToType(new[] { 3.0, 4.0 }, self.evaluate(var1)); // Run 1 step of sgd sgd_op.run(); // Validate updated params self.assertAllCloseAccordingToType( - new double[] { 1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1 }, - self.evaluate(var0)); + new[] { 1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1 }, + self.evaluate(var0)); self.assertAllCloseAccordingToType( - new double[] { 3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01 }, - self.evaluate(var1)); + new[] { 3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01 }, + self.evaluate(var1)); // TODO: self.assertEqual(0, len(optimizer.variables())); } } From f7b8dba00b2465114926072d4a82924dc35596d7 Mon Sep 17 00:00:00 2001 From: Alexander Date: Wed, 8 Nov 2023 15:16:02 +0000 Subject: [PATCH 72/98] small fixes --- .../Training/GradientDescentOptimizerTests.cs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs b/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs index 3059068f4..1a650a864 100644 --- a/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs +++ b/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs @@ -1,4 +1,5 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; +using Microsoft.VisualStudio.TestPlatform.Utilities; +using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.Linq; using Tensorflow.NumPy; @@ -20,7 +21,7 @@ private static TF_DataType GetTypeForNumericType() where T : struct }; } - private void TestBasicGeneric() where T : struct + private void TestBasic() where T : struct { var dtype = GetTypeForNumericType(); @@ -42,11 +43,9 @@ private void TestBasicGeneric() where T : struct var global_variables = tf.global_variables_initializer(); sess.run(global_variables); - // Fetch params to validate initial values var initialVar0 = sess.run(var0); - var valu = var0.eval(sess); var initialVar1 = sess.run(var1); - // TODO: use self.evaluate instead of self.evaluate + // Fetch params to validate initial values self.assertAllCloseAccordingToType(new[] { 1.0, 2.0 }, self.evaluate(var0)); self.assertAllCloseAccordingToType(new[] { 3.0, 4.0 }, self.evaluate(var1)); // Run 1 step of sgd @@ -66,10 +65,9 @@ private void TestBasicGeneric() where T : struct public void TestBasic() { //TODO: add np.half - TestBasicGeneric(); - TestBasicGeneric(); + TestBasic(); + TestBasic(); } - } } From c906f46aadaf2e2f0d1769f026270ba912ef95be Mon Sep 17 00:00:00 2001 From: Alexander Date: Wed, 8 Nov 2023 15:24:13 +0000 Subject: [PATCH 73/98] learning rate test --- .../Training/GradientDescentOptimizerTests.cs | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs b/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs index 1a650a864..92fe97706 100644 --- a/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs +++ b/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs @@ -1,6 +1,7 @@ using Microsoft.VisualStudio.TestPlatform.Utilities; using Microsoft.VisualStudio.TestTools.UnitTesting; using System; +using System.Diagnostics; using System.Linq; using Tensorflow.NumPy; using TensorFlowNET.UnitTest; @@ -69,5 +70,53 @@ public void TestBasic() TestBasic(); } + private void TestTensorLearningRate() where T : struct + { + var dtype = GetTypeForNumericType(); + + // train.GradientDescentOptimizer is V1 only API. + tf.Graph().as_default(); + using (var sess = self.cached_session()) + { + var var0 = tf.Variable(new[] { 1.0, 2.0 }, dtype: dtype); + var var1 = tf.Variable(new[] { 3.0, 4.0 }, dtype: dtype); + var grads0 = tf.constant(new[] { 0.1, 0.1 }, dtype: dtype); + var grads1 = tf.constant(new[] { 0.01, 0.01 }, dtype: dtype); + var lrate = constant_op.constant(3.0); + var grads_and_vars = new[] { + Tuple.Create(grads0, var0 as IVariableV1), + Tuple.Create(grads1, var1 as IVariableV1) + }; + var sgd_op = tf.train.GradientDescentOptimizer(lrate) + .apply_gradients(grads_and_vars); + + var global_variables = tf.global_variables_initializer(); + sess.run(global_variables); + + var initialVar0 = sess.run(var0); + var initialVar1 = sess.run(var1); + // Fetch params to validate initial values + self.assertAllCloseAccordingToType(new[] { 1.0, 2.0 }, self.evaluate(var0)); + self.assertAllCloseAccordingToType(new[] { 3.0, 4.0 }, self.evaluate(var1)); + // Run 1 step of sgd + sgd_op.run(); + // Validate updated params + self.assertAllCloseAccordingToType( + new[] { 1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1 }, + self.evaluate(var0)); + self.assertAllCloseAccordingToType( + new[] { 3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01 }, + self.evaluate(var1)); + // TODO: self.assertEqual(0, len(optimizer.variables())); + } + } + + [TestMethod] + public void TestTensorLearningRate() + { + //TODO: add np.half + TestTensorLearningRate(); + TestTensorLearningRate(); + } } } From 149caaec11b649e6f9e85320a1f18689c32cae6c Mon Sep 17 00:00:00 2001 From: Alexander Date: Fri, 10 Nov 2023 02:44:01 +0000 Subject: [PATCH 74/98] test ci --- .../Training/GradientDescentOptimizerTests.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs b/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs index 92fe97706..98738528d 100644 --- a/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs +++ b/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs @@ -27,8 +27,8 @@ private void TestBasic() where T : struct var dtype = GetTypeForNumericType(); // train.GradientDescentOptimizer is V1 only API. - tf.Graph().as_default(); - using (var sess = self.cached_session()) + //tf.Graph().as_default(); + /*using (var sess = self.cached_session()) { var var0 = tf.Variable(new[] { 1.0, 2.0 }, dtype: dtype); var var1 = tf.Variable(new[] { 3.0, 4.0 }, dtype: dtype); @@ -59,7 +59,7 @@ private void TestBasic() where T : struct new[] { 3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01 }, self.evaluate(var1)); // TODO: self.assertEqual(0, len(optimizer.variables())); - } + }*/ } [TestMethod] @@ -67,7 +67,7 @@ public void TestBasic() { //TODO: add np.half TestBasic(); - TestBasic(); + // TestBasic(); } private void TestTensorLearningRate() where T : struct @@ -115,8 +115,8 @@ private void TestTensorLearningRate() where T : struct public void TestTensorLearningRate() { //TODO: add np.half - TestTensorLearningRate(); - TestTensorLearningRate(); + // TestTensorLearningRate(); + // TestTensorLearningRate(); } } } From 2cb5fd66f842832a2254155f296a54764473f5cd Mon Sep 17 00:00:00 2001 From: Alexander Date: Fri, 10 Nov 2023 13:53:40 +0000 Subject: [PATCH 75/98] new graph --- .../Training/BasicLinearModel.cs | 2 ++ .../Training/GradientDescentOptimizerTests.cs | 17 +++++++---------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/test/TensorFlowNET.UnitTest/Training/BasicLinearModel.cs b/test/TensorFlowNET.UnitTest/Training/BasicLinearModel.cs index 1283ecaf2..a37f28920 100644 --- a/test/TensorFlowNET.UnitTest/Training/BasicLinearModel.cs +++ b/test/TensorFlowNET.UnitTest/Training/BasicLinearModel.cs @@ -15,6 +15,8 @@ public class BasicLinearModel [TestMethod] public void LinearRegression() { + tf.Graph().as_default(); + // Initialize the weights to `5.0` and the bias to `0.0` // In practice, these should be initialized to random values (for example, with `tf.random.normal`) var W = tf.Variable(5.0f); diff --git a/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs b/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs index 98738528d..1632f1e73 100644 --- a/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs +++ b/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs @@ -1,8 +1,5 @@ -using Microsoft.VisualStudio.TestPlatform.Utilities; -using Microsoft.VisualStudio.TestTools.UnitTesting; +using Microsoft.VisualStudio.TestTools.UnitTesting; using System; -using System.Diagnostics; -using System.Linq; using Tensorflow.NumPy; using TensorFlowNET.UnitTest; using static Tensorflow.Binding; @@ -27,8 +24,8 @@ private void TestBasic() where T : struct var dtype = GetTypeForNumericType(); // train.GradientDescentOptimizer is V1 only API. - //tf.Graph().as_default(); - /*using (var sess = self.cached_session()) + tf.Graph().as_default(); + using (var sess = self.cached_session()) { var var0 = tf.Variable(new[] { 1.0, 2.0 }, dtype: dtype); var var1 = tf.Variable(new[] { 3.0, 4.0 }, dtype: dtype); @@ -59,7 +56,7 @@ private void TestBasic() where T : struct new[] { 3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01 }, self.evaluate(var1)); // TODO: self.assertEqual(0, len(optimizer.variables())); - }*/ + } } [TestMethod] @@ -67,7 +64,7 @@ public void TestBasic() { //TODO: add np.half TestBasic(); - // TestBasic(); + TestBasic(); } private void TestTensorLearningRate() where T : struct @@ -115,8 +112,8 @@ private void TestTensorLearningRate() where T : struct public void TestTensorLearningRate() { //TODO: add np.half - // TestTensorLearningRate(); - // TestTensorLearningRate(); + TestTensorLearningRate(); + TestTensorLearningRate(); } } } From 09d466d697e58d97598bbee248ffd7ceb8a7be92 Mon Sep 17 00:00:00 2001 From: Alexander Date: Fri, 10 Nov 2023 14:00:51 +0000 Subject: [PATCH 76/98] ci test --- test/TensorFlowNET.UnitTest/Training/BasicLinearModel.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/TensorFlowNET.UnitTest/Training/BasicLinearModel.cs b/test/TensorFlowNET.UnitTest/Training/BasicLinearModel.cs index a37f28920..d0da1d5b9 100644 --- a/test/TensorFlowNET.UnitTest/Training/BasicLinearModel.cs +++ b/test/TensorFlowNET.UnitTest/Training/BasicLinearModel.cs @@ -15,7 +15,9 @@ public class BasicLinearModel [TestMethod] public void LinearRegression() { - tf.Graph().as_default(); + var graph = tf.Graph().as_default(); + var sess = new Session(graph); + sess.as_default(); // Initialize the weights to `5.0` and the bias to `0.0` // In practice, these should be initialized to random values (for example, with `tf.random.normal`) From c5b4928bd6eaa9fcff9d0e71932cd7c1587d1eb6 Mon Sep 17 00:00:00 2001 From: Alexander Date: Fri, 10 Nov 2023 14:28:41 +0000 Subject: [PATCH 77/98] correct namespace passing --- test/TensorFlowNET.UnitTest/Training/BasicLinearModel.cs | 4 ---- .../Training/GradientDescentOptimizerTests.cs | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/test/TensorFlowNET.UnitTest/Training/BasicLinearModel.cs b/test/TensorFlowNET.UnitTest/Training/BasicLinearModel.cs index d0da1d5b9..1283ecaf2 100644 --- a/test/TensorFlowNET.UnitTest/Training/BasicLinearModel.cs +++ b/test/TensorFlowNET.UnitTest/Training/BasicLinearModel.cs @@ -15,10 +15,6 @@ public class BasicLinearModel [TestMethod] public void LinearRegression() { - var graph = tf.Graph().as_default(); - var sess = new Session(graph); - sess.as_default(); - // Initialize the weights to `5.0` and the bias to `0.0` // In practice, these should be initialized to random values (for example, with `tf.random.normal`) var W = tf.Variable(5.0f); diff --git a/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs b/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs index 1632f1e73..d766890b2 100644 --- a/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs +++ b/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs @@ -1,10 +1,10 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using System; +using Tensorflow; using Tensorflow.NumPy; -using TensorFlowNET.UnitTest; using static Tensorflow.Binding; -namespace Tensorflow.Keras.UnitTest.Optimizers +namespace TensorFlowNET.UnitTest.Training { [TestClass] public class GradientDescentOptimizerTest : PythonTest From fc8f493187bd382bc994c4f79c17b369611cca36 Mon Sep 17 00:00:00 2001 From: Alexander Date: Fri, 10 Nov 2023 20:47:49 +0000 Subject: [PATCH 78/98] common assembly for python test --- TensorFlow.NET.sln | 23 +- .../PythonTest.cs | 448 ------------------ .../TensorFlowNET.Graph.UnitTest.csproj | 1 + .../Tensorflow.Binding.UnitTest.csproj | 1 + .../PythonTest.cs | 3 - .../Tensorflow.UnitTest.csproj | 24 + 6 files changed, 48 insertions(+), 452 deletions(-) delete mode 100644 test/TensorFlowNET.Graph.UnitTest/PythonTest.cs rename test/{TensorFlowNET.UnitTest => Tensorflow.UnitTest}/PythonTest.cs (99%) create mode 100644 test/Tensorflow.UnitTest/Tensorflow.UnitTest.csproj diff --git a/TensorFlow.NET.sln b/TensorFlow.NET.sln index 214b039d4..e0c273568 100644 --- a/TensorFlow.NET.sln +++ b/TensorFlow.NET.sln @@ -39,7 +39,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tensorflow.Benchmark", "too EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tensorflow.Console", "tools\TensorFlowNET.Console\Tensorflow.Console.csproj", "{1DC32255-BA1F-4D6D-A9C9-5BD5ED71CAA0}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TensorFlow.Kernel.UnitTest", "test\TensorFlow.Kernel.UnitTest\TensorFlow.Kernel.UnitTest.csproj", "{654A027D-1364-4729-880B-144DFE1FF5BB}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TensorFlow.Kernel.UnitTest", "test\TensorFlow.Kernel.UnitTest\TensorFlow.Kernel.UnitTest.csproj", "{654A027D-1364-4729-880B-144DFE1FF5BB}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tensorflow.UnitTest", "test\Tensorflow.UnitTest\Tensorflow.UnitTest.csproj", "{A73DF5A6-866E-4AED-9017-AA2EE86368C4}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -342,6 +344,24 @@ Global {654A027D-1364-4729-880B-144DFE1FF5BB}.Release|x64.Build.0 = Release|Any CPU {654A027D-1364-4729-880B-144DFE1FF5BB}.Release|x86.ActiveCfg = Release|Any CPU {654A027D-1364-4729-880B-144DFE1FF5BB}.Release|x86.Build.0 = Release|Any CPU + {A73DF5A6-866E-4AED-9017-AA2EE86368C4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A73DF5A6-866E-4AED-9017-AA2EE86368C4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A73DF5A6-866E-4AED-9017-AA2EE86368C4}.Debug|x64.ActiveCfg = Debug|Any CPU + {A73DF5A6-866E-4AED-9017-AA2EE86368C4}.Debug|x64.Build.0 = Debug|Any CPU + {A73DF5A6-866E-4AED-9017-AA2EE86368C4}.Debug|x86.ActiveCfg = Debug|Any CPU + {A73DF5A6-866E-4AED-9017-AA2EE86368C4}.Debug|x86.Build.0 = Debug|Any CPU + {A73DF5A6-866E-4AED-9017-AA2EE86368C4}.GPU|Any CPU.ActiveCfg = Debug|Any CPU + {A73DF5A6-866E-4AED-9017-AA2EE86368C4}.GPU|Any CPU.Build.0 = Debug|Any CPU + {A73DF5A6-866E-4AED-9017-AA2EE86368C4}.GPU|x64.ActiveCfg = Debug|Any CPU + {A73DF5A6-866E-4AED-9017-AA2EE86368C4}.GPU|x64.Build.0 = Debug|Any CPU + {A73DF5A6-866E-4AED-9017-AA2EE86368C4}.GPU|x86.ActiveCfg = Debug|Any CPU + {A73DF5A6-866E-4AED-9017-AA2EE86368C4}.GPU|x86.Build.0 = Debug|Any CPU + {A73DF5A6-866E-4AED-9017-AA2EE86368C4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A73DF5A6-866E-4AED-9017-AA2EE86368C4}.Release|Any CPU.Build.0 = Release|Any CPU + {A73DF5A6-866E-4AED-9017-AA2EE86368C4}.Release|x64.ActiveCfg = Release|Any CPU + {A73DF5A6-866E-4AED-9017-AA2EE86368C4}.Release|x64.Build.0 = Release|Any CPU + {A73DF5A6-866E-4AED-9017-AA2EE86368C4}.Release|x86.ActiveCfg = Release|Any CPU + {A73DF5A6-866E-4AED-9017-AA2EE86368C4}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -363,6 +383,7 @@ Global {C23563DB-FE21-48E7-A411-87A109E4A899} = {E1A5D2B7-10AF-4876-85C0-7714EF274214} {1DC32255-BA1F-4D6D-A9C9-5BD5ED71CAA0} = {E1A5D2B7-10AF-4876-85C0-7714EF274214} {654A027D-1364-4729-880B-144DFE1FF5BB} = {1B0918B9-65AD-4F34-A287-AF4597B27DBD} + {A73DF5A6-866E-4AED-9017-AA2EE86368C4} = {1B0918B9-65AD-4F34-A287-AF4597B27DBD} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {2DEAD3CC-486B-4918-A607-50B0DE7B114A} diff --git a/test/TensorFlowNET.Graph.UnitTest/PythonTest.cs b/test/TensorFlowNET.Graph.UnitTest/PythonTest.cs deleted file mode 100644 index ccf59f5ae..000000000 --- a/test/TensorFlowNET.Graph.UnitTest/PythonTest.cs +++ /dev/null @@ -1,448 +0,0 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; -using Newtonsoft.Json.Linq; -using Tensorflow.NumPy; -using System; -using System.Collections; -using System.Linq; -using Tensorflow; -using static Tensorflow.Binding; -using OneOf.Types; -using System.Collections.Generic; - -namespace TensorFlowNET.UnitTest -{ - /// - /// Use as base class for test classes to get additional assertions - /// - public class PythonTest - { - #region python compatibility layer - protected PythonTest self { get => this; } - protected int None => -1; - #endregion - - #region pytest assertions - - public void assertItemsEqual(ICollection given, ICollection expected) - { - if (given is Hashtable && expected is Hashtable) - { - Assert.AreEqual(JObject.FromObject(expected).ToString(), JObject.FromObject(given).ToString()); - return; - } - Assert.IsNotNull(expected); - Assert.IsNotNull(given); - var e = expected.OfType().ToArray(); - var g = given.OfType().ToArray(); - Assert.AreEqual(e.Length, g.Length, $"The collections differ in length expected {e.Length} but got {g.Length}"); - for (int i = 0; i < e.Length; i++) - { - /*if (g[i] is NDArray && e[i] is NDArray) - assertItemsEqual((g[i] as NDArray).GetData(), (e[i] as NDArray).GetData()); - else*/ - if (e[i] is ICollection && g[i] is ICollection) - assertEqual(g[i], e[i]); - else - Assert.AreEqual(e[i], g[i], $"Items differ at index {i}, expected {e[i]} but got {g[i]}"); - } - } - - public void assertAllEqual(ICollection given, ICollection expected) - { - assertItemsEqual(given, expected); - } - - public void assertFloat32Equal(float expected, float actual, string msg) - { - float eps = 1e-6f; - Assert.IsTrue(Math.Abs(expected - actual) < eps * Math.Max(1.0f, Math.Abs(expected)), $"{msg}: expected {expected} vs actual {actual}"); - } - - public void assertFloat64Equal(double expected, double actual, string msg) - { - double eps = 1e-16f; - Assert.IsTrue(Math.Abs(expected - actual) < eps * Math.Max(1.0f, Math.Abs(expected)), $"{msg}: expected {expected} vs actual {actual}"); - } - - public void assertEqual(object given, object expected) - { - /*if (given is NDArray && expected is NDArray) - { - assertItemsEqual((given as NDArray).GetData(), (expected as NDArray).GetData()); - return; - }*/ - if (given is Hashtable && expected is Hashtable) - { - Assert.AreEqual(JObject.FromObject(expected).ToString(), JObject.FromObject(given).ToString()); - return; - } - if (given is ICollection && expected is ICollection) - { - assertItemsEqual(given as ICollection, expected as ICollection); - return; - } - if (given is float && expected is float) - { - assertFloat32Equal((float)expected, (float)given, ""); - return; - } - if (given is double && expected is double) - { - assertFloat64Equal((double)expected, (double)given, ""); - return; - } - Assert.AreEqual(expected, given); - } - - public void assertEquals(object given, object expected) - { - assertEqual(given, expected); - } - - public void assert(object given) - { - if (given is bool) - Assert.IsTrue((bool)given); - Assert.IsNotNull(given); - } - - public void assertIsNotNone(object given) - { - Assert.IsNotNull(given); - } - - public void assertFalse(bool cond) - { - Assert.IsFalse(cond); - } - - public void assertTrue(bool cond) - { - Assert.IsTrue(cond); - } - - public void assertAllClose(NDArray array1, NDArray array2, double eps = 1e-5) - { - Assert.IsTrue(np.allclose(array1, array2, rtol: eps)); - } - - public void assertAllClose(double value, NDArray array2, double eps = 1e-5) - { - var array1 = np.ones_like(array2) * value; - // Assert.IsTrue(np.allclose(array1, array2, rtol: eps)); - } - - public void assertProtoEquals(object toProto, object o) - { - throw new NotImplementedException(); - } - - #endregion - - #region tensor evaluation and test session - - private Session _cached_session = null; - private Graph _cached_graph = null; - private object _cached_config = null; - private bool _cached_force_gpu = false; - - private void _ClearCachedSession() - { - if (self._cached_session != null) - { - self._cached_session.Dispose(); - self._cached_session = null; - } - } - - - //protected object _eval_helper(Tensor[] tensors) - //{ - // if (tensors == null) - // return null; - // return nest.map_structure(self._eval_tensor, tensors); - //} - - protected object _eval_tensor(object tensor) - { - if (tensor == null) - return None; - //else if (callable(tensor)) - // return self._eval_helper(tensor()) - else - { - try - { - //TODO: - // if sparse_tensor.is_sparse(tensor): - // return sparse_tensor.SparseTensorValue(tensor.indices, tensor.values, - // tensor.dense_shape) - //return (tensor as Tensor).numpy(); - } - catch (Exception) - { - throw new ValueError("Unsupported type: " + tensor.GetType()); - } - return null; - } - } - - /// - /// This function is used in many original tensorflow unit tests to evaluate tensors - /// in a test session with special settings (for instance constant folding off) - /// - /// - public T evaluate(Tensor tensor) - { - object result = null; - // if context.executing_eagerly(): - // return self._eval_helper(tensors) - // else: - { - var sess = tf.Session(); - var ndarray = tensor.eval(sess); - if (typeof(T) == typeof(double)) - { - double x = ndarray; - result = x; - } - else if (typeof(T) == typeof(int)) - { - int x = ndarray; - result = x; - } - else - { - result = ndarray; - } - - return (T)result; - } - } - - ///Returns a TensorFlow Session for use in executing tests. - public Session cached_session( - Graph graph = null, object config = null, bool use_gpu = false, bool force_gpu = false) - { - // This method behaves differently than self.session(): for performance reasons - // `cached_session` will by default reuse the same session within the same - // test.The session returned by this function will only be closed at the end - // of the test(in the TearDown function). - - // Use the `use_gpu` and `force_gpu` options to control where ops are run.If - // `force_gpu` is True, all ops are pinned to `/ device:GPU:0`. Otherwise, if - // `use_gpu` is True, TensorFlow tries to run as many ops on the GPU as - // possible.If both `force_gpu and `use_gpu` are False, all ops are pinned to - // the CPU. - - // Example: - // python - // class MyOperatorTest(test_util.TensorFlowTestCase) : - // def testMyOperator(self): - // with self.cached_session() as sess: - // valid_input = [1.0, 2.0, 3.0, 4.0, 5.0] - // result = MyOperator(valid_input).eval() - // self.assertEqual(result, [1.0, 2.0, 3.0, 5.0, 8.0] - // invalid_input = [-1.0, 2.0, 7.0] - // with self.assertRaisesOpError("negative input not supported"): - // MyOperator(invalid_input).eval() - - - // Args: - // graph: Optional graph to use during the returned session. - // config: An optional config_pb2.ConfigProto to use to configure the - // session. - // use_gpu: If True, attempt to run as many ops as possible on GPU. - // force_gpu: If True, pin all ops to `/device:GPU:0`. - - // Yields: - // A Session object that should be used as a context manager to surround - // the graph building and execution code in a test case. - - - // TODO: - // if context.executing_eagerly(): - // return self._eval_helper(tensors) - // else: - { - var sess = self._get_cached_session( - graph, config, force_gpu, crash_if_inconsistent_args: true); - using var cached = self._constrain_devices_and_set_default(sess, use_gpu, force_gpu); - return cached; - } - } - - //Returns a TensorFlow Session for use in executing tests. - public Session session(Graph graph = null, object config = null, bool use_gpu = false, bool force_gpu = false) - { - //Note that this will set this session and the graph as global defaults. - - //Use the `use_gpu` and `force_gpu` options to control where ops are run.If - //`force_gpu` is True, all ops are pinned to `/device:GPU:0`. Otherwise, if - //`use_gpu` is True, TensorFlow tries to run as many ops on the GPU as - //possible.If both `force_gpu and `use_gpu` are False, all ops are pinned to - //the CPU. - - //Example: - //```python - //class MyOperatorTest(test_util.TensorFlowTestCase): - // def testMyOperator(self): - // with self.session(use_gpu= True): - // valid_input = [1.0, 2.0, 3.0, 4.0, 5.0] - // result = MyOperator(valid_input).eval() - // self.assertEqual(result, [1.0, 2.0, 3.0, 5.0, 8.0] - // invalid_input = [-1.0, 2.0, 7.0] - // with self.assertRaisesOpError("negative input not supported"): - // MyOperator(invalid_input).eval() - //``` - - //Args: - // graph: Optional graph to use during the returned session. - // config: An optional config_pb2.ConfigProto to use to configure the - // session. - // use_gpu: If True, attempt to run as many ops as possible on GPU. - // force_gpu: If True, pin all ops to `/device:GPU:0`. - - //Yields: - // A Session object that should be used as a context manager to surround - // the graph building and execution code in a test case. - - Session s = null; - //if (context.executing_eagerly()) - // yield None - //else - //{ - s = self._create_session(graph, config, force_gpu); - //} - return s.as_default(); - } - - private Session _constrain_devices_and_set_default(Session sess, bool use_gpu, bool force_gpu) - { - // Set the session and its graph to global default and constrain devices.""" - if (tf.executing_eagerly()) - return null; - else { - sess.graph.as_default(); - sess.as_default(); - { - if (force_gpu) - { - // TODO: - - // Use the name of an actual device if one is detected, or - // '/device:GPU:0' otherwise - /* var gpu_name = gpu_device_name(); - if (!gpu_name) - gpu_name = "/device:GPU:0" - using (sess.graph.device(gpu_name)) { - yield return sess; - }*/ - return sess; - } - else if (use_gpu) - return sess; - else - using (sess.graph.device("/device:CPU:0")) - return sess; - } - - } - } - - // See session() for details. - private Session _create_session(Graph graph, object cfg, bool forceGpu) - { - var prepare_config = new Func((config) => - { - // """Returns a config for sessions. - // Args: - // config: An optional config_pb2.ConfigProto to use to configure the - // session. - // Returns: - // A config_pb2.ConfigProto object. - - //TODO: config - - // # use_gpu=False. Currently many tests rely on the fact that any device - // # will be used even when a specific device is supposed to be used. - // allow_soft_placement = not force_gpu - // if config is None: - // config = config_pb2.ConfigProto() - // config.allow_soft_placement = allow_soft_placement - // config.gpu_options.per_process_gpu_memory_fraction = 0.3 - // elif not allow_soft_placement and config.allow_soft_placement: - // config_copy = config_pb2.ConfigProto() - // config_copy.CopyFrom(config) - // config = config_copy - // config.allow_soft_placement = False - // # Don't perform optimizations for tests so we don't inadvertently run - // # gpu ops on cpu - // config.graph_options.optimizer_options.opt_level = -1 - // # Disable Grappler constant folding since some tests & benchmarks - // # use constant input and become meaningless after constant folding. - // # DO NOT DISABLE GRAPPLER OPTIMIZERS WITHOUT CONSULTING WITH THE - // # GRAPPLER TEAM. - // config.graph_options.rewrite_options.constant_folding = ( - // rewriter_config_pb2.RewriterConfig.OFF) - // config.graph_options.rewrite_options.pin_to_host_optimization = ( - // rewriter_config_pb2.RewriterConfig.OFF) - return config; - }); - //TODO: use this instead of normal session - //return new ErrorLoggingSession(graph = graph, config = prepare_config(config)) - return new Session(graph);//, config = prepare_config(config)) - } - - private Session _get_cached_session( - Graph graph = null, - object config = null, - bool force_gpu = false, - bool crash_if_inconsistent_args = true) - { - // See cached_session() for documentation. - if (self._cached_session == null) - { - var sess = self._create_session(graph, config, force_gpu); - self._cached_session = sess; - self._cached_graph = graph; - self._cached_config = config; - self._cached_force_gpu = force_gpu; - return sess; - } else { - - if (crash_if_inconsistent_args && !self._cached_graph.Equals(graph)) - throw new ValueError(@"The graph used to get the cached session is - different than the one that was used to create the - session. Maybe create a new session with - self.session()"); - if (crash_if_inconsistent_args && !self._cached_config.Equals(config)) { - throw new ValueError(@"The config used to get the cached session is - different than the one that was used to create the - session. Maybe create a new session with - self.session()"); - } - if (crash_if_inconsistent_args && !self._cached_force_gpu.Equals(force_gpu)) { - throw new ValueError(@"The force_gpu value used to get the cached session is - different than the one that was used to create the - session. Maybe create a new session with - self.session()"); - } - return _cached_session; - } - } - - [TestCleanup] - public void Cleanup() - { - _ClearCachedSession(); - } - - #endregion - - public void AssetSequenceEqual(T[] a, T[] b) - { - Assert.IsTrue(Enumerable.SequenceEqual(a, b)); - } - } -} diff --git a/test/TensorFlowNET.Graph.UnitTest/TensorFlowNET.Graph.UnitTest.csproj b/test/TensorFlowNET.Graph.UnitTest/TensorFlowNET.Graph.UnitTest.csproj index 78a0938c5..74663c1cb 100644 --- a/test/TensorFlowNET.Graph.UnitTest/TensorFlowNET.Graph.UnitTest.csproj +++ b/test/TensorFlowNET.Graph.UnitTest/TensorFlowNET.Graph.UnitTest.csproj @@ -36,6 +36,7 @@ + diff --git a/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj b/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj index 7a6a7f92c..5264cb104 100644 --- a/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj +++ b/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj @@ -51,6 +51,7 @@ + diff --git a/test/TensorFlowNET.UnitTest/PythonTest.cs b/test/Tensorflow.UnitTest/PythonTest.cs similarity index 99% rename from test/TensorFlowNET.UnitTest/PythonTest.cs rename to test/Tensorflow.UnitTest/PythonTest.cs index 090ef097c..b2412ea9f 100644 --- a/test/TensorFlowNET.UnitTest/PythonTest.cs +++ b/test/Tensorflow.UnitTest/PythonTest.cs @@ -1,12 +1,9 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Newtonsoft.Json.Linq; using Tensorflow.NumPy; -using System; using System.Collections; -using System.Linq; using Tensorflow; using static Tensorflow.Binding; -using System.Collections.Generic; namespace TensorFlowNET.UnitTest { diff --git a/test/Tensorflow.UnitTest/Tensorflow.UnitTest.csproj b/test/Tensorflow.UnitTest/Tensorflow.UnitTest.csproj new file mode 100644 index 000000000..66a7d63bd --- /dev/null +++ b/test/Tensorflow.UnitTest/Tensorflow.UnitTest.csproj @@ -0,0 +1,24 @@ + + + + net6.0 + enable + enable + + false + true + + + + + + + + + + + + + + + From 165e9169e49841bb2d326ff903949244565a1a00 Mon Sep 17 00:00:00 2001 From: Alexander Date: Fri, 10 Nov 2023 21:01:12 +0000 Subject: [PATCH 79/98] assert all close --- .../GradientTest/GradientTest.cs | 22 +------------------ test/Tensorflow.UnitTest/PythonTest.cs | 18 +++++++-------- 2 files changed, 10 insertions(+), 30 deletions(-) diff --git a/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs b/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs index e2d6db912..cea6de172 100644 --- a/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs +++ b/test/TensorFlowNET.Graph.UnitTest/GradientTest/GradientTest.cs @@ -625,25 +625,6 @@ public void testPartialDerivatives() } } - // TODO: remove when np.testing.assert_allclose(a, b) is implemented - private class CollectionComparer : System.Collections.IComparer - { - private readonly double _epsilon = 1e-07; - - public int Compare(object x, object y) - { - var a = (double)x; - var b = (double)y; - - double delta = Math.Abs(a - b); - if (delta < _epsilon) - { - return 0; - } - return a.CompareTo(b); - } - } - private struct Case { public Tensor[] grad1; @@ -748,8 +729,7 @@ Tensor[] gradients(Tensor[] ys, Tensor[] xs, Tensor[] stop_gradients = null) var npgrad2 = result[1]; foreach (var (a, b) in npgrad1.Zip(npgrad2)) { - // TODO: np.testing.assert_allclose(a, b); - CollectionAssert.AreEqual(a.ToArray(), b.ToArray(), new CollectionComparer()); + self.assertAllClose(a, b); } } } diff --git a/test/Tensorflow.UnitTest/PythonTest.cs b/test/Tensorflow.UnitTest/PythonTest.cs index b2412ea9f..650f70f2c 100644 --- a/test/Tensorflow.UnitTest/PythonTest.cs +++ b/test/Tensorflow.UnitTest/PythonTest.cs @@ -185,9 +185,9 @@ public void assertProtoEquals(object toProto, object o) #region tensor evaluation and test session - private Session _cached_session = null; - private Graph _cached_graph = null; - private object _cached_config = null; + private Session? _cached_session = null; + private Graph? _cached_graph = null; + private object? _cached_config = null; private bool _cached_force_gpu = false; private void _ClearCachedSession() @@ -237,7 +237,7 @@ protected object _eval_tensor(object tensor) /// public T evaluate(Tensor tensor) { - object result = null; + object? result = null; // if context.executing_eagerly(): // return self._eval_helper(tensors) // else: @@ -274,7 +274,7 @@ public T evaluate(Tensor tensor) ///Returns a TensorFlow Session for use in executing tests. public Session cached_session( - Graph graph = null, object config = null, bool use_gpu = false, bool force_gpu = false) + Graph? graph = null, object? config = null, bool use_gpu = false, bool force_gpu = false) { // This method behaves differently than self.session(): for performance reasons // `cached_session` will by default reuse the same session within the same @@ -325,7 +325,7 @@ public Session cached_session( } //Returns a TensorFlow Session for use in executing tests. - public Session session(Graph graph = null, object config = null, bool use_gpu = false, bool force_gpu = false) + public Session session(Graph? graph = null, object? config = null, bool use_gpu = false, bool force_gpu = false) { //Note that this will set this session and the graph as global defaults. @@ -359,7 +359,7 @@ public Session session(Graph graph = null, object config = null, bool use_gpu = // A Session object that should be used as a context manager to surround // the graph building and execution code in a test case. - Session s = null; + Session? s = null; //if (context.executing_eagerly()) // yield None //else @@ -448,8 +448,8 @@ private Session _create_session(Graph graph, object cfg, bool forceGpu) } private Session _get_cached_session( - Graph graph = null, - object config = null, + Graph? graph = null, + object? config = null, bool force_gpu = false, bool crash_if_inconsistent_args = true) { From b906c9a69a15ad413f519db741335bdb1aedf07a Mon Sep 17 00:00:00 2001 From: Alexander Date: Fri, 10 Nov 2023 21:16:42 +0000 Subject: [PATCH 80/98] fix nullability --- .../Tensorflow.Keras.UnitTest.csproj | 1 + test/Tensorflow.UnitTest/PythonTest.cs | 29 ++++++++++++++----- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj index 3910eba1c..e8b8d42b3 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj +++ b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj @@ -25,6 +25,7 @@ + diff --git a/test/Tensorflow.UnitTest/PythonTest.cs b/test/Tensorflow.UnitTest/PythonTest.cs index 650f70f2c..5d1b1e0e1 100644 --- a/test/Tensorflow.UnitTest/PythonTest.cs +++ b/test/Tensorflow.UnitTest/PythonTest.cs @@ -86,9 +86,9 @@ public void assertEqual(object given, object expected) Assert.AreEqual(JObject.FromObject(expected).ToString(), JObject.FromObject(given).ToString()); return; } - if (given is ICollection && expected is ICollection) + if (given is ICollection collectionGiven && expected is ICollection collectionExpected) { - assertItemsEqual(given as ICollection, expected as ICollection); + assertItemsEqual(collectionGiven, collectionExpected); return; } if (given is float && expected is float) @@ -150,8 +150,21 @@ public CollectionComparer(double eps = 1e-06) { _epsilon = eps; } - public int Compare(object x, object y) + public int Compare(object? x, object? y) { + if (x == null && y == null) + { + return 0; + } + else if (x == null) + { + return -1; + } + else if (y == null) + { + return 1; + } + var a = (double)x; var b = (double)y; @@ -206,7 +219,7 @@ private void _ClearCachedSession() // return nest.map_structure(self._eval_tensor, tensors); //} - protected object _eval_tensor(object tensor) + protected object? _eval_tensor(object tensor) { if (tensor == null) return None; @@ -273,7 +286,7 @@ public T evaluate(Tensor tensor) ///Returns a TensorFlow Session for use in executing tests. - public Session cached_session( + public Session? cached_session( Graph? graph = null, object? config = null, bool use_gpu = false, bool force_gpu = false) { // This method behaves differently than self.session(): for performance reasons @@ -369,7 +382,7 @@ public Session session(Graph? graph = null, object? config = null, bool use_gpu return s.as_default(); } - private Session _constrain_devices_and_set_default(Session sess, bool use_gpu, bool force_gpu) + private Session? _constrain_devices_and_set_default(Session sess, bool use_gpu, bool force_gpu) { // Set the session and its graph to global default and constrain devices.""" if (tf.executing_eagerly()) @@ -404,7 +417,7 @@ private Session _constrain_devices_and_set_default(Session sess, bool use_gpu, b } // See session() for details. - private Session _create_session(Graph graph, object cfg, bool forceGpu) + private Session _create_session(Graph? graph, object? cfg, bool forceGpu) { var prepare_config = new Func((config) => { @@ -485,7 +498,7 @@ different than the one that was used to create the session. Maybe create a new session with self.session()"); } - return _cached_session; + return self._cached_session; } } From b6db9410b3c66ad30ac900330708060231e39809 Mon Sep 17 00:00:00 2001 From: Alexander Date: Fri, 10 Nov 2023 21:20:13 +0000 Subject: [PATCH 81/98] update packages --- .../TensorFlow.Kernel.UnitTest.csproj | 2 +- .../TensorFlowNET.Graph.UnitTest.csproj | 2 +- .../Tensorflow.Keras.UnitTest.csproj | 2 +- .../Tensorflow.Native.UnitTest.csproj | 2 +- test/Tensorflow.UnitTest/Tensorflow.UnitTest.csproj | 4 ++-- .../TensorflowNET.Hub.Unittest/Tensorflow.Hub.Unittest.csproj | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/TensorFlow.Kernel.UnitTest/TensorFlow.Kernel.UnitTest.csproj b/test/TensorFlow.Kernel.UnitTest/TensorFlow.Kernel.UnitTest.csproj index 21b2731b7..461993408 100644 --- a/test/TensorFlow.Kernel.UnitTest/TensorFlow.Kernel.UnitTest.csproj +++ b/test/TensorFlow.Kernel.UnitTest/TensorFlow.Kernel.UnitTest.csproj @@ -10,7 +10,7 @@ - + diff --git a/test/TensorFlowNET.Graph.UnitTest/TensorFlowNET.Graph.UnitTest.csproj b/test/TensorFlowNET.Graph.UnitTest/TensorFlowNET.Graph.UnitTest.csproj index 74663c1cb..40dd53f74 100644 --- a/test/TensorFlowNET.Graph.UnitTest/TensorFlowNET.Graph.UnitTest.csproj +++ b/test/TensorFlowNET.Graph.UnitTest/TensorFlowNET.Graph.UnitTest.csproj @@ -24,7 +24,7 @@ - + diff --git a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj index e8b8d42b3..edac1c2ff 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj +++ b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj @@ -13,7 +13,7 @@ - + diff --git a/test/TensorFlowNET.Native.UnitTest/Tensorflow.Native.UnitTest.csproj b/test/TensorFlowNET.Native.UnitTest/Tensorflow.Native.UnitTest.csproj index a4f1ec567..c054a8707 100644 --- a/test/TensorFlowNET.Native.UnitTest/Tensorflow.Native.UnitTest.csproj +++ b/test/TensorFlowNET.Native.UnitTest/Tensorflow.Native.UnitTest.csproj @@ -44,7 +44,7 @@ - + diff --git a/test/Tensorflow.UnitTest/Tensorflow.UnitTest.csproj b/test/Tensorflow.UnitTest/Tensorflow.UnitTest.csproj index 66a7d63bd..9ad6bc7a5 100644 --- a/test/Tensorflow.UnitTest/Tensorflow.UnitTest.csproj +++ b/test/Tensorflow.UnitTest/Tensorflow.UnitTest.csproj @@ -1,4 +1,4 @@ - + net6.0 @@ -10,7 +10,7 @@ - + diff --git a/test/TensorflowNET.Hub.Unittest/Tensorflow.Hub.Unittest.csproj b/test/TensorflowNET.Hub.Unittest/Tensorflow.Hub.Unittest.csproj index 4c3918e4a..c93b89256 100644 --- a/test/TensorflowNET.Hub.Unittest/Tensorflow.Hub.Unittest.csproj +++ b/test/TensorflowNET.Hub.Unittest/Tensorflow.Hub.Unittest.csproj @@ -9,7 +9,7 @@ - + From 7968dc360fbcbb57265e8a49192c8b028e9d0196 Mon Sep 17 00:00:00 2001 From: Alexander Date: Sat, 11 Nov 2023 05:54:38 +0000 Subject: [PATCH 82/98] fix test --- test/Tensorflow.UnitTest/PythonTest.cs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test/Tensorflow.UnitTest/PythonTest.cs b/test/Tensorflow.UnitTest/PythonTest.cs index 5d1b1e0e1..dff652933 100644 --- a/test/Tensorflow.UnitTest/PythonTest.cs +++ b/test/Tensorflow.UnitTest/PythonTest.cs @@ -133,13 +133,23 @@ public void assertTrue(bool cond) public void assertAllClose(NDArray array1, NDArray array2, double eps = 1e-5) { - Assert.IsTrue(np.allclose(array1, array2, rtol: eps)); + CollectionAssert.AreEqual(array1.ToArray(), array2.ToArray(), new CollectionComparer(eps)); + + //TODO: Assert.IsTrue(np.allclose(array1, array2, rtol: eps)); } public void assertAllClose(double value, NDArray array2, double eps = 1e-5) { + if (array2.shape.IsScalar) + { + double value2 = array2; + Assert.AreEqual(value, value2, eps); + return; + } var array1 = np.ones_like(array2) * value; - Assert.IsTrue(np.allclose(array1, array2, rtol: eps)); + CollectionAssert.AreEqual(array1.ToArray(), array2.ToArray(), new CollectionComparer(eps)); + + //TODO: Assert.IsTrue(np.allclose(array1, array2, rtol: eps)); } private class CollectionComparer : IComparer @@ -158,7 +168,7 @@ public int Compare(object? x, object? y) } else if (x == null) { - return -1; + return -1; } else if (y == null) { From d54f7a62e0e66dee73eff78ce5c93acb195ce813 Mon Sep 17 00:00:00 2001 From: Alexander Date: Mon, 13 Nov 2023 10:33:14 +0000 Subject: [PATCH 83/98] test: more gradients tests --- .../Training/GradientDescentOptimizerTests.cs | 113 ++++++++++++++++++ test/Tensorflow.UnitTest/PythonTest.cs | 45 +++++-- 2 files changed, 149 insertions(+), 9 deletions(-) diff --git a/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs b/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs index d766890b2..f7062f00d 100644 --- a/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs +++ b/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs @@ -1,5 +1,6 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using System; +using System.Linq; using Tensorflow; using Tensorflow.NumPy; using static Tensorflow.Binding; @@ -67,6 +68,51 @@ public void TestBasic() TestBasic(); } + private void TestMinimizeResourceVariable() where T : struct + { + var dtype = GetTypeForNumericType(); + + // train.GradientDescentOptimizer is V1 only API. + tf.Graph().as_default(); + using (var sess = self.cached_session()) + { + var var0 = tf.Variable(new[,] { { 1.0f, 2.0f } }, dtype: dtype); + var var1 = tf.Variable(new[] { 3.0 }, dtype: dtype); + var x = tf.constant(new[,] { { 4.0f }, { 5.0f } }, dtype: dtype); + + var pred = math_ops.matmul(var0, x) + var1; + var loss = pred * pred; + var sgd_op = tf.train.GradientDescentOptimizer(3.0f).minimize(loss); + + var global_variables = tf.global_variables_initializer(); + sess.run(global_variables); + + sess.run(new[] { var0, var1 }); + // Fetch params to validate initial values + self.assertAllCloseAccordingToType(new[,] { { 1.0, 2.0 } }, self.evaluate(var0)); + self.assertAllCloseAccordingToType(new[] { 3.0 }, self.evaluate(var1)); + // Run 1 step of sgd + sgd_op.run(); + // Validate updated params + var np_pred = 1.0 * 4.0 + 2.0 * 5.0 + 3.0; + var np_grad = 2 * np_pred; + self.assertAllCloseAccordingToType( + new[,] { { 1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0 } }, + self.evaluate(var0)); + self.assertAllCloseAccordingToType( + new[] { 3.0 - np_grad }, + self.evaluate(var1)); + } + } + + [TestMethod] + public void TestMinimizeResourceVariable() + { + //TODO: add np.half + TestMinimizeResourceVariable(); + TestMinimizeResourceVariable(); + } + private void TestTensorLearningRate() where T : struct { var dtype = GetTypeForNumericType(); @@ -115,5 +161,72 @@ public void TestTensorLearningRate() TestTensorLearningRate(); TestTensorLearningRate(); } + + public void TestGradWrtRef() where T : struct + { + var dtype = GetTypeForNumericType(); + + var graph = tf.Graph().as_default(); + using (var sess = self.cached_session()) + { + var opt = tf.train.GradientDescentOptimizer(3.0f); + var values = new[] { 1.0, 3.0 }; + var vars_ = values.Select( + v => tf.Variable(new[] { v }, dtype: dtype) as IVariableV1 + ).ToList(); + var grads_and_vars = opt.compute_gradients(tf.add(vars_[0], vars_[1]), vars_); + sess.run(tf.global_variables_initializer()); + foreach (var (grad, _) in grads_and_vars) + self.assertAllCloseAccordingToType(new[] { 1.0 }, self.evaluate(grad)); + + } + } + + [TestMethod] + public void TestGradWrtRef() + { + TestGradWrtRef(); + TestGradWrtRef(); + } + + public void TestWithGlobalStep() where T : struct + { + var dtype = GetTypeForNumericType(); + + tf.Graph().as_default(); + using (var sess = self.cached_session()) + { + var global_step = tf.Variable(0, trainable: false); + var var0 = tf.Variable(new[] { 1.0, 2.0 }, dtype: dtype); + var var1 = tf.Variable(new[] { 3.0, 4.0 }, dtype: dtype); + var grads0 = tf.constant(new[] { 0.1, 0.1 }, dtype: dtype); + var grads1 = tf.constant(new[] { 0.01, 0.01 }, dtype: dtype); + var grads_and_vars = new[] { + Tuple.Create(grads0, var0 as IVariableV1), + Tuple.Create(grads1, var1 as IVariableV1) + }; + var sgd_op = tf.train.GradientDescentOptimizer(3.0f) + .apply_gradients(grads_and_vars, global_step: global_step); + + sess.run(tf.global_variables_initializer()); + // Fetch params to validate initial values + self.assertAllCloseAccordingToType(new[] { 1.0, 2.0 }, self.evaluate(var0)); + self.assertAllCloseAccordingToType(new[] { 3.0, 4.0 }, self.evaluate(var1)); + // Run 1 step of sgd + sgd_op.run(); + // Validate updated params and global_step + self.assertAllCloseAccordingToType(new[] { 1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1 }, self.evaluate(var0)); + self.assertAllCloseAccordingToType(new[] { 3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01 }, self.evaluate(var1)); + Assert.AreEqual(1, self.evaluate(global_step)); + } + + } + + [TestMethod] + public void TestWithGlobalStep() + { + TestWithGlobalStep(); + TestWithGlobalStep(); + } } } diff --git a/test/Tensorflow.UnitTest/PythonTest.cs b/test/Tensorflow.UnitTest/PythonTest.cs index dff652933..1ccd39f02 100644 --- a/test/Tensorflow.UnitTest/PythonTest.cs +++ b/test/Tensorflow.UnitTest/PythonTest.cs @@ -175,8 +175,8 @@ public int Compare(object? x, object? y) return 1; } - var a = (double)x; - var b = (double)y; + var a = Convert.ToDouble(x); + var b = Convert.ToDouble(y); double delta = Math.Abs(a - b); if (delta < _epsilon) @@ -187,6 +187,19 @@ public int Compare(object? x, object? y) } } + public void assertAllCloseAccordingToType( + double[,] expected, + T[,] given, + double eps = 1e-6, + float float_eps = 1e-6f) + { + Assert.AreEqual(expected.GetLength(0), given.GetLength(0)); + Assert.AreEqual(expected.GetLength(1), given.GetLength(1)); + + var flattenGiven = given.Cast().ToArray(); + assertAllCloseAccordingToType(expected, flattenGiven, eps, float_eps); + } + public void assertAllCloseAccordingToType( ICollection expected, ICollection given, @@ -267,21 +280,35 @@ public T evaluate(Tensor tensor) { var sess = tf.get_default_session(); var ndarray = tensor.eval(sess); - if (typeof(T) == typeof(double) - || typeof(T) == typeof(float) - || typeof(T) == typeof(int)) + + if (typeof(T) == typeof(int)) + { + int i = ndarray; + result = i; + } + else if (typeof(T) == typeof(float)) + { + float f = ndarray; + result = f; + } + else if (typeof(T) == typeof(double)) { - result = Convert.ChangeType(ndarray, typeof(T)); + double d = ndarray; + result = d; } - else if (typeof(T) == typeof(double[])) + else if ( + typeof(T) == typeof(double[]) + || typeof(T) == typeof(double[,])) { result = ndarray.ToMultiDimArray(); } - else if (typeof(T) == typeof(float[])) + else if (typeof(T) == typeof(float[]) + || typeof(T) == typeof(float[,])) { result = ndarray.ToMultiDimArray(); } - else if (typeof(T) == typeof(int[])) + else if (typeof(T) == typeof(int[]) + || typeof(T) == typeof(int[,])) { result = ndarray.ToMultiDimArray(); } From eb0f02577290d930930349870b161e85553e967a Mon Sep 17 00:00:00 2001 From: barfeous Date: Mon, 12 Feb 2024 13:28:54 -0600 Subject: [PATCH 84/98] avoid modifying collection --- .../Training/Saving/SavedModel/AugmentedGraphView.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/AugmentedGraphView.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/AugmentedGraphView.cs index a91933357..c6b26ff49 100644 --- a/src/TensorFlowNET.Core/Training/Saving/SavedModel/AugmentedGraphView.cs +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/AugmentedGraphView.cs @@ -88,7 +88,7 @@ private ConcreteFunction maybe_uncache_variable_captures(ConcreteFunction concre public override (IList, IDictionary>) breadth_first_traversal() { - Trackable get_merged_trackable(Trackable x) + void merged_trackable(Trackable x) { // TODO: complete it with new definitions `Asset` and `TrackableConstant`. return x; @@ -100,7 +100,7 @@ Trackable get_merged_trackable(Trackable x) // skip the deletion of cache (maybe do it later). foreach(var pair in _children_cache[obj]) { - _children_cache[obj][pair.Key] = get_merged_trackable(pair.Value); + merged_trackable(pair.Value); } } From 3448b6434680270026a0f938e913ff1f08f1df9b Mon Sep 17 00:00:00 2001 From: barfeous Date: Wed, 14 Feb 2024 20:25:15 -0600 Subject: [PATCH 85/98] Remove parameter return from newly void local method --- .../Training/Saving/SavedModel/AugmentedGraphView.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/AugmentedGraphView.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/AugmentedGraphView.cs index c6b26ff49..3b4bbdc63 100644 --- a/src/TensorFlowNET.Core/Training/Saving/SavedModel/AugmentedGraphView.cs +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/AugmentedGraphView.cs @@ -91,8 +91,8 @@ public override (IList, IDictionary Date: Mon, 11 Mar 2024 03:05:42 +0800 Subject: [PATCH 86/98] docs: update README.md --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 0198c873c..75cad0aa7 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,14 @@ English | [中文](docs/README-CN.md) +> [!IMPORTANT] +> We're happy that our work on tensorflow.net has attracted many users. However, at this time, none of the main maintainers of this repo is available for new features and bug fix. We won't refuse PRs and will help to review them. +> +> If you would like to be a contributor or maintainer of tensorflow.net, we'd like to help you to start up. +> +> We feel sorry for that and we'll resume the maintaining for this project once one of us has bandwidth for it. +> + *master branch and v0.100.x is corresponding to tensorflow v2.10, v0.6x branch is from tensorflow v2.6, v0.15-tensorflow1.15 is from tensorflow1.15. Please add `https://www.myget.org/F/scisharp/api/v3/index.json` to nuget source to use nightly release.* From 4a31621a5632c7d6b2ebca1d36561458b91367c5 Mon Sep 17 00:00:00 2001 From: barfeous Date: Sun, 28 Apr 2024 13:04:07 -0500 Subject: [PATCH 87/98] Use TryGetValue instead of ContainsKey + [] --- .../Training/Saving/SavedModel/AugmentedGraphView.cs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/AugmentedGraphView.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/AugmentedGraphView.cs index 3b4bbdc63..9d0b3f001 100644 --- a/src/TensorFlowNET.Core/Training/Saving/SavedModel/AugmentedGraphView.cs +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/AugmentedGraphView.cs @@ -109,15 +109,11 @@ void merged_trackable(Trackable x) public List<(string, Trackable)> list_dependencies(Trackable obj) { - IDictionary children; - if (!_children_cache.ContainsKey(obj)) + if (!_children_cache.TryGetValue(obj, out var children)) { children= new Dictionary(); } - else - { - children= _children_cache[obj]; - } + List<(string, Trackable)> res = new(); foreach(var pair in obj.deserialization_dependencies(children)) { From f5ba382e49ab0132308739c219ea09b6ac254223 Mon Sep 17 00:00:00 2001 From: Schoen Tannenbaum <169845314+SchoenTannenbaum@users.noreply.github.com> Date: Mon, 20 May 2024 12:09:06 -0400 Subject: [PATCH 88/98] Regularizer addition and fixes --- .../Keras/Regularizers/IRegularizer.cs | 17 ++++-- .../CustomizedRegularizerJsonConverter.cs | 57 +++++++++++++++++++ .../Operations/Regularizers/L1.cs | 33 +++++++++++ .../Operations/Regularizers/L1L2.cs | 48 ++++++++++++++++ .../Operations/Regularizers/L2.cs | 33 +++++++++++ src/TensorFlowNET.Keras/Regularizers.cs | 19 +++++-- src/TensorFlowNET.Keras/Regularizers/L1.cs | 19 ------- src/TensorFlowNET.Keras/Regularizers/L1L2.cs | 24 -------- src/TensorFlowNET.Keras/Regularizers/L2.cs | 17 ------ 9 files changed, 198 insertions(+), 69 deletions(-) create mode 100644 src/TensorFlowNET.Core/Keras/Saving/Json/CustomizedRegularizerJsonConverter.cs create mode 100644 src/TensorFlowNET.Core/Operations/Regularizers/L1.cs create mode 100644 src/TensorFlowNET.Core/Operations/Regularizers/L1L2.cs create mode 100644 src/TensorFlowNET.Core/Operations/Regularizers/L2.cs delete mode 100644 src/TensorFlowNET.Keras/Regularizers/L1.cs delete mode 100644 src/TensorFlowNET.Keras/Regularizers/L1L2.cs delete mode 100644 src/TensorFlowNET.Keras/Regularizers/L2.cs diff --git a/src/TensorFlowNET.Core/Keras/Regularizers/IRegularizer.cs b/src/TensorFlowNET.Core/Keras/Regularizers/IRegularizer.cs index f4045c7b2..e5de76ddb 100644 --- a/src/TensorFlowNET.Core/Keras/Regularizers/IRegularizer.cs +++ b/src/TensorFlowNET.Core/Keras/Regularizers/IRegularizer.cs @@ -1,7 +1,16 @@ -namespace Tensorflow.Keras +using Newtonsoft.Json; +using System.Collections.Generic; +using Tensorflow.Keras.Saving.Common; + +namespace Tensorflow.Keras { - public interface IRegularizer - { - Tensor Apply(RegularizerArgs args); + [JsonConverter(typeof(CustomizedRegularizerJsonConverter))] + public interface IRegularizer + { + [JsonProperty("class_name")] + string ClassName { get; } + [JsonProperty("config")] + IDictionary Config { get; } + Tensor Apply(RegularizerArgs args); } } diff --git a/src/TensorFlowNET.Core/Keras/Saving/Json/CustomizedRegularizerJsonConverter.cs b/src/TensorFlowNET.Core/Keras/Saving/Json/CustomizedRegularizerJsonConverter.cs new file mode 100644 index 000000000..4b1790aca --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Saving/Json/CustomizedRegularizerJsonConverter.cs @@ -0,0 +1,57 @@ +using Newtonsoft.Json.Linq; +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Operations.Regularizers; + +namespace Tensorflow.Keras.Saving.Common +{ + class RegularizerInfo + { + public string class_name { get; set; } + public JObject config { get; set; } + } + + public class CustomizedRegularizerJsonConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return objectType == typeof(IRegularizer); + } + + public override bool CanRead => true; + + public override bool CanWrite => true; + + public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer) + { + var regularizer = value as IRegularizer; + if (regularizer is null) + { + JToken.FromObject(null).WriteTo(writer); + return; + } + JToken.FromObject(new RegularizerInfo() + { + class_name = regularizer.ClassName, + config = JObject.FromObject(regularizer.Config) + }, serializer).WriteTo(writer); + } + + public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer) + { + var info = serializer.Deserialize(reader); + if (info is null) + { + return null; + } + return info.class_name switch + { + "L1L2" => new L1L2 (info.config["l1"].ToObject(), info.config["l2"].ToObject()), + "L1" => new L1(info.config["l1"].ToObject()), + "L2" => new L2(info.config["l2"].ToObject()), + }; + } + } +} diff --git a/src/TensorFlowNET.Core/Operations/Regularizers/L1.cs b/src/TensorFlowNET.Core/Operations/Regularizers/L1.cs new file mode 100644 index 000000000..8a5c68895 --- /dev/null +++ b/src/TensorFlowNET.Core/Operations/Regularizers/L1.cs @@ -0,0 +1,33 @@ +using System; + +using Tensorflow.Keras; + +namespace Tensorflow.Operations.Regularizers +{ + public class L1 : IRegularizer + { + float _l1; + private readonly Dictionary _config; + + public string ClassName => "L2"; + public virtual IDictionary Config => _config; + + public L1(float l1 = 0.01f) + { + // l1 = 0.01 if l1 is None else l1 + // validate_float_arg(l1, name = "l1") + // self.l1 = ops.convert_to_tensor(l1) + this._l1 = l1; + + _config = new(); + _config["l1"] = _l1; + } + + + public Tensor Apply(RegularizerArgs args) + { + //return self.l1 * ops.sum(ops.absolute(x)) + return _l1 * math_ops.reduce_sum(math_ops.abs(args.X)); + } + } +} diff --git a/src/TensorFlowNET.Core/Operations/Regularizers/L1L2.cs b/src/TensorFlowNET.Core/Operations/Regularizers/L1L2.cs new file mode 100644 index 000000000..e3af00eb5 --- /dev/null +++ b/src/TensorFlowNET.Core/Operations/Regularizers/L1L2.cs @@ -0,0 +1,48 @@ +using System; + +using Tensorflow.Keras; + +namespace Tensorflow.Operations.Regularizers +{ + public class L1L2 : IRegularizer + { + float _l1; + float _l2; + private readonly Dictionary _config; + + public string ClassName => "L1L2"; + public virtual IDictionary Config => _config; + + public L1L2(float l1 = 0.0f, float l2 = 0.0f) + { + //l1 = 0.0 if l1 is None else l1 + //l2 = 0.0 if l2 is None else l2 + // validate_float_arg(l1, name = "l1") + // validate_float_arg(l2, name = "l2") + + // self.l1 = l1 + // self.l2 = l2 + this._l1 = l1; + this._l2 = l2; + + _config = new(); + _config["l1"] = l1; + _config["l2"] = l2; + } + + public Tensor Apply(RegularizerArgs args) + { + //regularization = ops.convert_to_tensor(0.0, dtype = x.dtype) + //if self.l1: + // regularization += self.l1 * ops.sum(ops.absolute(x)) + //if self.l2: + // regularization += self.l2 * ops.sum(ops.square(x)) + //return regularization + + Tensor regularization = tf.constant(0.0, args.X.dtype); + regularization += _l1 * math_ops.reduce_sum(math_ops.abs(args.X)); + regularization += _l2 * math_ops.reduce_sum(math_ops.square(args.X)); + return regularization; + } + } +} diff --git a/src/TensorFlowNET.Core/Operations/Regularizers/L2.cs b/src/TensorFlowNET.Core/Operations/Regularizers/L2.cs new file mode 100644 index 000000000..6c0e950a9 --- /dev/null +++ b/src/TensorFlowNET.Core/Operations/Regularizers/L2.cs @@ -0,0 +1,33 @@ +using System; + +using Tensorflow.Keras; + +namespace Tensorflow.Operations.Regularizers +{ + public class L2 : IRegularizer + { + float _l2; + private readonly Dictionary _config; + + public string ClassName => "L2"; + public virtual IDictionary Config => _config; + + public L2(float l2 = 0.01f) + { + // l2 = 0.01 if l2 is None else l2 + // validate_float_arg(l2, name = "l2") + // self.l2 = l2 + this._l2 = l2; + + _config = new(); + _config["l2"] = _l2; + } + + + public Tensor Apply(RegularizerArgs args) + { + //return self.l2 * ops.sum(ops.square(x)) + return _l2 * math_ops.reduce_sum(math_ops.square(args.X)); + } + } +} diff --git a/src/TensorFlowNET.Keras/Regularizers.cs b/src/TensorFlowNET.Keras/Regularizers.cs index 98da27a7f..9c6d07ca6 100644 --- a/src/TensorFlowNET.Keras/Regularizers.cs +++ b/src/TensorFlowNET.Keras/Regularizers.cs @@ -1,8 +1,17 @@ namespace Tensorflow.Keras { - public class Regularizers - { - public IRegularizer l2(float l2 = 0.01f) - => new L2(l2); - } + public class Regularizers + { + public IRegularizer l1(float l1 = 0.01f) + => new Tensorflow.Operations.Regularizers.L1(l1); + public IRegularizer l2(float l2 = 0.01f) + => new Tensorflow.Operations.Regularizers.L2(l2); + + //From TF source + //# The default value for l1 and l2 are different from the value in l1_l2 + //# for backward compatibility reason. Eg, L1L2(l2=0.1) will only have l2 + //# and no l1 penalty. + public IRegularizer l1l2(float l1 = 0.00f, float l2 = 0.00f) + => new Tensorflow.Operations.Regularizers.L1L2(l1, l2); + } } diff --git a/src/TensorFlowNET.Keras/Regularizers/L1.cs b/src/TensorFlowNET.Keras/Regularizers/L1.cs deleted file mode 100644 index 0f904b6f9..000000000 --- a/src/TensorFlowNET.Keras/Regularizers/L1.cs +++ /dev/null @@ -1,19 +0,0 @@ -using System; - -namespace Tensorflow.Keras -{ - public class L1 : IRegularizer - { - float l1; - - public L1(float l1 = 0.01f) - { - this.l1 = l1; - } - - public Tensor Apply(RegularizerArgs args) - { - return l1 * math_ops.reduce_sum(math_ops.abs(args.X)); - } - } -} diff --git a/src/TensorFlowNET.Keras/Regularizers/L1L2.cs b/src/TensorFlowNET.Keras/Regularizers/L1L2.cs deleted file mode 100644 index f619f1582..000000000 --- a/src/TensorFlowNET.Keras/Regularizers/L1L2.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; -using static Tensorflow.Binding; -namespace Tensorflow.Keras -{ - public class L1L2 : IRegularizer - { - float l1; - float l2; - - public L1L2(float l1 = 0.0f, float l2 = 0.0f) - { - this.l1 = l1; - this.l2 = l2; - - } - public Tensor Apply(RegularizerArgs args) - { - Tensor regularization = tf.constant(0.0, args.X.dtype); - regularization += l1 * math_ops.reduce_sum(math_ops.abs(args.X)); - regularization += l2 * math_ops.reduce_sum(math_ops.square(args.X)); - return regularization; - } - } -} diff --git a/src/TensorFlowNET.Keras/Regularizers/L2.cs b/src/TensorFlowNET.Keras/Regularizers/L2.cs deleted file mode 100644 index 034bbd236..000000000 --- a/src/TensorFlowNET.Keras/Regularizers/L2.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace Tensorflow.Keras -{ - public class L2 : IRegularizer - { - float l2; - - public L2(float l2 = 0.01f) - { - this.l2 = l2; - } - - public Tensor Apply(RegularizerArgs args) - { - return l2 * math_ops.reduce_sum(math_ops.square(args.X)); - } - } -} From 5f9fce572d07768de9c1386bf29264a345e16c8c Mon Sep 17 00:00:00 2001 From: Schoen Tannenbaum <169845314+SchoenTannenbaum@users.noreply.github.com> Date: Mon, 20 May 2024 12:10:09 -0400 Subject: [PATCH 89/98] RegularizerAPI and UnitTest --- .../Keras/Regularizers/IRegularizer.cs | 11 ++++- .../Operations/Regularizers/L1.cs | 2 +- src/TensorFlowNET.Keras/Regularizers.cs | 44 +++++++++++++++-- .../Model/ModelLoadTest.cs | 48 +++++++++++++++++++ 4 files changed, 98 insertions(+), 7 deletions(-) diff --git a/src/TensorFlowNET.Core/Keras/Regularizers/IRegularizer.cs b/src/TensorFlowNET.Core/Keras/Regularizers/IRegularizer.cs index e5de76ddb..06dbb7c8c 100644 --- a/src/TensorFlowNET.Core/Keras/Regularizers/IRegularizer.cs +++ b/src/TensorFlowNET.Core/Keras/Regularizers/IRegularizer.cs @@ -12,5 +12,14 @@ public interface IRegularizer [JsonProperty("config")] IDictionary Config { get; } Tensor Apply(RegularizerArgs args); - } + } + + public interface IRegularizerApi + { + IRegularizer GetRegularizerFromName(string name); + IRegularizer L1 { get; } + IRegularizer L2 { get; } + IRegularizer L1L2 { get; } + } + } diff --git a/src/TensorFlowNET.Core/Operations/Regularizers/L1.cs b/src/TensorFlowNET.Core/Operations/Regularizers/L1.cs index 8a5c68895..9e0619454 100644 --- a/src/TensorFlowNET.Core/Operations/Regularizers/L1.cs +++ b/src/TensorFlowNET.Core/Operations/Regularizers/L1.cs @@ -9,7 +9,7 @@ public class L1 : IRegularizer float _l1; private readonly Dictionary _config; - public string ClassName => "L2"; + public string ClassName => "L1"; public virtual IDictionary Config => _config; public L1(float l1 = 0.01f) diff --git a/src/TensorFlowNET.Keras/Regularizers.cs b/src/TensorFlowNET.Keras/Regularizers.cs index 9c6d07ca6..73b72a051 100644 --- a/src/TensorFlowNET.Keras/Regularizers.cs +++ b/src/TensorFlowNET.Keras/Regularizers.cs @@ -1,17 +1,51 @@ -namespace Tensorflow.Keras +using Tensorflow.Operations.Regularizers; + +namespace Tensorflow.Keras { - public class Regularizers + public class Regularizers: IRegularizerApi { + private static Dictionary _nameActivationMap; + public IRegularizer l1(float l1 = 0.01f) - => new Tensorflow.Operations.Regularizers.L1(l1); + => new L1(l1); public IRegularizer l2(float l2 = 0.01f) - => new Tensorflow.Operations.Regularizers.L2(l2); + => new L2(l2); //From TF source //# The default value for l1 and l2 are different from the value in l1_l2 //# for backward compatibility reason. Eg, L1L2(l2=0.1) will only have l2 //# and no l1 penalty. public IRegularizer l1l2(float l1 = 0.00f, float l2 = 0.00f) - => new Tensorflow.Operations.Regularizers.L1L2(l1, l2); + => new L1L2(l1, l2); + + static Regularizers() + { + _nameActivationMap = new Dictionary(); + _nameActivationMap["L1"] = new L1(); + _nameActivationMap["L1"] = new L2(); + _nameActivationMap["L1"] = new L1L2(); + } + + public IRegularizer L1 => l1(); + + public IRegularizer L2 => l2(); + + public IRegularizer L1L2 => l1l2(); + + public IRegularizer GetRegularizerFromName(string name) + { + if (name == null) + { + throw new Exception($"Regularizer name cannot be null"); + } + if (!_nameActivationMap.TryGetValue(name, out var res)) + { + throw new Exception($"Regularizer {name} not found"); + } + else + { + return res; + } + } } } diff --git a/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs b/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs index 53a67cbfa..c733537e7 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs @@ -1,6 +1,7 @@ using Microsoft.VisualStudio.TestPlatform.Utilities; using Microsoft.VisualStudio.TestTools.UnitTesting; using Newtonsoft.Json.Linq; +using System.Collections.Generic; using System.Linq; using System.Xml.Linq; using Tensorflow.Keras.Engine; @@ -129,6 +130,53 @@ public void TestModelBeforeTF2_5() } + [TestMethod] + public void BiasRegularizerSaveAndLoad() + { + var savemodel = keras.Sequential(new List() + { + tf.keras.layers.InputLayer((227, 227, 3)), + tf.keras.layers.Conv2D(96, (11, 11), (4, 4), activation:"relu", padding:"valid"), + tf.keras.layers.BatchNormalization(), + tf.keras.layers.MaxPooling2D((3, 3), strides:(2, 2)), + + tf.keras.layers.Conv2D(256, (5, 5), (1, 1), "same", activation: keras.activations.Relu, bias_regularizer:keras.regularizers.L1L2), + tf.keras.layers.BatchNormalization(), + + tf.keras.layers.Conv2D(256, (5, 5), (1, 1), "same", activation: keras.activations.Relu, bias_regularizer:keras.regularizers.L2), + tf.keras.layers.BatchNormalization(), + + tf.keras.layers.Conv2D(256, (5, 5), (1, 1), "same", activation: keras.activations.Relu, bias_regularizer:keras.regularizers.L1), + tf.keras.layers.BatchNormalization(), + tf.keras.layers.MaxPooling2D((3, 3), (2, 2)), + + tf.keras.layers.Flatten(), + + tf.keras.layers.Dense(1000, activation: "linear"), + tf.keras.layers.Softmax(1) + }); + + savemodel.compile(tf.keras.optimizers.Adam(), tf.keras.losses.SparseCategoricalCrossentropy(from_logits: true), new string[] { "accuracy" }); + + var num_epochs = 1; + var batch_size = 8; + + var trainDataset = new RandomDataSet(new Shape(227, 227, 3), 16); + + savemodel.fit(trainDataset.Data, trainDataset.Labels, batch_size, num_epochs); + + savemodel.save(@"./bias_regularizer_save_and_load", save_format: "tf"); + + var loadModel = tf.keras.models.load_model(@"./bias_regularizer_save_and_load"); + loadModel.summary(); + + loadModel.compile(tf.keras.optimizers.Adam(), tf.keras.losses.SparseCategoricalCrossentropy(from_logits: true), new string[] { "accuracy" }); + + var fitDataset = new RandomDataSet(new Shape(227, 227, 3), 16); + + loadModel.fit(fitDataset.Data, fitDataset.Labels, batch_size, num_epochs); + } + [TestMethod] public void CreateConcatenateModelSaveAndLoad() From b3ce158ec3304469bf776bc582b847e685a9df73 Mon Sep 17 00:00:00 2001 From: novikov-alexander <79649566+novikov-alexander@users.noreply.github.com> Date: Fri, 14 Jun 2024 14:40:06 +0300 Subject: [PATCH 90/98] Update tensor_util.cs --- src/TensorFlowNET.Core/Tensors/tensor_util.cs | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/src/TensorFlowNET.Core/Tensors/tensor_util.cs b/src/TensorFlowNET.Core/Tensors/tensor_util.cs index f688d4d5d..f2003c9d4 100644 --- a/src/TensorFlowNET.Core/Tensors/tensor_util.cs +++ b/src/TensorFlowNET.Core/Tensors/tensor_util.cs @@ -1,4 +1,4 @@ -/***************************************************************************** +/***************************************************************************** Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); @@ -135,6 +135,23 @@ T[] ExpandArrayToSize(IList src) TF_DataType.TF_QINT32 }; + private static TOut[,] ConvertArray2D(TIn[,] inputArray, Func converter) + { + var rows = inputArray.GetLength(0); + var cols = inputArray.GetLength(1); + var outputArray = new TOut[rows, cols]; + + for (var i = 0; i < rows; i++) + { + for (var j = 0; j < cols; j++) + { + outputArray[i, j] = converter(inputArray[i, j]); + } + } + + return outputArray; + } + /// /// Create a TensorProto, invoked in graph mode /// @@ -157,19 +174,16 @@ public static TensorProto make_tensor_proto(object values, TF_DataType dtype = T else if(origin_dtype != dtype) { var new_system_dtype = dtype.as_system_dtype(); - if (values is long[] long_values) - { - if (dtype == TF_DataType.TF_INT32) - values = long_values.Select(x => (int)Convert.ChangeType(x, new_system_dtype)).ToArray(); - } - else if (values is double[] double_values) + + values = values switch { - if (dtype == TF_DataType.TF_FLOAT) - values = double_values.Select(x => (float)Convert.ChangeType(x, new_system_dtype)).ToArray(); - } - else - values = Convert.ChangeType(values, new_system_dtype); - + long[] longValues when dtype == TF_DataType.TF_INT32 => longValues.Select(x => (int)x).ToArray(), + float[] floatValues when dtype == TF_DataType.TF_DOUBLE => floatValues.Select(x => (double)x).ToArray(), + float[,] float2DValues when dtype == TF_DataType.TF_DOUBLE => ConvertArray2D(float2DValues, Convert.ToDouble), + double[] doubleValues when dtype == TF_DataType.TF_FLOAT => doubleValues.Select(x => (float)x).ToArray(), + double[,] double2DValues when dtype == TF_DataType.TF_DOUBLE => ConvertArray2D(double2DValues, Convert.ToSingle), + _ => Convert.ChangeType(values, new_system_dtype), + }; dtype = values.GetDataType(); } From 18db147eb40a07931e8421bbd63c64ce11edd558 Mon Sep 17 00:00:00 2001 From: novikov-alexander <79649566+novikov-alexander@users.noreply.github.com> Date: Fri, 14 Jun 2024 14:40:37 +0300 Subject: [PATCH 91/98] Update GradientDescentOptimizerTests.cs --- .../Training/GradientDescentOptimizerTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs b/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs index f7062f00d..3b53ff9cd 100644 --- a/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs +++ b/test/TensorFlowNET.UnitTest/Training/GradientDescentOptimizerTests.cs @@ -1,4 +1,4 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; +using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.Linq; using Tensorflow; @@ -82,7 +82,7 @@ private void TestMinimizeResourceVariable() where T : struct var pred = math_ops.matmul(var0, x) + var1; var loss = pred * pred; - var sgd_op = tf.train.GradientDescentOptimizer(3.0f).minimize(loss); + var sgd_op = tf.train.GradientDescentOptimizer(1.0f).minimize(loss); var global_variables = tf.global_variables_initializer(); sess.run(global_variables); From 483ac82cd2db273c2c0520ce6923f5951638daba Mon Sep 17 00:00:00 2001 From: novikov-alexander <79649566+novikov-alexander@users.noreply.github.com> Date: Fri, 14 Jun 2024 15:02:17 +0300 Subject: [PATCH 92/98] Update tensor_util.cs --- src/TensorFlowNET.Core/Tensors/tensor_util.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Core/Tensors/tensor_util.cs b/src/TensorFlowNET.Core/Tensors/tensor_util.cs index f2003c9d4..873579e42 100644 --- a/src/TensorFlowNET.Core/Tensors/tensor_util.cs +++ b/src/TensorFlowNET.Core/Tensors/tensor_util.cs @@ -178,10 +178,15 @@ public static TensorProto make_tensor_proto(object values, TF_DataType dtype = T values = values switch { long[] longValues when dtype == TF_DataType.TF_INT32 => longValues.Select(x => (int)x).ToArray(), + long[] longValues => values, float[] floatValues when dtype == TF_DataType.TF_DOUBLE => floatValues.Select(x => (double)x).ToArray(), + float[] floatValues => values, float[,] float2DValues when dtype == TF_DataType.TF_DOUBLE => ConvertArray2D(float2DValues, Convert.ToDouble), + float[,] float2DValues => values, double[] doubleValues when dtype == TF_DataType.TF_FLOAT => doubleValues.Select(x => (float)x).ToArray(), - double[,] double2DValues when dtype == TF_DataType.TF_DOUBLE => ConvertArray2D(double2DValues, Convert.ToSingle), + double[] doubleValues => values, + double[,] double2DValues when dtype == TF_DataType.TF_FLOAT => ConvertArray2D(double2DValues, Convert.ToSingle), + double[,] double2DValues => values, _ => Convert.ChangeType(values, new_system_dtype), }; dtype = values.GetDataType(); From def57745b66d0537cdb70251584c940f327cd929 Mon Sep 17 00:00:00 2001 From: Alexander Novikov Date: Wed, 19 Jun 2024 12:30:38 +0300 Subject: [PATCH 93/98] fix: more generic array cast --- src/TensorFlowNET.Core/Tensors/tensor_util.cs | 88 +++++++++++++------ 1 file changed, 59 insertions(+), 29 deletions(-) diff --git a/src/TensorFlowNET.Core/Tensors/tensor_util.cs b/src/TensorFlowNET.Core/Tensors/tensor_util.cs index 873579e42..6e5024efd 100644 --- a/src/TensorFlowNET.Core/Tensors/tensor_util.cs +++ b/src/TensorFlowNET.Core/Tensors/tensor_util.cs @@ -67,7 +67,7 @@ public static NDArray MakeNdarray(TensorProto tensor) T[] ExpandArrayToSize(IList src) { - if(src.Count == 0) + if (src.Count == 0) { return new T[0]; } @@ -77,7 +77,7 @@ T[] ExpandArrayToSize(IList src) var first_elem = src[0]; var last_elem = src[src.Count - 1]; T[] res = new T[num_elements]; - for(long i = 0; i < num_elements; i++) + for (long i = 0; i < num_elements; i++) { if (i < pre) res[i] = first_elem; else if (i >= num_elements - after) res[i] = last_elem; @@ -121,7 +121,7 @@ T[] ExpandArrayToSize(IList src) $"https://www.tensorflow.org/api_docs/python/tf/dtypes for supported TF dtypes."); } - if(values.size == 0) + if (values.size == 0) { return np.zeros(shape, tensor_dtype); } @@ -135,23 +135,47 @@ T[] ExpandArrayToSize(IList src) TF_DataType.TF_QINT32 }; - private static TOut[,] ConvertArray2D(TIn[,] inputArray, Func converter) + private static Array ConvertArray(Array inputArray, Func converter) { - var rows = inputArray.GetLength(0); - var cols = inputArray.GetLength(1); - var outputArray = new TOut[rows, cols]; + if (inputArray == null) + throw new ArgumentNullException(nameof(inputArray)); - for (var i = 0; i < rows; i++) + var elementType = typeof(TOut); + var lengths = new int[inputArray.Rank]; + for (var i = 0; i < inputArray.Rank; i++) { - for (var j = 0; j < cols; j++) - { - outputArray[i, j] = converter(inputArray[i, j]); - } + lengths[i] = inputArray.GetLength(i); } + var outputArray = Array.CreateInstance(elementType, lengths); + + FillArray(inputArray, outputArray, converter, new int[inputArray.Rank], 0); + return outputArray; } + private static void FillArray(Array inputArray, Array outputArray, Func converter, int[] indices, int dimension) + { + if (dimension == inputArray.Rank - 1) + { + for (int i = 0; i < inputArray.GetLength(dimension); i++) + { + indices[dimension] = i; + var inputValue = (TIn)inputArray.GetValue(indices); + var convertedValue = converter(inputValue); + outputArray.SetValue(convertedValue, indices); + } + } + else + { + for (int i = 0; i < inputArray.GetLength(dimension); i++) + { + indices[dimension] = i; + FillArray(inputArray, outputArray, converter, indices, dimension + 1); + } + } + } + /// /// Create a TensorProto, invoked in graph mode /// @@ -171,24 +195,30 @@ public static TensorProto make_tensor_proto(object values, TF_DataType dtype = T var origin_dtype = values.GetDataType(); if (dtype == TF_DataType.DtInvalid) dtype = origin_dtype; - else if(origin_dtype != dtype) + else if (origin_dtype != dtype) { var new_system_dtype = dtype.as_system_dtype(); - - values = values switch + + if (dtype != TF_DataType.TF_STRING && dtype != TF_DataType.TF_VARIANT && dtype != TF_DataType.TF_RESOURCE) + { + if (values is Array arrayValues) + { + values = dtype switch + { + TF_DataType.TF_INT32 => ConvertArray(arrayValues, Convert.ToInt32), + TF_DataType.TF_FLOAT => ConvertArray(arrayValues, Convert.ToSingle), + TF_DataType.TF_DOUBLE => ConvertArray(arrayValues, Convert.ToDouble), + _ => values, + }; + } else + { + values = Convert.ChangeType(values, new_system_dtype); + } + + } else { - long[] longValues when dtype == TF_DataType.TF_INT32 => longValues.Select(x => (int)x).ToArray(), - long[] longValues => values, - float[] floatValues when dtype == TF_DataType.TF_DOUBLE => floatValues.Select(x => (double)x).ToArray(), - float[] floatValues => values, - float[,] float2DValues when dtype == TF_DataType.TF_DOUBLE => ConvertArray2D(float2DValues, Convert.ToDouble), - float[,] float2DValues => values, - double[] doubleValues when dtype == TF_DataType.TF_FLOAT => doubleValues.Select(x => (float)x).ToArray(), - double[] doubleValues => values, - double[,] double2DValues when dtype == TF_DataType.TF_FLOAT => ConvertArray2D(double2DValues, Convert.ToSingle), - double[,] double2DValues => values, - _ => Convert.ChangeType(values, new_system_dtype), - }; + + } dtype = values.GetDataType(); } @@ -306,7 +336,7 @@ bool hasattr(Graph property, string attr) if (tensor is EagerTensor eagerTensor) { - if(tensor.dtype == tf.int64) + if (tensor.dtype == tf.int64) return new Shape(tensor.ToArray()); else return new Shape(tensor.ToArray()); @@ -481,7 +511,7 @@ bool hasattr(Graph property, string attr) var d_ = new int[value.size]; foreach (var (index, d) in enumerate(value.ToArray())) d_[index] = d >= 0 ? d : -1; - + ret = ret.merge_with(new Shape(d_)); } return ret; From 5142ad658cf9233abd2c9fe727c2daeea84a88f6 Mon Sep 17 00:00:00 2001 From: Aleksej Solomatin Date: Sun, 30 Jun 2024 22:06:12 +0300 Subject: [PATCH 94/98] test: Added an `evaluate` method call to a unit test for a multi-input model. --- test/TensorFlowNET.Keras.UnitTest/MultiInputModelTest.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/TensorFlowNET.Keras.UnitTest/MultiInputModelTest.cs b/test/TensorFlowNET.Keras.UnitTest/MultiInputModelTest.cs index dd8ef8f91..bb293bd90 100644 --- a/test/TensorFlowNET.Keras.UnitTest/MultiInputModelTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/MultiInputModelTest.cs @@ -54,6 +54,13 @@ public void LeNetModel() var x = new NDArray[] { x1, x2 }; model.fit(x, dataset.Train.Labels, batch_size: 8, epochs: 3); + x1 = x1["0:8"]; + x2 = x1; + + x = new NDArray[] { x1, x2 }; + var y = dataset.Train.Labels["0:8"]; + (model as Engine.Model).evaluate(x, y); + x1 = np.ones((1, 28, 28, 1), TF_DataType.TF_FLOAT); x2 = np.zeros((1, 28, 28, 1), TF_DataType.TF_FLOAT); var pred = model.predict((x1, x2)); From f8b7bdeb9b7fa10bf49b888934683f04febfc6e2 Mon Sep 17 00:00:00 2001 From: Aleksej Solomatin Date: Sun, 30 Jun 2024 22:43:01 +0300 Subject: [PATCH 95/98] test: Added a unit test of training a multi-input model using a dataset. --- .../MultiInputModelTest.cs | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/test/TensorFlowNET.Keras.UnitTest/MultiInputModelTest.cs b/test/TensorFlowNET.Keras.UnitTest/MultiInputModelTest.cs index bb293bd90..54b76d41a 100644 --- a/test/TensorFlowNET.Keras.UnitTest/MultiInputModelTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/MultiInputModelTest.cs @@ -2,6 +2,7 @@ using System; using Tensorflow.Keras.Optimizers; using Tensorflow.NumPy; +using static Tensorflow.Binding; using static Tensorflow.KerasApi; namespace Tensorflow.Keras.UnitTest @@ -66,5 +67,79 @@ public void LeNetModel() var pred = model.predict((x1, x2)); Console.WriteLine(pred); } + + [TestMethod] + public void LeNetModelDataset() + { + var inputs = keras.Input((28, 28, 1)); + var conv1 = keras.layers.Conv2D(16, (3, 3), activation: "relu", padding: "same").Apply(inputs); + var pool1 = keras.layers.MaxPooling2D((2, 2), 2).Apply(conv1); + var conv2 = keras.layers.Conv2D(32, (3, 3), activation: "relu", padding: "same").Apply(pool1); + var pool2 = keras.layers.MaxPooling2D((2, 2), 2).Apply(conv2); + var flat1 = keras.layers.Flatten().Apply(pool2); + + var inputs_2 = keras.Input((28, 28, 1)); + var conv1_2 = keras.layers.Conv2D(16, (3, 3), activation: "relu", padding: "same").Apply(inputs_2); + var pool1_2 = keras.layers.MaxPooling2D((4, 4), 4).Apply(conv1_2); + var conv2_2 = keras.layers.Conv2D(32, (1, 1), activation: "relu", padding: "same").Apply(pool1_2); + var pool2_2 = keras.layers.MaxPooling2D((2, 2), 2).Apply(conv2_2); + var flat1_2 = keras.layers.Flatten().Apply(pool2_2); + + var concat = keras.layers.Concatenate().Apply((flat1, flat1_2)); + var dense1 = keras.layers.Dense(512, activation: "relu").Apply(concat); + var dense2 = keras.layers.Dense(128, activation: "relu").Apply(dense1); + var dense3 = keras.layers.Dense(10, activation: "relu").Apply(dense2); + var output = keras.layers.Softmax(-1).Apply(dense3); + + var model = keras.Model((inputs, inputs_2), output); + model.summary(); + + var data_loader = new MnistModelLoader(); + + var dataset = data_loader.LoadAsync(new ModelLoadSetting + { + TrainDir = "mnist", + OneHot = false, + ValidationSize = 59900, + }).Result; + + var loss = keras.losses.SparseCategoricalCrossentropy(); + var optimizer = new Adam(0.001f); + model.compile(optimizer, loss, new string[] { "accuracy" }); + + NDArray x1 = np.reshape(dataset.Train.Data, (dataset.Train.Data.shape[0], 28, 28, 1)); + + var multiInputDataset = tf.data.Dataset.zip( + tf.data.Dataset.from_tensor_slices(x1), + tf.data.Dataset.from_tensor_slices(x1), + tf.data.Dataset.from_tensor_slices(dataset.Train.Labels) + ).batch(8); + multiInputDataset.FirstInputTensorCount = 2; + + model.fit(multiInputDataset, epochs: 3); + + x1 = x1["0:8"]; + + multiInputDataset = tf.data.Dataset.zip( + tf.data.Dataset.from_tensor_slices(x1), + tf.data.Dataset.from_tensor_slices(x1), + tf.data.Dataset.from_tensor_slices(dataset.Train.Labels["0:8"]) + ).batch(8); + multiInputDataset.FirstInputTensorCount = 2; + + (model as Engine.Model).evaluate(multiInputDataset); + + x1 = np.ones((1, 28, 28, 1), TF_DataType.TF_FLOAT); + var x2 = np.zeros((1, 28, 28, 1), TF_DataType.TF_FLOAT); + + multiInputDataset = tf.data.Dataset.zip( + tf.data.Dataset.from_tensor_slices(x1), + tf.data.Dataset.from_tensor_slices(x2) + ).batch(8); + multiInputDataset.FirstInputTensorCount = 2; + + var pred = model.predict(multiInputDataset); + Console.WriteLine(pred); + } } } From 93dda17944b6e34380897ad3480ac2218fb7398e Mon Sep 17 00:00:00 2001 From: Aleksej Solomatin Date: Sun, 30 Jun 2024 22:44:03 +0300 Subject: [PATCH 96/98] fix: Added support for training a multi-input model using a dataset. --- src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs | 14 +++++++++++++- src/TensorFlowNET.Keras/Engine/Model.Fit.cs | 13 ++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs index b3264429e..ec99d7ef9 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs @@ -112,7 +112,19 @@ public Dictionary evaluate(IDatasetV2 x, int verbose = 1, bool is Steps = data_handler.Inferredsteps }); - return evaluate(data_handler, callbacks, is_val, test_function); + Func> testFunction; + + if (data_handler.DataAdapter.GetDataset().structure.Length > 2 || + data_handler.DataAdapter.GetDataset().FirstInputTensorCount > 1) + { + testFunction = test_step_multi_inputs_function; + } + else + { + testFunction = test_function; + } + + return evaluate(data_handler, callbacks, is_val, testFunction); } /// diff --git a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs index 13a1b63bc..e1303513e 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs @@ -179,9 +179,20 @@ public ICallback fit(IDatasetV2 dataset, StepsPerExecution = _steps_per_execution }); + Func> trainStepFunction; + + if (data_handler.DataAdapter.GetDataset().structure.Length > 2 || + data_handler.DataAdapter.GetDataset().FirstInputTensorCount > 1) + { + trainStepFunction = train_step_multi_inputs_function; + } + else + { + trainStepFunction = train_step_function; + } return FitInternal(data_handler, epochs, validation_step, verbose, callbacks, validation_data: validation_data, - train_step_func: train_step_function); + train_step_func: trainStepFunction); } History FitInternal(DataHandler data_handler, int epochs, int validation_step, int verbose, List callbackList, IDatasetV2 validation_data, From b6c5d26fab9a5eab72c0c81c554fec8412d86771 Mon Sep 17 00:00:00 2001 From: Leonardo Doherty <73901464+eLDoherty@users.noreply.github.com> Date: Mon, 13 Jan 2025 23:29:04 -0500 Subject: [PATCH 97/98] fix: Resolve fixed-size array issue Replace .ToArray() with .ToList() to allow dynamic modification of network_nodes in MapGraphNetwork() Replaced .ToArray() with .ToList() to resolve the issue where .Add() was called on a fixed-size array. This preventing the "Collection was of a fixed size" error when called something like this var model = keras.Model(new Tensors(new Tensor[] { encoder_inputs, decoder_inputs }), outputs: decoder_dense); --- src/TensorFlowNET.Keras/Engine/Functional.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/TensorFlowNET.Keras/Engine/Functional.cs b/src/TensorFlowNET.Keras/Engine/Functional.cs index 7347585f8..75854d82c 100644 --- a/src/TensorFlowNET.Keras/Engine/Functional.cs +++ b/src/TensorFlowNET.Keras/Engine/Functional.cs @@ -180,7 +180,7 @@ void ComputeTensorUsageCount() var (nodes_in_decreasing_depth, layer_indices) = BuildMap(outputs); var network_nodes = nodes_in_decreasing_depth .Select(node => MakeNodeKey(node.Layer.Name, node.Layer.InboundNodes.IndexOf(node))) - .ToArray(); + .ToList(); var nodes_depths = new Dictionary(); var layers_depths = new Dictionary(); @@ -221,7 +221,7 @@ void ComputeTensorUsageCount() layers_depths[input_layer] = 0; layer_indices[input_layer] = -1; nodes_depths[input_layer.InboundNodes[0]] = 0; - network_nodes.add(MakeNodeKey(input_layer.Name, 0)); + network_nodes.Add(MakeNodeKey(input_layer.Name, 0)); } } @@ -231,7 +231,7 @@ void ComputeTensorUsageCount() { if (!nodes_by_depth.ContainsKey(depth)) nodes_by_depth[depth] = new List(); - nodes_by_depth[depth].append(node); + nodes_by_depth[depth].Add(node); } var layers_by_depth = new Dictionary>(); @@ -239,7 +239,7 @@ void ComputeTensorUsageCount() { if (!layers_by_depth.ContainsKey(depth)) layers_by_depth[depth] = new List(); - layers_by_depth[depth].append(layer); + layers_by_depth[depth].Add(layer); } // Get sorted list of layer depths. @@ -260,7 +260,7 @@ void ComputeTensorUsageCount() // Get sorted list of node depths. depth_keys = nodes_by_depth.Keys.OrderBy(x => x).Reverse(); - return (network_nodes, nodes_by_depth, layers, layers_by_depth); + return (network_nodes.ToArray(), nodes_by_depth, layers, layers_by_depth); } string MakeNodeKey(string layer_name, int node_index) From 6ce6066551ce80202119a121a05b006aadd9ef37 Mon Sep 17 00:00:00 2001 From: Haiping Date: Wed, 22 Jan 2025 09:46:45 -0600 Subject: [PATCH 98/98] Update release.yml --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8f862e329..02601764c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,7 +53,7 @@ jobs: } - name: Upload packages artifacts - uses: actions/upload-artifact@v1.0.0 + uses: actions/upload-artifact@v4.0.0 with: name: "drop-ci-packages" path: './packages'