From eb67aae241f373caf69a7f1ad1208ee877831552 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Fri, 8 Dec 2017 15:14:04 -0500 Subject: [PATCH 001/272] Clip to good statistical tange after normalization --- plasma/preprocessor/normalize.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plasma/preprocessor/normalize.py b/plasma/preprocessor/normalize.py index 06e9ed3d..951513ab 100644 --- a/plasma/preprocessor/normalize.py +++ b/plasma/preprocessor/normalize.py @@ -43,6 +43,7 @@ def __init__(self,conf): self.remapper = conf['data']['target'].remapper self.machines = set() self.inference_mode = False + self.bound = self.conf['data']['norm_stat_range'] @abc.abstractmethod def __str__(self): @@ -192,6 +193,7 @@ def __init__(self,conf): Normalizer.__init__(self,conf) self.means = dict() self.stds = dict() + self.bound = self.conf['data']['norm_stat_range'] def __str__(self): s = '' @@ -246,6 +248,7 @@ def apply(self,shot): if stds_curr == 0.0: stds_curr = 1.0 shot.signals_dict[sig] = (shot.signals_dict[sig] - means[i])/stds_curr + shot.signals_dict[sig] = np.clip(shot.signals_dict[sig],-self.bound,self.bound) shot.ttd = self.remapper(shot.ttd,self.conf['data']['T_warning']) self.cut_end_of_shot(shot) @@ -288,6 +291,7 @@ def apply(self,shot): if stds_curr == 0.0: stds_curr = 1.0 shot.signals_dict[sig] = (shot.signals_dict[sig])/stds_curr + shot.signals_dict[sig] = np.clip(shot.signals_dict[sig],-self.bound,self.bound) shot.ttd = self.remapper(shot.ttd,self.conf['data']['T_warning']) self.cut_end_of_shot(shot) @@ -312,6 +316,7 @@ def apply(self,shot): for (i,sig) in enumerate(shot.signals): if sig.normalize: shot.signals_dict[sig] = apply_along_axis(lambda m : correlate(m,window,'valid'),axis=0,arr=shot.signals_dict[sig]) + shot.signals_dict[sig] = np.clip(shot.signals_dict[sig],-self.bound,self.bound) shot.ttd = shot.ttd[-shot.signals.shape[0]:] def __str__(self): @@ -330,6 +335,7 @@ def __init__(self,conf): Normalizer.__init__(self,conf) self.minimums = None self.maximums = None + self.bound = self.conf['data']['norm_stat_range'] def __str__(self): @@ -379,6 +385,7 @@ def apply(self,shot): for (i,sig) in enumerate(shot.signals): if sig.normalize: shot.signals_dict[sig] = (shot.signals_dict[sig] - self.minimums[m])/(self.maximums[m] - self.minimums[m]) + shot.signals_dict[sig] = np.clip(shot.signals_dict[sig],-self.bound,self.bound) shot.ttd = self.remapper(shot.ttd,self.conf['data']['T_warning']) self.cut_end_of_shot(shot) # self.apply_positivity_mask(shot) From 189a21cd2a91335aad147341e91b0bed55505992 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Fri, 8 Dec 2017 15:14:38 -0500 Subject: [PATCH 002/272] Introduce a tunable parameter to control statistical range after normalization from conf --- examples/conf.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/conf.yaml b/examples/conf.yaml index 358a5adf..d914b636 100644 --- a/examples/conf.yaml +++ b/examples/conf.yaml @@ -46,6 +46,7 @@ data: window_size: 10 #TODO optimize normalizer: 'var' + norm_stat_range: 100.0 equalize_classes: False # shallow_sample_prob: 0.01 #the fraction of samples with which to train the shallow model floatx: 'float32' From bcf0a1182b77fd4efab9fad817003ebe49a91021 Mon Sep 17 00:00:00 2001 From: Alexey Svyatkovskiy Date: Sun, 17 Dec 2017 19:57:42 -0500 Subject: [PATCH 003/272] Changes to Jenkins as we move to pull request based trigger --- examples/jenkins.sh | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/examples/jenkins.sh b/examples/jenkins.sh index 9d9087e6..c9ac520d 100644 --- a/examples/jenkins.sh +++ b/examples/jenkins.sh @@ -4,13 +4,26 @@ rm /tigress/alexeys/model_checkpoints/* ls ${PWD} -module load anaconda +echo "Jenkins test Python3.6" +export PYTHONHASHSEED=0 +module load anaconda3 +source activate PPPL_dev3 module load cudatoolkit/8.0 -module load openmpi/intel-17.0/2.1.0/64 intel/17.0/64/17.0.4.196 intel-mkl/2017.3/4/64 module load cudnn/cuda-8.0/6.0 -source activate PPPL - +module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 +module load intel/17.0/64/17.0.4.196 export OMPI_MCA_btl="tcp,self,sm" echo $SLURM_NODELIST srun python mpi_learn.py + +echo "Jenkins test Python2.7" +module purge +module load anaconda +source activate PPPL +module load cudatoolkit/8.0 +module load cudnn/cuda-8.0/6.0 +module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 +module load intel/17.0/64/17.0.4.196 + +srun python mpi_learn.py From ff8227e2f307e77fa744af77a534ec3aa0d74db7 Mon Sep 17 00:00:00 2001 From: Alexey Svyatkovskiy Date: Mon, 8 Jan 2018 14:49:14 -0500 Subject: [PATCH 004/272] Update PrincetonUTutorial.md --- docs/PrincetonUTutorial.md | 76 +++++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 18 deletions(-) diff --git a/docs/PrincetonUTutorial.md b/docs/PrincetonUTutorial.md index 04323b07..61f45d52 100644 --- a/docs/PrincetonUTutorial.md +++ b/docs/PrincetonUTutorial.md @@ -1,30 +1,57 @@ ## Tutorials +### Login to Tigergpu + +First, login to TigerGPU cluster headnode via ssh: +``` +ssh -XC @tigergpu.princeton.edu +``` + ### Sample usage on Tigergpu -First, create an isolated Anaconda environment and load CUDA drivers: +Next, check out the source code from github: ``` -module load anaconda3 -module load cudatoolkit/8.0 cudnn/cuda-8.0/6.0 openmpi/cuda-8.0/intel-17.0/2.1.0/64 intel/17.0/64/17.0.2.174 -module load intel/17.0/64/17.0.4.196 intel-mkl/2017.3/4/64 +git clone https://github.com/PPPLDeepLearning/plasma-python +cd plasma-python +``` + +After that, create an isolated Anaconda environment and load CUDA drivers: +``` +#cd plasma-python +module load anaconda3/4.4.0 conda create --name my_env --file requirements-travis.txt source activate my_env + +export OMPI_MCA_btl="tcp,self,sm" +module load cudatoolkit/8.0 +module load cudnn/cuda-8.0/6.0 +module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 +module load intel/17.0/64/17.0.4.196 ``` -Then install the plasma-python package: +and install the `plasma-python` package: ```bash #source activate my_env -git clone https://github.com/PPPLDeepLearning/plasma-python -cd plasma-python python setup.py install ``` Where `my_env` should contain the Python packages as per `requirements-travis.txt` file. +#### Common issue + +Common issue is Intel compiler mismatch in the `PATH` and what you use in the module. With the modules loaded as above, +you should see something like this: +``` +$ which mpicc +/usr/local/openmpi/cuda-8.0/2.1.0/intel170/x86_64/bin/mpicc +``` + +If you source activate the Anaconda environment after loading the openmpi, you would pick the MPI from Anaconda, which is not good and could lead to errors. + #### Location of the data on Tigress -The JET and D3D datasets containing multi-modal time series of sensory measurements leading up to deleterious events called plasma disruptions are located on /tigress filesystem on Princeton U clusters. +The JET and D3D datasets containing multi-modal time series of sensory measurements leading up to deleterious events called plasma disruptions are located on `/tigress/FRNN` filesystem on Princeton U clusters. Fo convenience, create following symbolic links: ```bash @@ -39,16 +66,22 @@ ln -s /tigress/FRNN/signal_data signal_data cd examples/ python guarantee_preprocessed.py ``` -This will preprocess the data and save it in `/tigress//processed_shots` and `/tigress//normalization` +This will preprocess the data and save it in `/tigress//processed_shots`, `/tigress//processed_shotlists` and `/tigress//normalization` +You would only have to run preprocessing once for each dataset. The dataset is specified in the config file `examples/conf.yaml`: +```yaml +paths: + data: jet_data_0D +``` +It take takes about 20 minutes to preprocess in parallel and can normally be done on the cluster headnode. #### Training and inference -Use Slurm scheduler to perform batch or interactive analysis on Tiger cluster. +Use Slurm scheduler to perform batch or interactive analysis on TigerGPU cluster. ##### Batch analysis -For batch analysis, make sure to allocate 1 process per GPU: +For batch analysis, make sure to allocate 1 MPI process per GPU. Save the following to slurm.cmd file (or make changes to the existing `examples/slurm.cmd`): ```bash #!/bin/bash @@ -58,15 +91,20 @@ For batch analysis, make sure to allocate 1 process per GPU: #SBATCH --ntasks-per-socket=2 #SBATCH --gres=gpu:4 #SBATCH -c 4 +#SBATCH --mem-per-cpu=0 -module load anaconda3 +module load anaconda3/4.4.0 source activate my_env -module load cudatoolkit/8.0 cudnn/cuda-8.0/6.0 openmpi/cuda-8.0/intel-17.0/2.1.0/64 intel/17.0/64/17.0.2.174 -module load intel/17.0/64/17.0.4.196 intel-mkl/2017.3/4/64 +export OMPI_MCA_btl="tcp,self,sm" +module load cudatoolkit/8.0 +module load cudnn/cuda-8.0/6.0 +module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 +module load intel/17.0/64/17.0.4.196 + srun python mpi_learn.py ``` -where X is the number of nodes for distibuted training. +where `X` is the number of nodes for distibuted training. Submit the job with: ```bash @@ -82,11 +120,11 @@ Optionally, add an email notification option in the Slurm about the job completi ##### Interactive analysis -Interactive option is preferred for debugging or running in the notebook, for all other case batch is preferred. +Interactive option is preferred for **debugging** or running in the **notebook**, for all other case batch is preferred. The workflow is to request an interactive session: ```bash -salloc -N [X] --ntasks-per-node=4 --ntasks-per-socket=2 --gres=gpu:4 -t 0-6:00 +salloc -N [X] --ntasks-per-node=4 --ntasks-per-socket=2 --gres=gpu:4 -c 4 --mem-per-cpu=0 -t 0-6:00 ``` where the number of GPUs is X * 4. @@ -104,7 +142,7 @@ Currently, FRNN is capable of working with JET and D3D data as well as cross-mac ```yaml paths: ... - data: 'jet_data' + data: 'jet_data_0D' ``` use `d3d_data` for D3D signals, use `jet_to_d3d_data` ir `d3d_to_jet_data` for cross-machine regime. @@ -116,6 +154,8 @@ paths: ``` if left empty `[]` will use all valid signals defined on a machine. Only use if need a custom set. +Other parameters configured in the conf.yaml include batch size, learning rate, neural network topology and special conditions foir hyperparameter sweeps. + ### Current signals and notations Signal name | Description From e839dadd201fc98b3759001d88a46e335e42d288 Mon Sep 17 00:00:00 2001 From: Alexey Svyatkovskiy Date: Wed, 10 Jan 2018 00:08:54 -0500 Subject: [PATCH 005/272] Add Jenkins build status badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d2e34803..9a96c8f5 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# FRNN [![Build Status](https://travis-ci.org/PPPLDeepLearning/plasma-python.svg?branch=master)](https://travis-ci.org/PPPLDeepLearning/plasma-python.svg?branch=master) +# FRNN [![Build Status](https://travis-ci.org/PPPLDeepLearning/plasma-python.svg?branch=master)](https://travis-ci.org/PPPLDeepLearning/plasma-python.svg?branch=master) [![Build Status](https://jenkins.princeton.edu/buildStatus/icon?job=FRNM/PPPL)](https://jenkins.princeton.edu/job/FRNM/job/PPPL/) ## Package description From 6bddbf3f10850fce8be03d3607341c8c0d7edba9 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Wed, 10 Jan 2018 20:55:33 -0500 Subject: [PATCH 006/272] notebook updated --- Analyze Hyperparameter Tuning.ipynb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Analyze Hyperparameter Tuning.ipynb b/Analyze Hyperparameter Tuning.ipynb index b42cc3fc..d4c1de3c 100644 --- a/Analyze Hyperparameter Tuning.ipynb +++ b/Analyze Hyperparameter Tuning.ipynb @@ -124,16 +124,16 @@ }, { "cell_type": "code", - "execution_count": 626, + "execution_count": 631, "metadata": { "collapsed": false }, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAisAAAF7CAYAAAATyMtCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3X94XFd95/HPkU3ihI0ly5ASlm4tyVkSt7uxJZFueApt\nbEvhR0gX27GjwCZlG8cKhe26jS0ZaNNSii3ZQHeftv6Zp4XSKLZlh222C7YlN0C3C7ElOQ+UhCbS\nGEri/sC25BDi/JC++8e5I41Go9GM5se9M3q/nuc+I53763uPRpqvzj3nXGdmAgAAiKqKsAMAAABI\nh2QFAABEGskKAACINJIVAAAQaSQrAAAg0khWAABApJGsAACASJsfdgBR5JxbLOk2SWclXQ43GgAA\nSsoCSUskHTOz8/k4IMlKardJ+suwgwAAoIR9SNIj+TgQyUpqZyXpy1/+sm688caQQykNmzdv1he+\n8IWwwygZ1Fd2qK/sUF/Zo86yk66+nn76aX34wx+Wgs/SfCBZSe2yJN14442qr68PO5aSUFlZSV1l\ngfrKDvWVHeore9RZdjKsr7x1o6CDLQAAiDSSFQAAEGkkKwAAINJIVpAXLS0tYYdQUqiv7FBf2aG+\nskedZafY9eXMrKgnLAXOuXpJfX19fXS4AgAgC/39/WpoaJCkBjPrz8cxaVkBAACRRrICAAAijWQF\nAABEGskKAACINJIVAAAQaSQrAAAg0khWAABApJGsAACASCNZAQAAkUayAgAAIo1kBQAARBrJCgAA\niDSSFQAAEGkkKwAAINJIVgAAQKSRrAAAgEgjWQEAAJFGsgIAACKNZAUAAEQayQoAAIg0khUAABBp\nJCsAACDSSFYAAECkkawAAIBII1kBAACRRrICAAAijWQFAABEGskKAACINJIVAAAQaSQrAAAg0khW\nAABApJGsAACASCNZAQAAkUayAgAAIo1kBQAARBrJCgAAiDSSFQAAEGkkKwAAINJIVgAAQKSRrAAA\ngEgjWQEAAJFGsgIAACKNZAUAAEQayQoAAIg0khUAABBpJCsAACDSSFYAAECkkawAAIBII1kBAACR\nRrICAAAijWQFAABEGskKAACItLJOVpxzNc654865Mefcc865tWHHBAAAslPWyYqkvZK2SKqXNCjp\nkHNuebghAQCAbMwPO4BCCVpR7jezs0HRbc65MUmrJZ0JLTAAAJCVyLasOOeqnHM7nHNLZrO/mR1J\nSFTihiX15RgaAAAooki2rDjntkjaJqlS0qPTbFMpqUPSIkku2LbdzAam2b5W0nkz+5uCBA0AAAoi\nci0rQZ+SvZJOz7Bpv6QqM9tgZusldUrqTdMnZaukpvxFCgAAiiFyyYqZnTGzS5KGptvGOdchaYmk\njQn79Qb7HE6x/UZJx1PcFgIAABEXydtAGdgoacjMXkwqPyhph3NuuZmdkSTn3CpJg2Z2sthBAgAK\nZ/du6YknJr53burXqcpmWl+sfaISR7p9rr1W+q3fUuhKLllxzq2QVCXpRIrV/fL9VzZJesA5t1rS\nCkndzrmaYN1aSXuD1hsAQIkxk373d6XPfEZ617ukBQt8WeL6xNfZlOXrOMUoK+Sx6+pIVmarMXi9\nkGJdvKw2aFE5Lskk7UjYptvMdhYwPgBAgZhJ7e1SZ6e0c6f04INhR4RiKMVkpSp4HU6xLl5WG/Rh\niVyfHADA7Jj5//L/6I+k//E/pP/238KOCMVSislKqiSlIJ5++ulp1y1YsEDLli1Lu//3vvc9Xb58\nedr11113na677rpp17/88stpY5CkG2+8UVddddW068+dO6dz585Nu57rmMB1TOA6PK5jQtjXMTYm\nfe5zC/Too8v0p38qPfBA6mNE/Tqk8vh5SJldR96YWSQXSXskjUpanlS+StKYpN0p9lkRrDuV47nr\n5W8fTbssW7bMZrJs2bK0x3jooYfS7v/d73437f6S7Lvf/W7aYzz00ENcB9fBdXAdZXEd0jI7cKD0\nr6Ncfh7Lli2zRx55xD7wgQ9MWt797nfHt6m3POUEzhJ71ESIc26P/KifBgtG9gTlNfLP+TlsZhuS\n9lkl3/F2r5lNk3dndO56SX1f/vKXdeONN6bcZi5lxlyHx3VM4DomcB1eoa5jdFT69Kelv/5r6Q//\ncIG2bSvN60hUyj+PRNNdR39/vxoaGiT/+d2fNogMlVyyEqy7ID8b7fVJ5VvkO9Outhxmqo0nK319\nfaqvr5/tYQAAOXj9denee6WDB6UvfUm6++6wI0ImCpGslGoH1H3yI34WJpU3y8+pMutEBQAQvtde\n88nJoUNSVxeJylwX5WRl8XQrzKxdfrba/fGyYE6VlZLuLHxoAIBCefVVaf166StfkQ4flu7kr/qc\nF7nRQMGkbxskrQmKOpxzh83sQOJ2Zna9c263c+6gpIuSauQ78zxV3IgBAPly+bK0bp104oT02GPS\n+98fdkSIgsglK+afmjwgqT2DbWfdiRYAEC0vvyx98IPS178u/dVfSbfdFnZEiIrIJSsAgLnnpZek\nO+6QvvUtP/Jn5cqwI0KUkKwAAEL14ovS7bdLfX3SV78qvfvdYUeEqCFZAQCE5tIl6b3vlb7zHen4\ncemd7ww7IkQRyQoAIBTDw75fyve/L/X0SDffHHZEiCqSlTQ2b96syspKtbS0qKWlJexwAKBsXLgg\nNTVJZ89KJ09KzL9Z+rq6utTV1aWRkZG8HzuyM9iGiRlsAaBw/vVffaLy/PO+ReWmm8KOCPlUiBls\naVkBABTNP/+ztGqV9OMfS088If38z4cdEUoByQoAoCheeMEnKiMjPlG54YawI0KpIFkBABTcP/6j\nnzvl8mU/6dv118+8DxBHsgIAKKizZ32iMjYmfeMbUk1N2BGh1ET5QYYAgBI3OCj98i9LzvkWFRIV\nzAbJCgCgIP7hH3yicuWVPlH5uZ8LOyKUKpIVAEDePf20T1QWLvSJytveFnZEKGUkKwCAvPrOd3yi\n8uY3+1E/110XdkQodSQrAIC8OXNGuvVW6d/+Wz8z7bXXhh0RygHJCgAgL06f9qN+amqk3l7pTW8K\nOyKUC4Yup8GzgQAgM9/6ln8o4bJl0te+JlVWhh0Rio1nAxUZzwYCgMz97d9K73uff8bPX/+171SL\nuasQzwbiNhAAYNaeeEJ6z3ukhgbpq18lUUFhkKwAAGalp8e3qNxyi29R+Tf/JuyIUK5IVgAAWfva\n16Tbb5d+5Vekxx+Xrr467IhQzkhWAABZefxx6Vd/VWpulh57TFqwIOyIUO5IVgAAGTt6VFqzxreq\ndHf7qfSBQiNZAQBk5OBBaf16ae1a6dFHpSuuCDsizBUkKwCAGX35y9Ldd0stLf7rN7wh7Igwl5Cs\nAADS+vM/l+65R7r3Xv/1fKYTRZGRrAAAprVvn/SRj0gbN0oHDkjz5oUdEeYikhUAQEp//MfSpk3S\nxz4m7dkjVfCJgZDw1gMATPGFL0gf/7i0ebP0P/+n5FzYEWEuI1kBAEzS0SH91m9JbW3S5z5HooLw\n0U0qDZ66DGAuGB2VfvQjaXDQT5v/+c9Lv/M70u//PokKMsdTl4uMpy4DKDcvvyzFYj4hSV7OnpVe\nfdVvN2+e9OlPS5/4RKjhooQV4qnLtKwAQJm4cCF1MjI4KD3//MR2CxZItbVSXZ1/EGH867o6ackS\nZqVF9JCsAECJGBubuF2TvAwNScPDE9tWV08kIO9618TXdXXSddcxsgelhWQFACLm0iXpm9+Unntu\nckISi03crnFOetvbfPKxYoW0bt3khKSqKtxrAPKJZAUAIuKVV6Tdu6XPfEY6f97fjonfonnPeyYn\nI9yuwVxCsgIAIRsbkx55xI/A+eEPpV//dWnrVp+ocLsGIFkBgNCYSceOSe3t0lNPSR/8oPR//o90\n441hRwZECzk7AITg1Clp1Srpve+VrrlG+ru/k44eJVEBUiFZAYAievZZaf166eabpX/5F+nxx6Vv\nfEO65ZawIwOii2QFAIrgn/5J+uhHpWXLpP/3/6Q/+zN/6+f225klFpgJfVYAoIAuXZJ27fLP2Lny\nSmn7duk3fkO66qqwIwNKB8kKABTAK69Ie/dKf/AH0k9+Iv3mb/oHAy5aFHZkQOnJOVlxzq2R1CSp\nVtKQpNNm9nCuxwWAUjQ2Jj36qPSpT0k/+IH0X/+r9NBDfgI3ALMz62TFOVcp6bR8kpJ4x3WTc65T\n0n1m9liO8QFASTCTTpzwrSdnzkj/+T/7JxgzugfIXS4dbPdLqpPUK+nO4OtFwWuHpIedc7fmHCEA\nRNzp01JTk3TbbdIb3yj93/8rPfYYiQqQL7ncBlot6YSZ3ZZUPiKp0znXLWm3pL/J4Ryh2rx5syor\nK9XS0qKWlpawwwEQMc8952/3HDzoR/n81V8xugdzV1dXl7q6ujQyMpL3Yzszm92Ozp2WtMfMDqTZ\n5liKZCbynHP1kvr6+vpUX18fdjgAIuaf/1n69Kelffukt7zFf33PPdK8eWFHBoSvv79fDQ0NktRg\nZv35OGYut4H2SmqYYZvFyQXOuQdzOCcAhObFF31n2bo6/yyfz35W+od/kD7yERIVoJByuQ00JGmd\nc+7XJcVSrN8k6YRzbmVCWVVQviuH8wJAUb366sQw5EuX/DDk9naGIQPFkkuy0iZplXzflVScJJO0\nNUUZAETe2Jjvj/KpT0lnz0q/9mvS7/2e9LM/G3JgwByTS7KyT37Y8t4s9nmTpPtyOCdQNmKxmGpq\nasIOA9OID0MeGJDuuMN3nv35nw87KmBumnWyYmbdzjkzsyPZ7Oec+/FszwmUulgspo6ODh06dEh1\ndXU6depU2CEhSV+fv8XT0yO9853SN78p/dIvhR0VMLfl9CDDdImKc+4+59ySFPvszOWcQCmrqalR\nW1ubhoeHww4FSQYHpZYWqbFRev556Stfkf72b0lUgCjIZQbbgzNsslr+FtEnZnsOoBxx6yda/uVf\nfMfZPXukn/kZ6cAB6d57pfk8OQ2IjFx+He/McBuSFQCR8+KL0uc/75+IPG+e9JnPSB//uHT11WFH\nBiBZLslKj/ww5Asp1r1D0joza83h+ACQd6++Ku3f7ydyGxnxCcq2bVJ1ddiRAZhOLn1WOswsZmYj\nKZYeScPOud/OV6CIvpGREe3fv1/Nzc06cOCAYrGYmpubVVFRocbGRl26dEmStHPnTi1dulTV1dVq\nb2+fcpyBgQGtX79ejY2NWrp0acptOjs7tWHDBrW3t6u5uVk7d07uCjUyMqLW1lY98MADam1t1dKl\nS7Vr167x81dXV6uiokJnzpyRJPX29o7HumHDhvHjdHd3q7m5Wb29vdq3b5+qq6vV2jqRg2cSazyW\n9evXa8OGDVNiRXZeftkPI37ySemJJ7JbvvhFPy3+xz8uve99fkK3nTtJVICoy2U0UO8MmwxK2iHp\nc7M9Rzn76U+lZ54J59w33FCYpu6hoSH19fWpp6dHzjkNDg6qs7NTQ0NDWrdundatW6e6ujo1NTWp\np6dHW7du1c6dO3XXXXdp+fLlkvw0zdu2bdOxY8ckSSdPntTq1asVi8V08KDvJtXW1qZdu3ZpdHRU\nkk80mpqaVFdXpzVr1kiSNm7cqLq6Om3fvl2SxhMVSdqyZYsGBwe1f//+8bJVq1aptrZWdXV142VH\njhxRe3u7YrGYamtrdfHiRdXV1enkyZMZxzo0NKTGxkYdOXJEt97qn+tJsjJ7P/yh9Pa3S5cvz/4Y\nH/iA7zz7C7+Qv7gAFJiZzWqRtDzNcp+k85LOz/b4YS6S6iVZX1+fFUpfn5l/qHzxlwJelvX09Jhz\nzlpbWyeV19XVWUVFhZ09e3bKtu3t7ZO2GxgYSLlvLBYzM7OmpiZbvHjx+Prh4eEp51y0aNGk45qZ\n7dy5c/zrtrY2q6iomHSu+HHWr18/XtbZ2TmlLJtY161bZ83NzVP2dc5ZY2PjlHKk96Uv+ffw44+b\nPfWU2XPPZbf86EdhXwFQ/vr6+kx+Ath6y9Pnci59Vvo1/Wy08WeObp1m/Zx3ww1+Poewzl0o1UF7\nelVV1aTy+vp6xWIxLUqYn7y2tlaSxofxDgwMaGhoSG1tbTIzOedkZlq0aJGccxoaGtKSJUvU3d09\n6dinT5+eEkdjY6M6OjpUXV2tLVu2SJIefDD7x1JVVVXJOTfp1lCmsZqZjhw5os7OzqzPi9SefFK6\n/nr/ZGMAc0cuycqwpEPBa7LzknrMbCCH45e1q6+W5voDneOJzYULvo/24OCgnHPq7u7WNddcM+1+\nCxculORv0/T09Gj16tWTjiNJe/fuVXNzs9ra2rR3714dPnxYK1asyFvsQ0NDM8ba29sr59x4Uobc\nPfmk9Iu/GHYUAIotlw62282s1czaUyw75fusABmLxfzzMAcH07914v1ALl68qN27d2vt2rVTtqmp\nqVFfX5/uvPNOxWIxNTQ0TOqjkquhoaEZY423rsS3RW5eeUU6c0a6+eawIwFQbLNOVmzmmWi3pZrB\nFphObW2tzGy8c2qyo0ePSpKampq0ePFi3Xff9I+ZisViWrhwoQ4ePKgTJ05I0qRRPIWO9ciRI+Mt\nKkypnx9PPeWHHdOyAsw9MyYrzrka59ypLJdn5furTP2XF5hG/HZOZ2enensnDzZrbW1VbW2tYrHY\neAtMXKqp6zs6Osa/Xrlypfbu9c/bPHv2rCRp8eLFMrNJ/V2ySSpmirWurk6NjY2S/PDn+LDtmeLG\n9L79bemKK6Sbbgo7EgDFNmOyYmYxSXWSGrJY6uQ72Zb0pHCbN2/WHXfcoa6urrBDKRnnz59PWX7x\n4kVJmnRLJPkWSmVlpdra2iT51pPm5ma1t7ersbFRixYt0vLly8f7ufT09Gjnzp3av3+/2tvb5ZxT\nf3+/Dhw4oJGRER06dGhSUmNmqq2t1ZIlSyRNJBttbW3jx4h33O3p6dGBAwckSc8991x8hFjWsVZW\nVo4nTfX19ert7dXAwMD4XCxDQ0Patm1bykQGUz35pLR8uXTllWFHAiCVrq4u3XHHHdq8eXP+D57J\nkCFJeyTVJJVtkbQlzT47JC3J17ClYi4qwtDlctTT02MNDQ1WUVFh1dXVtn//fjPzw38rKiqsoqLC\nmpubbWBgwPr7+62pqWnKtmZ+iPHSpUutoqLCli5dakePHp10nv3791t1dbUtXbp0fDhyW1ubVVdX\n27Zt28zMD12uqKiw1tZW27Rpk61fv358OHHieaqrq626utp27dplZmZLly61bdu2WSwWs3379ll1\ndfV4HIlDnzONNR5vfJvm5maLxWK2dOlS27Vr15SYML3rrzf72MfCjgLATAoxdNlZiv8akznnasy3\nsCSWHTSzDWn2WSFph5ndNpskKkzOuXpJfX19faqf60N2gAi4cEFavFj6i7+QPvzhsKMBkE5/f78a\nGhokqcHM+vNxzIw62CYnKgGXoixRtfyTlwEgJ/GuRYwEAuamXIYuX3DO3ZpqhXNuoaS9khizCSBn\n3/62VFXlJ4QDMPfkMilcm6SYc+6UpBPyiUmtgicuB9tsyi08APCda2++WXIztecCKEu5PMhwxDnX\nKKlDUqd8Z5rEPyVtZnYgx/gAzHFmvmUlj9PkACgxubSsyMyGJN3pnKuRb1WplW9hOW1mI3mID8Ac\n94MfSP/6r/RXAeayXPqsSJKCWWprzKzXzPZLqpJUk+txAUDyrSoSyQowl+WUrDjnDso/A2hvvMzM\njkhqdc5tzzE2ANCTT0pLlkjXXht2JADCMuvbQM65HZLulL/tM2necDNrdc6dds49aWaP5RgjgBLz\n+uvSww9LIyPS6Ojk5fXXp5ZNt7z+uvSNb0jvfnfYVwQgTLn0WVknabWZnXTOHU+xvke+4y3JCjDH\nxDvELlwoveEN0vz50rx5U5dMyv/jf5Q2bgz7igCEKZdkZdjMTgZfp5oGt16+wy2AOeZHP/KvP/yh\nVFkZbiwASl8ufVYSJ3ybNPuBc26V/Oy1TAoHzEEvvCBdfbVvWQGAXOWSrGx3zh1zzi2XZM65a5xz\ny4OOtcflW1v2pj8EgHL0wgvSW9/KJG4A8iOXSeEGnHPbJHXL3+6Jd7KN/3nqMLNdOcYHoASdO+eT\nFQDIh1wnheuXtDR4wnLipHA9TAqHTMViMdXUMDVPOYm3rABAPuRrUrhFZnbEzHYGxXzyzFGxWEzV\n1dU6cCD9kxZisZhaW1tVXV2t9evXFySOnTt3qpU52kPxwgvSddeFHQWAcsGkcMir4eFhjYyMqL+/\nP+12NTU1amtr0/DwcNrtZqO3t1dtbW1qa2tTX19f3o+PmdGyAiCfmBQOebVixQpdvHhRCzMYBlKo\nWz+rVq3SqlWrVFGRc8MhZuHFF/1CsgIgX3L5ax6fFG6ppIsp1scnhcMck0migtl773ulqio/f0ll\npR8evHCh9O/+nfRP/xR2dL5zrUSyAiB/mBQOKCEvvywdOybddZdUXz95aPAnPyn9xV9IW7aEF5/k\nbwFJJCsA8ieXZCWTSeEGczg+StCRI0e0d+9eOed07NixSetGRkbU1tamCxcuyDmnxsbGlMcYGBjQ\n9u3bNTQ0pOHhYa1bt047duyYtE1nZ6f6+vpUU1Oj/v5+NTU1aUvYn9JF8Oyzkpn0sY9J73zn5HVP\nPil98YvSgw+GO79JPFmhgy2AfMklWdnunDsmqU3BpHCS6iRtkLRVZTAp3ObNm1VZWamWlha1tLTk\n/fjnzp3TuXibeQoLFizQsmXL0h7je9/7ni5fvjzt+uuuu07XFelTY2BgQKdOnVJPT4+ampomrRsa\nGlJjY6OOHDmiW2+9VZK0c+fOKcfo7+/Xtm3bxhOdkydPavXq1YrFYjp48KAkqa2tTbt27dLo6Kgk\n36G2qalJdXV1WrNmTSEvMXTPPONf3/72qevuvVd6//ulgQHf6hKWF16QrrnGLwDmjq6uLnV1dWlk\npAAzl5jZrBf5Wz3PSRqTNBosY8GyPZdjh7kE12V9fX1WSA899JDJJ3Upl2XLls14jGXLlqU9xkMP\nPVTQa0g2PDxszjlrbm6eVL5u3bopZWZmzjlrbGwc/76urs4GBgYmbVNXV2cVFRUWi8XMzKypqckW\nL1485Zytra1pj10Ofv/3zd785tTrXnvN7Gd+xuw3f7O4MSXbvNns7W8PNwYA4enr64t/BtVbnj6X\nmRQuRJs2bdIdd9wx7foFCxbMeIzDhw/P2LJSTJUpnloXi8V05MgRdXam7289MDCgoaEhtbW1yczk\nnJOZadGiRXLOaWhoSEuWLFF3d/ek/U6fPp3Xa4iyZ56Rbrgh9br586UPfcj3W9m50z/tONH3vied\nPy+NjUmPPSadPVuYGPv7pbq6whwbwNyUU7IiSc65NZKaNJGoXCBRyUw+btHMdJsoCoaGhuScU21t\n+v7Wg4ODcs6pu7tb16S5hxAfbXTkyBH19PRo9erVkqQLFy7kL+iI+v7309/iuece6fOfl772NekD\nH5gov3hR+oVf8P1dJN+nZeVK6aqr8h/jTTdJBbhrCmAOy2WelUpJp+WTlMTufJucc52S7jPmWIF8\nsmJmGhpK/xDuWCwmyScty5cvT3u89evXq7W1Vbt3785rrFE2NuZbVu6+e/ptbrrJL1/84uRk5Sc/\n8YnKgQPSL/2StGiRdO21hY8ZAPIhl3lW9st3qO2VnxyuTtKi4LVD0sPOuVtzjhAlL96icurUqRm3\nM7PxjrTJjh49KklqamrS4sWLdd999+U30Ih7/nnppz+d/jZQ3D33SI8/LiU2NL36qn+tqfGdc0lU\nAJSSXJKV1ZJOmFmz+ecCxcxsJHjtlNQoqT0/YaKUxYcod3d369KlS1PWx6fcj9/O6ezsVG9v76Rt\nWltbVVtbq1gsNt4Ck7x/uYuPBJopWbn7bml0VErM+V57zb9ecUVhYgOAQsolWRmSdHi6lWaWvs0f\nZSmeOCTe8qmsrFRHR4ckqb6+Xr29vRoYGFB7e/v4ttu2bZNzTlu3bpXkW0+am5vV3t6uxsZGLVq0\nSMuXL1d1dbUkqaenRzt37tT+/fvV3t4u55z6+/t14MABXbp0aTyOckpknnnGJxtLlqTf7i1vkW67\nTfrSlybK4slKcqdbACgFuSQreyU1zLDN4uQC59yDOZwTEdbb26v7779/fOTOrl27xltStmzZMj5Z\nXDwJaW1tVV1dnTo7O7Vp0yYtXLhQO3bsUEdHh+rq6tTb26sjR47ok5/8pLZv98/FrKys1L59+1RV\nVaV9+/ZpZGREe/bs0ZYtW3ThwgUNDg5qcHBQ69evH49j27ZtOluooS9F9Mwz0r//99K8eTNve++9\n0re+5TvkShO3gUhWAJQiZ5ZqpvwMdvSz1G6VdEhSLMUmm+RbX04klFVJ6jCz62d10iJxztVL6uvr\n61N9mLNrAQlWr/YdYw9P25454fJl38LyG78h/eEf+sTllluk73zHjwoCgELp7+9XQ0ODJDUEU5zk\nLJehy22S4tPqp+LkJ4XZmqIMmNOeflp6+GHft8Qs/SL519On/TT7mViwQNqwwc+58gd/wG0gAKUt\nl2Rln/yw5Wym1H+TpLk1hANIoa1N+vrXpbe9zc95El+kyd8nLjfcMHk48kzuvVfat0964omJpIdk\nBUApmnWyYmbdzjkzsyPZ7Oec+/FszwmUg7Nnpf/9v6W9e6WNGwt3nltukZYu9R1t77rLlzEaCEAp\nyqWDrbJNVIJ9pj69DphD9uyRFi5MP7lbPjjn51zp7vYz2Eq0rAAoTTklK6k455bk+5hAubh82fdV\n+bVfk974xsKf77/8F+mllybmXCFZAVCKMr4NFDwDqFp+htoq+Qnhjias366gM61z7qKkjUy3D0x2\n+LD04x9LH/1occ63ZIn0y7/sbztJ3AYCUJqy6bPSLT+Sp1/S/WY2EF/hnDsoaZ0mnhFULanbOVdv\nZk/lK1ig1P3Jn0hNTX6+lGK55x7fmVeiZQVAacr2NlCPmb0jKVFZK/9sIEnaamYVZlYh6YCkzjzF\nCZS8vj7p298uXqtK3Lp1E09XJlkBUIqyTVY2pSjbL9/i0mlmu+KFZrZJ/pYRAEl/+qfSz/6sdPvt\nxT3vwoWU+S1uAAAZjElEQVTSmjVSRYVfAKDUZHMbyMzsbGJBcPunStKgmW1Lsc/FHGIDysaFC9Ij\nj0i/8zvS/FxmN5qlT36yuLeeACCfsvk/a8Q5d038m6DD7Z3yrSqpWlwkP2kcMOf92Z/52WrvC2lK\nxBtvlH73d8M5NwDkKpv/8Q5LOumc+6z87Z0O+USl28xOJm/snNso3+oCzGljY9Lu3dKdd0rXXht2\nNABQerJJVrbKjwTqDr53kvrMbEPyhkGry17xHCBAx49Lg4N+JlkAQPYyTlbMbERSnXNunaQaSUPJ\nM9g65yrlW2BqJfXIP3UZmHN+9CPp5pulF1+UXnlFWr7cT38PAMhe1l39zKw7zboRSc05RQSUgWef\nlc6d8x1bFy/2c6vEH1QIAMhOCOMSgPI3MuJf//t/l970pnBjAYBSx6wLQAEMD/vXyspw4wCAckCy\nAhTAyIifNZYZYwEgdyQrQAEMD0tVDNwHgLwgWQEKYGSEW0AAkC90sE1j8+bNqqysVEtLi1paWsIO\nByWElhUAc01XV5e6uro0Eh9hkEfOjHnbkjnn6iX19fX1qb6+PuxwUILWrpVeekn62tfCjgQAiqu/\nv18NDQ2S1GBm/fk4Ji0rQAovv+znSnn99Ynltdcmf5/O0JD09rcXJ1YAKHckK0AKGzdKf/mXuR3j\nPe/JTywAMNeRrAAp/P3f+1s5n/iENH++H4I8f/7E1/PmzTwjLQ8tBID8IFkBUjh7Vlq/XqLLEgCE\nj6HLQJLhYb/U1IQdCQBAIlkBpjh71r+SrABANJCsAEliMf9KsgIA0UCyAiSJxaSrr5be/OawIwEA\nSCQrwBRnz0pLlsw82gcAUBwkK0CSWIxbQAAQJSQrQBKSFQCIFpIVIMHoKMkKAEQNyQqQ4MwZ6ac/\nlX7xF8OOBAAQR7ICJPj616WrrpLe8Y6wIwEAxJGsAAmeeEJ65zulK64IOxIAQBzJChAYHZW+8Q3p\nV34l7EgAAIlIVoDAU09JIyMkKwAQNSQrQOCJJ+ivAgBRRLICBJ54QrrlFunKK8OOBACQaH7YAQDF\n9Od/Lh09OvG92cTXJ09K7e1FDwkAMANaVjBnPPyw9JGPSMPDE2UVFX6ZN0963/ukD30ovPgAAKnR\nsoI54fBh6f77pQcekP7kT3hIIQCUElpWUPa++lXfYnLXXdIf/zGJCgCUGpIVlLVvflNau1Z6z3t8\nf5UK3vEAUHL4042y1dMj3X679J/+k3TokPSGN4QdEQBgNkhWUJbOn5eamqTKSul//S9pwYKwIwIA\nzBbJCspSfMTPgQPSNdeEGwsAIDckKyhLly/7VxIVACh9JCsoS6+84l+5/QMApY9kBWUp3rJCsgIA\npY9kBWWJZAUAygcz2KJsfPazUm+v9NJL0re/7ctIVgCg9JGsoGzs3SstXCi94x1+bpW3vlV6y1vC\njgoAkCuSFZSNV16RNmyQPvWpsCMBAOQTfVZQNl55RbryyrCjAADkG8kKygbJCgCUJ5IVlA2SFQAo\nTyQrKAujo9LYmHTFFWFHAgDINzrYInLMpFdf9S0lly9PLOm+f+klvy8tKwBQfkhWkLHnn5eefdYn\nE/FFmvx9vOzv/176+tel116TXn/dt3y8/vrUJTHhSPx6Nt7wBqm2Nj/XCgCIDpKVNDZv3qzKykq1\ntLSopaUl7HBC9/73S089ldm28+dLK1dKb3yj/3rePP8aX+LfL1jglyuvnPg6m7L491dcIVVwUxMA\nQtPV1aWuri6NjIzk/djO4v8KY5xzrl5SX19fn+rr68MOJxJ+8hM/4dqOHdKaNZJzfpEmvk4su+Ya\nadGi8OIFAISjv79fDQ0NktRgZv35OCYtK8jImTP+Fs9tt0lLl4YdDQBgLqHhHBk5fdrfblm2LOxI\nAABzDckKMnL6tHTTTb4TKwAAxUSygoz09UmNjWFHAQCYi+izgilef11qaZH+8R/9kOPRUen735fa\n2sKODAAwF5GsYIrnn5e6u6X3vU9629v8MOOVK6Vf/dWwIwMAzEUkK5ji3Dn/umOH9B/+Q7ixAABA\nnxVM8cIL/vW668KNAwAAiWQFKZw750f9LF4cdiQAAJCsIIVz56S3vGViNloAAMJEn5UIMvMP83v5\nZemnP/Vfj41NrEt+gGC2X8+03Xe+I731rYW9RgAAMkWyksYf/ZFUXe2fBnzhgnT+fOrt0j1e6dVX\npR/+0D99eGzML6OjE19P933Y7r477AgAAPBIVtJ44gn/QL4rr5SqqqQ3vckP481GRYX0rndJV13l\nv05e5s1LXX711X656io/zX1FxdQHByZ+nW7dbLarrZ1VlQEAkHckK2l85SsSD10GACBcdLAFAACR\nRrICAAAijWQFAABEGskKAACINJIVAAAQaSQrAAAg0khWAABApJGsAACASCNZAQAAkUayAgAAIo1k\nBQAARBrJCgAAiDSSFQAAEGkkKwAAINJIVgAAQKSRrAAAgEgjWQEAAJFGsgIAACKNZAUAAEQayQoA\nAIg0khUAABBpJCsAACDSSFYAAECkkawAAIBII1kBAACRRrICAAAijWQFAABEGskKAACINJIVAAAQ\naSQrAAAg0khWAABApJGsAACASCNZAQAAkUayAgAAIo1kBQAARBrJCgAAiDSSFQAAEGkkKwAAINJI\nVgAAQKSRrAAAgEgjWQEAAJFGsgIAACJtziQrzrnKsGMAAADZK/tkxTm3yjl3WtKOsGMBAADZK+tk\nxTm3UNLpsOMAAACzNz/sAArJzC5JknPuQtixAACA2Ylsy4pzrso5t8M5tyTsWAAAQHgi2bLinNsi\naZukSkmPTrNNpaQOSYskuWDbdjMbKFacAACg8CLXsuKcWy5pr2bua9IvqcrMNpjZekmdknqD/QEA\nQJmIXLJiZmeCviZD023jnOuQtETSxoT9eoN9Dhc6RkzV1dUVdgglhfrKDvWVHeore9RZdopdX5FL\nVjK0UdKQmb2YVH5QUi2tK8XHL3p2qK/sUF/Zob6yR51lh2RlBs65FZKq5G8DJeuX77+yKam8utBx\nAQCAwii5ZEVSY/CaajhyvKw2XuCcWytphaTVzrmVBY4NAADkWSRHA82gKngdTrEuXjaerJjZEUnz\nCh0UAAAojFJMVlIlKfm2QJKefvrpIpyqPIyMjKi/P9WdOaRCfWWH+soO9ZU96iw76eor4bNzQb7O\n58wsX8fKK+fcHvmOtA1mdiahfJWkE5L2mtkDSfuskNQnqc/M3pHDue+W9Jez3R8AAOhDZvZIPg5U\nii0r8SHNqTrNxstyfR7QMUkfknRW0uUcjwUAwFyyQH56kWP5OmDJJStmFnPODUuqT7G6XpJJOpTj\nOc5Lyks2CADAHPR3+TxYKY4GkqR98vOpLEwqb5Y0aGZ/E0JMAACgAKKcrCyeboWZtcvfDtofL3PO\nrZa0UtKdhQ8NAAAUS+Q62AadZDdI2hIU9Ug6bGYHUmy7W76fykVJNZK2mtlTxYoVAABkLviMb5B0\nKHi0Tmb7RS1ZQflzzlVJape0x8zOhhwOgIBzbq+CB8SGHUspcc6tMLOBsOOIKudcpaRPSFojqUNZ\nJipSCXawzURQMR2SFslPv18pqT3TN5NzrlbSDknn5W9HVUraF0wwl7fzREWx6ivYdoukbcE2j+bl\nAkJQxPdYbXCeVUFRj6SNZjaSj+soliLX1w5Jq4OiHkltZhbLx3UUSzF/JxP2qZefLqLkHgZb5L9h\nhyWtTSo+Iem2WV9AkRW5vurlfw+Pm9n1sw7azMpukTQo6dGE71fJT8W/PIN9a4Ntb0ooq5E0JmlN\nvs4TpaWI9bVc0kJJxyWNllo9FbvO5Ee3XZAf/ndM/g/DWPC6MOw6iGB91Up6TtKD8v/BHQy2eTbs\n649ifaXYL/57eTDs649qfcl/KCf+Th6T9LVS+1tWxPqqD8qfzDnmsCutAD+EjuAX7pqk8tOZ/NEK\n9p+yXfCLfCxf54nKUqz6Slq3RyWcrBTxPXZc0s8lfL8wOMeopO1h10ME62tPinMcCs69JOx6iFp9\nJa3bIT9AYUwllqwUs76Cepo24SuFpcj1dTHVuWazRHk00GxtlDRkZi8mlR+UH+68fIb9a5R6WHRt\n0ve5nicqilVf5aTgdeacq5FvNv1BvMz8Pd6N8s22qeYZiqpivcd2pDjHKUmy0uobVdTfyaCZvtLM\nTs4q2vAVs77ul1Qd/H6WqqLUl3Nuq3xLVHeKc2WtrJKVoJdxlaRUDyzol/8jv2mGw5wIthu/bxv8\nMtfIZ9X5Ok/oilVf5aSIdXbezHYl72gT95SL8YysnBXzPTZNQvIOSVuzCjpEIf1O7rCkR5eUimLW\nV9DfrlJ+nq9B59xzzrnkviuRVuT31/3yk7QOOeeOO+fGgjqb1edCWSUrkhqD1wsp1sXL0v7Hb2b7\nJXVLWp3wZjwkabVNTDaX83kiolj1VU6KUmc2TU/5hP/ojmcbeEhCe48F/9mZmX0u+7BDU9T6Cj44\nSvmfimLW12H5D/IO+T4fNZIOO+e25xB/sRWlvoIOvPHjnJe0Lvj+hKStzrmsp+Evt2SlKnhN9V9n\nvGzGJMLM1stPOFcr/wbdk/Smzct5IqBY9VVOwq6zO+Uf1PlwBttGQdHryzm31jl3WtJ2SXc653J6\n/EaRFa2+yuD2j1TE+jKzs2Z2wMy2mR/Vskm+5WCrc27lbC+gyIpVX/Fj9JvZLjO7FNTfA/ItOKuz\nrbNyS1by2TReLWmv/JuxM3gKdCHOE6Zi1Vc5Ca3OgvlpNsr/l1Iqil5f5odPrpNP7AYlrS2h/36L\nWV/bVUK3yKYR2u+j+YlKm1Uit/0Dxaqv+EOFh6bs5fdxkpqyOVm5JSvxiqlKsS5dRjnOOVcZ/Fd2\nLMgCG+V7NG9M+IOX83kiolj1VU7CrLNDku5P7HRbAkKpr+C/uKPBtk6lk+AVpb6C/heHJVUE21cG\nybAkVcXLcr2YIgj1b5iZ9cq3FJRCS7oUjc/I+Lqs6qxck5XqFOviZadnOEanpBXxZvagQ2ODpBH5\n5r6FeTpPFBSrvspJKHUWfLgcKsHba6G+x8xPntc9zfmjqBj1VSn/SJN98h8y8SXeZ6Ep+L5nltdQ\nTFH4Gzak1H1Aoqgo9WUTkzCmOk88hqzqrKySlaCChpV6WGe9fHPVTPevG5TUUzoYZbAv+LY2T+cJ\nXbHqK7cooyWMOgs6sF20FM/HirqIvMcuqDT+eShWfdVIuk9+lt/kRZL65CcJ25j1BRRZRN5fVUp9\nuyNyilxf/dOcJ77+uZkjnlBWyUpgn1KPAW+WNJjBf6ZDSv3mHJIkMzuTp/NERbHqq5wUrc6cc6sk\n1aRKVILWllIQ9ntstUprxEvB6ytYTiYv8h9kQ2b2NyX0uxva+yu4ddag0ur7U6z6apOkFB1p6+ST\nov3ZBB36bHqFWCQ9q4RZGOX/WI1q8vTAlfKzNZ5K2ndFsO2DSeWnJX022/OUwlKs+kpYd1glPINt\nsepME9PtH0paDgflK8Ouh4jVV7yuahLKdiTvVwpLsX8nk+qwpGawLUZ9BduMBe+vFQnHOyTpg2Ff\nf9TqK6HseHCuyoSy5yT9drYxl+1Tl51zu+Xvl12Ub/bcamZPJW3zrPww0LuSypfLj6WXfLZYLemE\npf7vdsbzlIJi1FcwIdEGSfEWgR5Jh1PVaykoZJ0F86kMyv8HkspFM3tTvq6lGAr9HguOv1r+v77u\nYLu9Vlqz144r1t+wpP3OB9vdlW67KCrw72N8Mrh6TX5/bbcsnx4cFUX8jNwuX2/x7R41s8eyjrdc\nkxUAAFAeyrHPCgAAKCMkKwAAINJIVgAAQKSRrAAAgEgjWQEAAJFGsgIAACKNZAUAAEQayQoAAIg0\nkhWgxASz2wKRw3sThUKyAhSBc67GObfFObcnRfkF59x9Gey/xzkXf/5NvmKa8dzF5pwriSck54tz\nrj742S6fYbuo/rzy/t4EkpGsAAmcc2udc88558aC5VSqDwfn3MaE7U6l+6AJnpzcESwNSaur5B8Y\nlupR6uPMP9q9I9g+XzI6dzE55+6XtGK6+gx+PocSfj7nnXMPJj5B1jlX6ZzbkbDNsRRPfo0nCYeC\nn+MF59xowj5jzrljBbzUeAwb5Z+RtVH+uSnpRO7nJRXsvQlMwrOBgCTOuSXyD90ySW1mtmua7dbK\nP9xsiZm9mMFxx+QfCvaOpPKFmT4MbbpjZLBfjTT+wTKrcxdD0KqyQlK3mW1Is90F+Q/uddM9FM05\nt0XS/WZ2fVJ5paQDktZK2iOpM/6wwyBJOiz/sLoeM7st54uagXNuh/zDPZvM7OQM20bq55Votu9N\nIBO0rABJgg+uTklO0qY0mzbJP3V1xkRlhvMV48PnhPyHexjnzkjwVO4L8vW+LrG1JIULwetImm36\n5ZPOVOVrJG00s48mPpXZzM4Eyc2AZm7pyJfzmW4YpZ8XUEwkK0Bq24PX2jS3eNbLt6xEmnPusPwj\n4KNuk6QdmqjT+/NwzOHEb5xzHfJ10WdmD6fZ7848nBtAnpCsACmY2Yikbvn/8rclr3fOrZN0Kv6f\nrnNuq3PuYNBX4nhwC2JGQR+M46n6RwR9L/YE/SoOTnfMdOcOblWtCL7tCPpvLM/g3FXBuXcH2xwP\n+t4kxna/c+60c26Nc25F8PWYc+5gJteewqrgNkiHZm7Vmq0t8rf3dqTbKLhdNmNH30x+7gk/x8S6\nXDHN8TZOV48z/LxWBO+T00EfnB0J61YFfXLi/auWJKyL9+3ZnsmxUlxT2vcmkDdmxsLCkmKR/w98\nTNKopIVJ605LujX4ukPSaMK6VcF+a5L2GZNPcOLfr5D/0ByTdCxp21r5Wx23JpRtSXGMGc8dnGNU\n0k0Znrs+OHfi9huDbbcn7H88OO4xSbslrZR0MNjuwSzremP82An1Oypp5TTbP5dufUJdHExRN6Py\n/YxyfX9kUve1Qaw/l1B2IfE9lfBzPSjforc8qM/xeszg53Us4fuV8eMllK1JLgvKK5PeT5kcK6P3\nJgtLPpfQA2BhifKS8IGc+EFaK+l80jY/Tvi+MvjDvTvpWFP+mCdsm/wBdDi5LNUxMjm3JpKV5Rme\nu2+ac59OPI6SEpigrCbVMTOo59OSrkn4PuWHa8L62SQra4NjjmYT2wzvjZnq/rSk307ab4uk1zU5\nWRmVdF+KY51KUZb883ouxc82Xj9LkstSxLImm2Nl+t5kYcnnwm0gIL34LYnE/hP3a3JflXXyCUxc\nY6YHN3+7aZJg5M5a+U6xMynEuVfId0JNtle+LuKjdOKdXAcTjhkfbZRx59TgnBcsoaOymR2V728y\nU0fbWcnTMdPWvXOuVr6lojex3Mx2mtl8m9pZdihhmxH5669KKpskuJ1UK3+L73hwm++4pIvB8RLj\n2xvssyahbENQ1xkdK8v3JpA388MOAIgyM+t1zg1JqnHO3WdmB+STlSUJ28T7rayVtFp+3gxp9qNJ\nauX7VaQayZIcX77PnW4Oj9MZbDMb7ZIanHOJo2KcJkYv3S8p5fDxLCXWZ62kM7kcLIO6r5f/OQ5P\n3Ttv4u+VdTbzqLR98sn3NklHg8TjVDbHCvotZfTeBPKJlhVgZvHWlbbgg+lU4n/FzrnaYH6QRWb2\ngJkdyfF8tcH5amfasADnTowh2XDSa76sNLPFSUu1fEvFdB1t4x+W6SYiq9JE64/MbCBhXcYtUNPJ\noO7jI7Bm/DnmIH7supk2tIlO4/VBorJJQWtLFsfK+L0J5BPJCjADM9sv/wFdK2m//ERiiU7I92E5\nkKdTxj+IM5lcK9/njt/+SdV6Ek8MnszTuRJncJ0iSC7itx+SZ6CN11G6D81aJdyiCuxT/kYazVT3\n/cG5Ug6DTjMkPhtDmnxrLvkca5OKtmvi+leYWWLrUibHyua9CeQNyQqQmfgfebOEGVOD/1AnzWHi\nnMt12vH47Zbp+mtUzfLcM94aCvqc9Cv1/DJN8v0X9s90nCy0aWrylyg+fLwtqfyw0nywBjYE+yef\nb1i+dWHa4bZBq8nuNOszqfv4z/H+5KTBOXdI+bmVEk/0tiYOLQ/OsUdJyVpCArhVvoNwtsfK6L0J\n5BvJCpCZeIfavUnl8dsMq51/UOFG+dE3Jv+BeJ9zbmHCB9mkP+YJ5eMtBEFzffzDuT+YJ2NFwnwX\ntcG8GGOZnFv+Q8ZJujM4zprpzh24U/4DfTwpCba9X37ESvwW2OLka5ruOlNxzq0Ozh1Ls1m8I+fq\nxLlJzKxXE7c0DsXXBfN/rA5uz3RZwuy0wX4j8q1GQ5Lic6OMHzeony3yw4g70sQ14889+DregnM4\nmLfkkPOPCjiWUI9LlfrWSlViWZr3SjzOE8H17Aiu/2JSy0nc3iC2SUlnJsfK9L1ZiE7RmOPCHo7E\nwlIqi/zcF0tSlN8nP2X6s5qYF2NHULZdk+ckGQ3KlsgPrT2UUP6gEuZzCY77rCbmMlkSfP/bmhhG\nmvbcCcc6FZTtDr6f6dwL5T+wTwXXvVsJQ1qD/eNDWp+VH2pcKd9KMn6daepyrSbmGzmmpOGyCduc\nTjjes5o6d82aYP/zwTbng+9vne7cSfWbat/7Zto3y7pfE9TjaPCaOD/JxoTzn5Kf1yS5Hndn8PN6\nMOG98qykD6aJu3KGn82Mx8rkvcnCks+FBxkCAIBI4zYQAACINJIVAAAQaSQrAAAg0khWAABApJGs\nAACASCNZAQAAkUayAgAAIo1kBQAARBrJCgAAiDSSFQAAEGkkKwAAINJIVgAAQKSRrAAAgEj7/2rC\ntCoYXlxNAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiQAAAF7CAYAAADiw5DPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3Xl0XGeZ5/Hv68RLFluLk4akh2BJDiSZJrElA03THU5i\ny6EHAoO3IEgfeoi30MCMm8iyG6bD6enBlmxomBnwmjPTTYLiLUCGBmxLZuDQNGBLduiQBGJJDmSD\nxLbkbF4iPfPHe0sqlUq1V92q0u9zzj1Xuve9977vrWvXo/e+izMzRERERMI0KewMiIiIiCggERER\nkdApIBEREZHQKSARERGR0CkgERERkdApIBEREZHQKSARERGR0F0cdgaKkXNuJnAbcAI4G25uRERE\nSso0YBaw38xOpnqQApL4bgMeCDsTIiIiJeyjwDdSTayAJL4TAPfffz/XX399yFkJx5o1a/iHf/iH\nsLMRmoleftA9mOjlB90DlT+z8j/++OPceeedEHyXpkoBSXxnAa6//nrq6+vDzksoKioqJmzZQeUH\n3YOJXn7QPVD5sy5/Wk0e1KhVREREQqeAREREREKngERERERCp4BE4mpqago7C6Ga6OUH3YOJXn7Q\nPVD5C1t+Z2YFvWApcM7VA11dXV0TukGTiIhIurq7u2loaABoMLPuVI9TDYmIiIiETgGJiIiIhE4B\niYiIiIROAYmIiIiETgGJiIiIhE4BiYiIiIROAYmIiIiETgGJiIiIhE4BiYiIiIROAYmIiIiETgGJ\niIiIhE4BiYiIiIROAYmIiIiETgGJiIiIhE4BiYiIiIROAYmIiIiETgGJiIiIhE4BiYiIiIROAYmI\niIiETgGJiIiIhK6sAxLnXI1z7oBzbsg5d9w5tzjsPImIiMhYZR2QANuAZqAe6AF2O+fmhJslERER\niXVx2BnIl6A2ZKWZnQg23eacGwIWAMdCy5iIiIiMUbQ1JM65SufcRufcrEyON7N9UcFIRD/QlWXW\nREREJMeKsobEOdcMrAcqgAfHSVMBtAJVgAvSrjOzo+OkrwVOmtkP8pJpERERyVjR1ZAEbTy2AUeS\nJO0GKs3sDjNbBrQBnQnaiKwFGnOXUxEREcmVogtIzOyYmZ0BesdL45xrBWYBK6KO6wyO2RMn/Qrg\nQJxXOCIiIlIEii4gSdEKoNfMXorZvguoja4lcc7NB3rM7KFCZlBERERSV3IBiXNuLlCJf2UTqxvf\nnmRVkHYBvstvXzAmSa1zrtk5N6NgGRYREZGkirJRaxLzgvWpOPsi22qDmpEDgAEbo9LsNbNNecyf\niIhISXrpJRgcBOdGlkmT4v8c+3u2SjEgqQzW/XH2RbbVBm1KSq4GSEREJAzf/jb8x/+Y+fGVlXD6\ndObHl2JAEi8QERERkSw8+CBcfz383d+BmV+GhlL/efLk7K5figFJpPdNZZx9iWpP0vb444+Pu2/a\ntGnccMMNCY9/7LHHOHv27Lj7r7rqKq666qpx97/22msJ8wBw/fXXc8kll4y7/7nnnuO5554bd7/K\nMULlGKFyeCrHCJVjRDmW4/x5+N73YM0aWLLE789FOdJiZkW5AFuBQWBOzPYaYAjYFeeY+cG+LVle\nux7f9mTc5YYbbrBkbrjhhoTnuPfeexMe/+ijjyY8HrBHH3004TnuvfdelUPlUDlUDpVD5UhYjoMH\nfX3H0aPpleMb3/iG3X777aOWm2++OZKm3tL47nXmv4CLjnNuK757b4OZHYvZdwo/6uq1Mdub8Q1Y\nF1gWI7I65+qBrvvvv5/rr78+bppijHDjKcVIPR6VY4TK4akcI1SOESqHl245Pv1p+Na34KmnRhqo\nZlqO7u5uGhoawH9/x+sRG1epBiQb8bP4VpkfRC2y/SDwZjN7S5bXrge6urq6qK+vz+ZUIiIiRc0M\namrg/e+H//W/sj9fpgFJMfdCmTneDjNbh29LsiOyLRhz5FZgaf6zJiIiUh5+8QtfM/LBD4abj6Jr\n1BoMfHYHsCjY1Oqc22NmO6PTmdm1zrktzrldwGl825J6M3uksDkWEREpXQ8/DNOnw3veE24+ii4g\nMT9b71FgXQpp785/jkRERMrXww/Dn/85TJkSbj6KLiApJmvWrKGiooKmpiaamprCzo6IiEhOPfMM\nHDniu/tmq729nfb2dgYGBjI6vmgbtYZJjVpFRGQi2LoVPvlJeOEFqKrKzTnLsVGriIiI5NHDD8PN\nN+cuGMmGAhIREZEJ6KWXoLMz/N41EQpIREREJqADB/yQ8bffHnZOPAUkIiIiE9DDD8Mf/RHU1oad\nE08BiYiIyATz+uvwne/ABz4Qdk5GqNtvAur2KyIi5egnP4FTp3LbfkTdfvNA3X5FRKSc3XMPPPCA\nH4dkUo7flajbr4iIiCRlBt/+tm/MmutgJBtFlBURERHJtyeegOPHi6e7b4QCEhERkQnk4Yfh0kvh\n1lvDzsloCkhEREQmkG9/GxYuhEsuCTsnoykgERERmSB+9zv46U+L73UNKCARERGZMP75n/36fe8L\nNx/xaBySBDQOiYiIlIPf/ha+/GXYsQPe8x648srcX0PjkOSBxiEREZFy8MgjsHkzPPggXH453H03\nrFmTn4AkItNxSFRDIiIiUoaWL4f77oNrroFNm+Cuu2D69LBzNT4FJCIiImXmwAEfjHzpS/DJT8Lk\nyWHnKDkFJCIiImVkcBCam+Hd74b/8l/AubBzlBoFJCIiImXkn/4JfvEL3723VIIRULdfERGRsvHK\nK/C5z8Edd8A73xl2btKjgERERKRMfOlL8OKLsGFD2DlJnwISERGRMvD889DaCp/6FNTUhJ2b9KkN\nSQIaGE1ERErF5z8PU6bAZz8bzvU1MFoeaGA0EREpJY89Bm97G3zxi75nTZgyHRhNr2xERERK3Nq1\n/jXNJz4Rdk4yp1c2IiIiJayz00+at2ePf2VTqlRDIiIiUqKGhuCee+Bd74LFi8POTXZUQyIiIlKi\n7r8fjh2Dn/yktAZBi0cBiYiISAk5cMD3pDl3Dk6cgKVLfQ1JqVNAIiIiUkL27YMjR+DTn4Zp02DN\nmrBzlBsKSEREREpIfz/Mnw9f+UrYOcktNWoVEREpIf39UFkZdi5yTzUkCWikVhERKTanT8M114Sd\ni7E0UmseaKRWEREpVm95C3zwg7BpU9g5iU8jtYqIiEwA5frKRgGJiIhIiTDzr2yqqsLOSe4pIBER\nESkRr74Kr7+uGhIREREJUX+/X5djQKJeNiIiIkXst7+Fj38cTp6ESAeWcnxlo4BERESkiB09Ch0d\n8LGPwcyZcOWV4DuxlBcFJCIiIkXs7Fm//spXoKIi3Lzkk9qQiIiIFLFIQDJtWrj5yDcFJCIiIkXs\n3Dm/njIl3HzkmwISERGRkL32GnzjG3Dw4Nh9Z8/62hHnCp+vQlIbEhERkZB997vw0Y/6n195BS69\ndGRfJCApd6ohERERCVlkfBGAZ54Zve/cOZg6tbD5CYNqSBLQbL8iIlIIL7888vPBg36JvKL5yU9K\no4ZEs/3mgWb7FRGRQvrv/x1aW+Gll0a2XRxVZfDe98L//b+Fz1cmNNuviIhIiXr5ZT/gWXW1//3e\ne+HChZGlVIKRbCggERERCdnLL8Pll8Mf/qH//Zprws1PGBSQiIiIhCw2IJk0Ab+dJ2CRRUREisPn\nPgc33ggPPeQDkquv9tsVkIiIiEjBfP3rfsyR//SfoKVlJCAp90HQ4lG3XxERkRBcuABPPw2f/Sys\nXOm3/frXfn3+fHj5CotqSERERELw9NMwNASzZo1si7QhefbZULIUKgUkIiIiIXjqKb+OF5DEjtY6\nESggERERCcGJE34d3cX3LW/x69tvL3h2Qqc2JCIiIiE4cQKuumr0sPCXXw4TdQB11ZCIiIgU2OAg\nbNwIb35z2DkpHgpIRERECuy55/wsvjfeGHZOiocCEhERkQK7cMGvly0LNx/FRAGJiIhIgb3+ul9P\nnhxuPoqJGrUmsGbNGioqKmhqaqKpqSns7IiISJmI1JBcXEbfwu3t7bS3tzMwMJDR8c4manPeBJxz\n9UBXV1cX9fX1YWdHRETKRH8/9PbCo4/Cxz4GP/85vP3tYecqt7q7u2loaABoMLPuVI8ro9hMRESk\nuL373fDYYyO/V1eHl5diozYkIiIiBfK738GnPuVrSJ56Curqws5R8VANiYiISIG8+qoPQv79vw87\nJ8VHNSQiIiIFYAavvQaXXhp2ToqTAhIREZECOHvWrxWQxKeAREREpABefdWvL7kk3HwUKwUkIiIi\nBfDyy359+eXh5qNYKSAREREpgJMn/XrmzHDzUawUkIiIiBSAApLEsu7265xbBDQCtUAvcMTM7sv2\nvCIiIuXkxRf9WgFJfBkHJM65CuAIPhBxUbtWOefagOVm9s0s8yciIlLyvvc92LDBT6anNiTxZfPK\nZgdQB3QCS4Ofq4J1K3Cfc+6WrHMoIiJS4jZtgjNn4G/+BpxLnn4iyuaVzQLgoJndFrN9AGhzzu0F\ntgA/yOIaIiIiJe2ll+DHP4YvfQk++cmwc1O8sqkh6QX2jLfTzHqzOLeIiEhZ+MEP4MIFeO97w85J\nccsmINkGNCRJM6bpjnPuniyuKSIiUlKOH/ftRmbPDjsnxS2bVza9wBLn3F1AX5z9q4CDzrlbo7ZV\nBts3Z3FdERGRknHuHEybFnYuil82AUkLMB/fliQeBxiwNs42ERGRCeH8eZgyJexcFL9sApLt+C6/\n29I45gpgeRbXFCkbfX191NTUhJ0NEcmjwUH4/OfhmmvCzknxyzggMbO9zjkzs33pHOecezHTa4qU\nur6+PlpbW9m9ezd1dXUcPnw47CyJSB5F5q+5445w81EKsho6PlEw4pxb7pybFeeYTdlcU6SU1dTU\n0NLSQn9/f9hZEZECOHvWr//sz8LNRynIZqTWXUmSLMC/zvmbTK8hUo70mkZk4jh3zq/VqDW5bNqQ\nLE0xjQISEREpa2fOwLPPwqOPwm9+A888Ay+84Lv8AkydGm7+SkE2AUkHvgvvqTj73g4sMbPVWZw/\ndGvWrKGiooKmpiaamprCzo6IiBSpP/1T+Ld/8z9fdJFvxHrFFVBTA+94B9TXh5u/Qmhvb6e9vZ2B\ngYHMTmBmGS3A/CT7NwKfyfT8YS5APWBdXV0mqevv77ft27dbY2Oj7dixw3p7e62xsdGcc9bQ0GAD\nAwNmZtbW1mZ1dXVWVVVlLS0tY87T3d1tS5cutYaGBqurq4ubprW11ZYtW2YtLS3W2NhobW1tY/Ky\natUqW716ta1atcrq6ups06ZNw9evqqoy55wdPXrUzMw6OjqG87ps2bLh8+zZs8caGxuto6PDtm3b\nZlVVVbZq1aq08hrJy9KlS23ZsmXW1tZmzjmbN29eBndZRIpRZaXZJz5hduxY2DkJX1dXl+GH+Ki3\nNL57s+ll05kkSU8QlHwx02uUs1dfhSeeCOfa110Hl16a+/P29vbS1dVFR0cHzjl6enpoa2ujt7eX\nJUuWsGTJEurq6mhsbKSjo4O1a9eyadMmPvzhDzNnzhwAuru7Wb9+Pfv37wfg0KFDLFiwgL6+Pnbt\n8s2WWlpa2Lx5M4ODgwB0dnbS2NhIXV0dixYtAmDFihXU1dWxYcMGADZvHhmLr7m5mZ6eHnbs2DG8\nbf78+dTW1lJXVze8bd++faxbt46+vj5qa2s5ffo0dXV1HDp0KOW89vb2Mm/ePPbt28ctt/i5Jjdt\nUrtukXIyNAQDA3DTTX6RDKUTvdjoWoQ5CZblwEngZKbnD3OhADUkXV1mEM6Sz4qfjo4Oc87Z6tWr\nR22vq6uzSZMm2YkTJ8akXbdu3ah0kVqL2GP7+vrMzKyxsdFmzpw5vL+/v3/MNauqqkad18yGa0jM\nzFpaWmzSpEmjrhU5T3QNSaQ2I3pbOnldsmSJLVy4cMyxqiERKR/f+Y7/v/XBB8POSXEoeA0J0M34\no65GJldeO87+Ce+666CrK7xr50t1dTUAlZWVo7bX19fT19dHVVXV8Lba2lqA4S6wR48epbe3l5aW\nFswM5xxmRlVVFc45ent7mTVrFnv37h117iNHjozJx7x582htbaW6uprm5mYA7rkn/WmUKisrcc5x\nR8wgAqnk1czYt28fbW1taV9XRErH3/+9X994Y7j5KHXZBCT9wO5gHesk0GFmR7M4f1m79NKJ0cgp\nkUjwcuqUbxfd09ODc469e/cyffr0cY+bMWMG4F+pdHR0sGDBglHnAdi2bRsLFy6kpaWFbdu2sWfP\nHubOnZuzvPf29ibNa2dnJ8654cBLRMrTuXOwejVcf33YOSlt2QyMtsHMVpvZujjLJnwbEpGU9fX5\nORp7ehI/OpF2GadPn2bLli0sXrx4TJqamhq6urpYunQpfX19NDQ0jGozkq3e3t6keY3UkkTSikh5\nOndO3XpzIZtGrcla5q13zm0zsxOZXkMmltraWsyMXbt2DTdyjfbQQw+xaNEiGhsbmT17NsuXjz8t\nUmSemF27dg03Nl29ejUrVqwoSF737ds3XDOi4eFFSs/Bg3DsGPT3+8nxzp/3o65G/x5ZensVkORC\n0hoS51yNc+5wmsuT+PYjY/90FRlH5NVLW1sbnZ2jO3GtXr2a2tpa+vr6hmtSIuINw97a2jr88623\n3sq2bX4OyBMnTgAwc+ZMzGxU+5N0Aodkea2rq2PevHkA7N27lzNnzow5h4aPFylOr70G732vnxTv\n/vvh4Yfh0CE4fNgPdhaZvbe62o838pGP+EWyk7SGxMz6nHN1QGWytHGsRt1+J5STJ0/G3X769GnA\nv8aI1CjEvu6oqKigpaWFtrY2GhsbWbBgAfX19XR0dNDY2MicOXOGB9zp6Ohg06ZNVFZW0tXVhXOO\n7u5udu7cydKlS9m9ezctLS3Dw7SbGbW1tcyaNQsYCSgijVJ7enqGA4SOjg527tzJ8uXLOX78eKTn\nVdp5BR8YrVu3jvr6erZt20Z1dfWoLsHr169n/fr1w+1iRCR8AwO+K++DD8Ltt4edmwkkla44wFag\nJmZbM9Cc4JiNwKx0uvwUy4IGRstIR0eHNTQ02KRJk6y6utp27NhhZr7r7KRJk2zSpEm2cOFCO3r0\nqHV3d1tjY+OYtGa+e+7s2bNt0qRJNnv2bHvooYdGXWfHjh1WXV1ts2fPHu7K29LSYtXV1bZ+/Xoz\n891+J02aNDww2rJly4a74kZfp7q62qqrq23z5s1mZjZ79mxbv3699fX12fbt2626uno4H9HdhlPN\nayS/kTQLFy60vr4+mz17tm3evHlMnkQkfL/6le/G+8Mfhp2T0pRpt19ncf76i+WcqzGzvphtu8xs\n3AmVnXNzgY1mdlsmgVKYnHP1QFdXVxf1E70rjIjIBHP4sB/u/ehRiNNETJLo7u6moaEBoMHMulM9\nLqVeNrHBSMDF2RatGj/jr4iISMn48If9OmrYJCmAbLr9nnLO3RJvh3NuBrANUH9HEREpKb290Njo\nG6xK4WQzMFoL0OecOwwcxAcftQQz/QZpVmWXPRERkcJbuhRcsvcAklPZjEMy4JybB7QCbfgGLNEf\nX4uZ7cwyfyIiInl16BA884zvXfPii37bRReFm6eJKJsaEsysF1jqnKvB147U4mtKjpjZQA7yJyIi\nkjc//SnMn+9/njIFrrgC5s7V1B5hyCogAXDOzcJ3Ce4EOp1zi4Ea4Fi25xYREcmn//2/4U1vgl/9\nCi65JOzcTGzZNGrFObcLP2fNtsg2M9sHrHbObcgybyIiInlz9izs2gUf+5iCkWKQcUDinNsILAX6\niJnx18xWA43OuQ9llz0REZH86Onx7UZuK7nRsspTNjUkS4AFZjYbOB1nfwe+sauIiEjRef11v542\nLdx8iJdNQNJvZoeCn+MN91qPb+QqIiJSdCIBycVZt6aUXMgmIIke9GxUb23n3Hz8KK0aGE1ERIrS\nli1+PXlyuPkQL5uAZINzbr9zbg5gzrnpzrk5QWPWA/hak22JTyEiIhKOnh6oqIC3vjXsnAhkNzDa\nUefcemAv/tVMpGFrpLak1cw2Z5k/ERGRjPz61/DLX/reNK+9BidPwu23w3XXwcGDPiB5//v1yqZY\nZDswWjcwO5jZN3pgtA4NjCap6uvro6amJuxsiEiZ+eAH4YknRm/r7IRt23xgUlUFf/In4eRNxspq\nHBIYHhitysz2mdmmYLO+XSaovr4+qqur2bkz8awBfX19rF69murqapYtW5aXfGzatInVq1fn/Nwi\nUvyOHPHByF//Nbz8sm/A+tWv+oBk5UofjDz5JHziE2HnVCImzMBozrmKsPMwEfT39zMwMEB3d3fC\ndDU1NbS0tNDf358wXSY6OztpaWmhpaWFrq6unJ9fRIrf177m17fdBpdd5uemWbTIByYHDsAXvgCX\nXx5uHmW0jF/ZRA2M1kucgdGcc0eccz83s29mmcesBD1+WoHDwN1h5mUimDt3LqdPn2bGjBlJ0+br\nNc38+fOZP38+kyZlXQEoIiXqqadgyRJYuHBk2xvfOPLznXcWPk+SWFkPjOacmwEcCTMPE1EqwYiI\nSK7t3g3Tp/tGqocOwZvfPDbNj34E992nrr7FKJtGrUU/MJqZnQFwzp0KMx8iIpJbZnDhApw759ev\nvw4/+5kPRv7n/4RLL/U9aGL92Z/5RYpPNgFJKgOj9WR6cudcJbAO2GpmJzI9jxTWvn372LZtG845\n9u/fP2rfwMAALS0tnDp1Cucc8+bNi3uOo0ePsmHDBnp7e+nv72fJkiVs3LhxVJq2tja6urqoqamh\nu7ubxsZGmpub81YuESkOP/4xLFsGv/89DA6O3X/ddXC3Xs6XpGwCkg3Ouf1AC8HAaEAdcAewliwG\nRnPONQPrgQrgwXHSVODbhlThA6IKYJ2ZHc3kmmF47rnneO6558bdP23aNG644YaE53jsscc4e/bs\nuPuvuuoqrrrqqozzmI6jR49y+PBhOjo6aGxsHLWvt7eXefPmsW/fPm655RYANm3aNOYc3d3drF+/\nfjiYOXToEAsWLKCvr49du3YB0NLSwubNmxkM/jfq7OyksbGRuro6Fi1alM8iikiIfvhDWLoUZs2C\n//pffS3I1Kn+9Utkectbws6lZMzMMl7wr2WOA0PAYLAMBcuGDM85B5iBH+11EJgzTroe4MGo3+cD\np+KlD861Jc1yWVdXl+XTvffea/jALe5yww03JD3HDTfckPAc9957b17LEKu/v9+cc7Zw4cJR25cs\nWTJmm5mZc87mzZs3/HtdXZ0dPXp0VJq6ujqbNGmS9fX1mZlZY2OjzZw5c8w1V69enfDcIlK6hobM\nqqrMpk83+9Wvws6NJNLV1RX5Dqq3NL7/i25gNDM7BuCc68UHGWM451qBWfjgJXJcZ3DMHuDaTK5d\naKtWreIDH/jAuPunpTAF5Z49e5LWkBRSRcXY3tV9fX3s27ePtrbEbZyPHj1Kb28vLS0tmBnOOcyM\nqqoqnHP09vYya9Ys9u7dO+q4I0fUblmk3D31FJw+Df/8z6oFKVdZD5jrnFsENDISjJzKNBhJwwqg\n18xeitm+C9jonJsTCWyKWS5epyR7pVMMent7cc5RW5u4jXNPTw/OOfbu3cv06dPHTRfpxbNv3z46\nOjpYsGABAKdOqe2ySLl65BG/vummcPMh+ZNxt1/nXIVz7kl8jcQqfFCyCtjhnDvpnPtQjvIYe925\nQCUQb+Stbnx7klUx26vzkRdJTW9vL2ZGb2/iyZ/7+voAH5gkO9+8efM4ffo0W7ZsYfHixTnLq4gU\np0cegZkz4eqrw86J5Es245DswDdi7cQPkFaHb2Bah29sep9z7pasczhWpGtGvD+HI9uG/xR3zi0G\n5gILnHO35iE/kkSkZuTw4cNJ05nZcOPVWA899BAAjY2NzJw5k+XLl+c2oyJSdE6ehH/9Vz9+yI03\ngnPJj5HSlE1AsgA4aGYLzc9j02dmA8G6DR84rMtNNkepDNbxxhyPbBsOSIK8XWRm19rIuClSQJHu\nvXv37uXMmTNj9keGj4+8emlra6Ozs3NUmtWrV1NbW0tfX99wTUrs8SJSfpYt8xPgdXbCH/9x2LmR\nfMomIIk0II3LzBLXz2dO3z5FLBIcRL+eqaiooLW1FYD6+no6Ozs5evQo69atG067fv16nHOsXbsW\n8LUgCxcuZN26dcybN4+qqirmzJlDdbV/+9bR0cGmTZvYsWMH69atwzlHd3c3O3fu5MyZM8P5ULAi\nUprOn4d/+Rc4fhz+8i/h17+G//bfws6V5FM2Ack2oCFJmpmxG5xz92RxTRgZkK0yzr5EtSeSZ52d\nnaxcuXK4R8zmzZuHa0Sam5uHB0yLBBqrV6+mrq6OtrY2Vq1axYwZM9i4cSOtra3U1dXR2dnJvn37\n+OxnP8uGDX6uxoqKCrZv305lZSXbt29nYGCArVu30tzczKlTp+jp6aGnp4dly5YN52P9+vWcOHEi\nxDsjIun62tfgT/8UfvMbeOc74dpr/QR5Ur6cWbxR31M40I/GuhbYDfTFSbIKHzwcjNpWCbSaWdJu\nuc65rfjeNA3RPWacczX4MUj2mNkdcfJ0ENhmZhmP1eecqwe6br755jHdWJuammhqasr01CIikoJV\nq3zbkYMH4Q1vCDs3Mp729nba29tHbRsYGOBHP/oR+O/vxFO/R8kmIDnAOOOERJIwdo4bB5iZJY1z\nxwtIgn2ngJOxgU0wwutG/KR/P0heinGvXQ90dXV1UV9fn+lpREQkQ+95D1x5JcQMOyQloLu7m4aG\nBkgzIMlmHJLt+Maj6QwPfwWQi64R24Fm59wMCybQCywEerIJRkREJFzPPed71fzn/xx2TqSQMg5I\nzGyvc87MbF86xznnXkwx6Zj2J1HXXhd0592BnzsH59wC4Fb8sO8iIlLkfvIT+OpX4f77fXfes2fh\n2DH/qgbgzjvDzZ8UVrZDx6cVjATHjJ1RLUow8NkdQGSWtFbn3B4z2xlznmudc1ucc7uA00ANftz8\nR9LNk4iIFN6990JHBwwM+AarP/85PP+831dZ6ccdkYkj66HjYznnZpnZiUyPNz9b71FSGMMkm4ar\nIiISrtnP2qnbAAAd+0lEQVSzfUDy+uu+hmTxYvjYx+CKK3z7kSlTws6hFFLKAUkwZ001fiTWSvyg\naA9F7d+A73WDc+40sMLMvpnb7BbWmjVrqKioUM8aEZEs/fa38JGP+JFXX3kFXn0V+vth/nz4/vfD\nzp3kQqTHzcBAZtPZpdzLxjk3hO810w2sDGoyIvt2AUvwvWgiIlMPl9wrFPWyERHJrW99Cz70IVi9\nGqqr4bLL/HLLLXo1U24K1cumw8xui94QNC5dig9A1prZ5mD7NqANuG3MWUREpOxt2wY7dsDQkK8Z\nAfgf/wMmTw43X1Kc0g1IYmfRBd/TxYC2SDACYGarnHPHs8mciIh4ZvD73/tGn4ODvt1FZH32LEyd\nCjffHHYuR5hBWxv09vpakUmT/GirCkZkPOkEJBbbWDV4VVOJH/tjfZxjTmeRNxGRsnTqFHz84/DM\nMz6giF4uXBj5+dw539YC/Bf8hQuJz/vEE/DWtya/vhm0tMDTT/vaC/BjfrzrXdmVK9pf/IUPRr7z\nHXjf+3J3Xilf6QQkA8656Wb2Egw3co28qolXcwJRs+6KiEwkJ0/6AOHFF33NxtCQDwSGhnzPkkOH\noKkJLr54ZJk8efTvU6fCpZf6HigAb3wj/OEfjqS76CK/PP+8b4vxu9+lFpA89xxs2gQ33eR7tJw4\nAUuWwOc+B6+95pdXX/UB0JkzvrfLl7/sazmSOX8eTp+GBx7wPWXmJxrPWyRKOgHJHuCQc+4L+J42\nrfhgZK+ZHYpN7JxbQfwJ8EREis7goP8CHhpKbzHzX8Kvvup7j5w548fV+Ku/8l/MEc75L/RJk3ww\n8eUvw8qVucn7FVf49alTY/d9/vOwfbvP47lzfn3+vN/3j//og5Knn4Y/+RP41Kd8AHTppXDJJT7w\nefJJn7a5Gd70puR5mTsXHnvM//yVr8C0aVkXTyaIdAKStfgeNpGZBRzQFTvBHQzXnmxj7Fw2JUXd\nfkUmhsOH4QMfGBmUKxecg/Xr4a67oLZ2pJYjHyqDP/0iAVBzM/zgBz7IOnYMVqzweZgyxde6TJni\ne7pEerf8u38HTz01ku9ox475IOP555MHJBcu+GBkzRp/P9/97tyVUYpfwbr9Dh/g3BL8qKi9sSO1\nOucq8DUptfiZfnvNbHVGOQuRuv2KlB8zeOghaG/3NQmRmoLz5+HXv/Y1BX/1VyO1GKkskVqPyZNH\nahYqKvxy2WX5DUJiVVTA3/6tb7tx9dXQ2AizZsHb3gZ33515Xp5/Hq66yteg/L//N36j1KEh+PSn\n/VDw3/0u/PmfZ1oSKXUFm1zPzMade9HMBvAT3ImIFJVdu3ybjT/+Y19bMHmyrymYMsWPj3HPPT6I\nKFVVVT7Q2hf8mfj1r4+8ysnGH/wBvOMdft6ZL34R1gVjaJ8/P9Lr5/nnfTuUr34V3vAGeOc7s7+u\nTDw5HzpeRCQbZr5BaLxeJ/F6pLz4ol8uXPBfkhcujP45Ugvy8MPw3vfC974Xdgnzo7rav7Lp6fG9\nZXIRjICvAfrZz0ZeQbW2+nv/8stj006Z4mtHqqtzc22ZWBSQiEhR+fu/968e0jVlykitx+TJo3+e\nMgWuucb3LClXFRWwZYuv5cnHeCTHjsGePf46F13kg443vnFkueIKv72Qr6mkvCggEZGc+81vYPfu\n8Ws3IgN6xVt++lPf7uELXxjbDTbectllvlHmRHfXXb5L8NVX52fcj5tu8otIviggEZGc+9rXfG1E\ndXXygCJ6uegiuO4639bj/e8PuxSl5c47/SJSqhSQiEjaXn8dHnwQurt974rBwdHrf/kXX8tx7FjY\nORWRUqGAJAGNQyLidXXBZz4DL73kA44XXvDDnr/lLf61ykUX+caPkfWMGX4cChGZOAo+DslEoHFI\nRLxTp+BHP4L77/fdSVet8kHH1KnwkY/AvHlh51BEik3BxiERkYlj48aRninLl8PWreHmR0TKlwIS\nERnjlVd8F9LOTj/I1Xe/OzI8uYhIPqQwd6OITDTf/76fD+XZZ2HhQt9bJpWZXkVEMqUaEpES9eqr\nfhr7vr6Rae2jp7gfGvIzzz71FPzyl77WIzL+R/Q63jYzPwDWs89qoCsRKQwFJCIl5ve/h//wH+DR\nR/108pdcMtK7JXqK+8jvV17pJ0arqvLpIuN9xK5jt113nYIRESkcBSQiJebQId8N9667YO1a3/VW\nRKTUKSARKRFnz8I//iP8n//j52b5yldKe3ZaEZFoCkgS0MBoUkx27oRPfQquugoeeEDBiIgUFw2M\nlgcaGE2Kzfr1cN99vi3It74Vdm5ERMaX6cBo6sgnUsR6euCjH/UDlNXXw9/+bdg5EhHJD72yEcnS\n0BBcuOC7zMZbIvsi3WkjC/j1yy9Db6/vnvvaayPn/c1voL3d95Jpa4M1a3zvFxGRcqT/3kRScOYM\nPP20DxC+/nU/ydwrr8D58yPBRbaqqkaPhnrJJX7Y9tWr/c8iIuVMAYlMeN//Ptx9t+/FcuGCDzLO\nn/e1GpGBxiKmTvVzurzpTb5R6dSpvtYi0TJ5sl9HxgWJXS65BGpr4dJLw7sHIiJhU0AiE87QEBw5\nAr/4hX+N8tWvwowZ8PGP++BhypSxQcT06T4Ieetb/SsUERHJLQUkUvZ++ENYtmx0DcjgoN83aZKv\n5fjOd+DWW8PNp4jIRKaARIral7/s52EZGvJtOJ59dnTD0Ogl8noldunv969f/u7vRl6h/NEfwbve\npUaiIiLFQv8dS9EaGoJ77oE3vxn+4A9g5ky45Zbx22KMt0ydCg0N8L73hV0iEREZjwKSBDRSq3fu\nHJw+7ddDQ/51R2Q22cjvFy7Aiy/CCy/4yd/6+/18K9HdYSOzyZ45A88/73+Onpk2dqbaSGPSzZvh\nQx8K9x6IiEhiGqk1DzRS64gXXvA9QF5+OfVjLrvMd2F1zr8eecc7Rs8ie9llcPXVfl/sDLWxP19y\nCSxZAtOm5a+MIiKSO5mO1KoaEonrm9+EH/8YTpzwwcg//ZOfQyV6avvIlPeTJvlA44orfA8UdV8V\nEZF0KSCRUYaG4IknYNUqH2i84Q2+huLOO32NhYiISD4oIJlAfvYz+Nd/9T9HD10O8PjjftK2V1/1\nC8D+/bBwYeHzKSIiE48CkjI1NASf+AQcOuSHOY8MdQ4j09ZHajyc869Zli/3r11uvBFqaqCuLpy8\ni4jIxKOApMS89hp87nPwb/820tsldj00BM88A889B297G/zFX8Dll/v2HUuX+m6wIiIixUQBSR4N\nDvrXH9FdXiPr11/33WhfeSW95Ze/hN/+1o+pERnaPLqBaWQ9bx5cey389V+HfRdERESSU0CSBbOR\nMTjOn/fr6OWjH/XtNtI1ZYp/rRJvaWiA3bvhpptyXx4REZGwKCBJ4L77fHfXF17wg36dPetrNi5c\n8K9O+vpG2mWMZ+1aePvbR8/+GhmPY7zAQ8OZi4jIRKOvvgQeeMAP4HXllSNjbETmQpk6Ff7yL/1w\n5tEzxEZmiZ082TcUffvbfQAiIiIi41NAksChQzDBB2oVEREpiElhZ0BERERENSQJaHI9ERGR1Ghy\nvTzQ5HoiIiKZyXRyPb2yERERkdApIBEREZHQKSARERGR0CkgERERkdApIBEREZHQKSARERGR0Ckg\nERERkdApIBEREZHQKSARERGR0CkgERERkdApIBEREZHQKSARERGR0CkgERERkdApIBEREZHQKSAR\nERGR0F0cdgaK2Zo1a6ioqKCpqYmmpqawsyMiIlK02tvbaW9vZ2BgIKPjnZnlOEulzzlXD3R1dXVR\nX18fdnZERERKRnd3Nw0NDQANZtad6nF6ZSMiIiKhU0AiIiIioVNAIiIiIqFTQCIiIiKhU0AiIiIi\noVNAIiIiIqFTQCIiIiKhU0AiIiIioVNAIiIiIqFTQCIiIiKhU0AiIiIioVNAIiIiIqFTQCIiIiKh\nU0AiIiIioVNAIiIiIqFTQCIiIiKhU0AiIiIioVNAIiIiIqFTQCIiIiKhU0AiIiIioVNAIiIiIqFT\nQCIiIiKhuzjsDBSzNWvWUFFRQVNTE01NTWFnR0REpGi1t7fT3t7OwMBARsc7M8txlkqfc64e6Orq\n6qK+vj7s7IiIiJSM7u5uGhoaABrMrDvV4/TKRkREREKngERERERCp4BEREREQqeAREREREKngERE\nRERCp4BEREREQqeAREREREKngERERERCp4BEREREQqeAREREREKngERERERCp4BEREREQqeARERE\nREKngERERERCp4BEREREQqeAREREREKngERERERCp4BEREREQqeAREREREKngERERERCp4BERERE\nQqeAREREREKngERERERCp4BEREREQqeAREREREKngERERERCd3HYGcgn51wF0AocB64AesxsR7i5\nEhERkVhlHZAQBCNmthnAOXfEOXfSzB4KOV8iIiISpWwDEudcLbASqIzavAtoAxSQiIiIFJGibUPi\nnKt0zm10zs3K8BQLADOzM1HbuoEa59yMbPMnIiIiuVOUAYlzrhnoBZoZXcMRnabCObfVObfLObfb\nObffOTc3Kkkt0B9z2KmofSIiIlIkii4gcc7NAbYBR5Ik7QYqzewOM1uGfxXTGRwfcSr+oSIiIlJM\nii4gMbNjwWuW3vHSOOdagVnAiqjjOoNj9gSbeoDqmEOrI9fIYZbLUnt7e9hZCNVELz/oHkz08oPu\ngcpf2PIXXUCSohVAr5m9FLN9F1Ab1JJ0AJUx7UXqSBDoyAj9Q5zY5Qfdg4leftA9UPkVkCQUtBOp\nxL+yidUNOGCVmfUBe4FlUftXAhvznkkRERFJSyl2+50XrOO1D4lttLoS2Oicq8QPjPagmd2X5/yJ\niIhImkoxIIn0uontQRO9rRbAzAaAuwuRKREREclcKQYk8QKRXJsG8PjjjxfgUsVpYGCA7u54b8Um\nholeftA9mOjlB90DlT+z8kd9d05L5zhnZmlfrBCcc1vxjVcbonvFOOfmAweBbWZ2d8wxc4EuoMvM\n3p7FtT8CPJDp8SIiIsJHzewbqSYuxRqSSC+Z2C690duSjWGSzH7go8AJ4GyW5xIREZlIpuGH5tif\nzkElF5CYWZ9zrh+oj7O7HjBgd5bXOAmkHNWJiIjIKD9J94CS6/Yb2I4fbyR2TpqFQI+Z/SCEPImI\niEiGijkgmTneDjNbh391syOyzTm3ALgVWJr/rImIiEguFV2j1qBh6h34ifXAj7i6x8x2xkm7Bd9u\n5DRQA6w1s0cKlVeZWIJnswHYHTOL9IQw0csvIvlVdAFJMXPOzTWzo2HnIyyplr+c7pNzrgL4G2AR\n0MoE+zLOtPzl9AxIZvQMSLrPQMk1ak1F8J9oK1CFH0q+AliXzo1xzu0BFsdsPgjclsvr5EOhyp9O\nukLL0T2ox9fQHTCza/N1nXwoVPmDdGX3DAQTeDYnSFIfGY6gHJ+BdMofpC+6ZyDbz8U5V4ufauQk\nvglBBbDdzPbl8jr5VKh7EKTN/hkws7Jb8DP9Phj1+3z8sPJzUjy+Iki/P2r5fuzx2V6nDMqfUroS\nvQf1wBDw83xepwzKX5bPQJDu58CWmOUwcLLcn4E0y1+Uz0CW5a8N0t4Uta0m+DexqBQ+/wLfg5w8\nA6HerDx9AK3AIDA9ZvsR4MkUz7Ex9obn4zqlXP500pXoPTgd7xwT6BlIWv5yfQbwf+UtH2ffbmBL\nOT8D6ZS/WJ+BbD+X4Pgx6YADwP5i//wLeQ9y+QyE/uDk4UM4Nc5NbA4+nFT/OlgO1OTzOqVc/nTS\nldo9ANbi/wrYle97XcrlL9dnAJiRYN8QcEs5PwPplL9Yn4Ec/BvYHaSbEbP9OKMDkqL8/At5D3L5\nDBRzt9+0Bb0AKoF4g+9349+hrUpyjmaC92RAj3PuuHNucUyarK+TD4UqfzrpCi1Hn81K/AB7vc65\nA865oaB8G3N8nZwrVPmDa5XlM2DjNNoNhhYwC8Y5KtdnINXyB9uK7hnI0edyMEi3J+q89fhXFhtz\neJ28KNQ9CLbl7Bkoq4AEmBesT8XZF9lWm+Qce/AfVCv+/VsNsCfmP+NcXCcf8ln+DRmmK7Ss7kHQ\nCCyy/ySwJPj9ILDWORcZCrksn4E0yg9l+gwksATYW4DrZKtQ5YfifAayLr+Z7cCXdUHUF+xuYEFU\nQFasnz8U7h5ADp+BcgtIKoN1vBmBI9uSfQgnzGynma0337NgFf6vxWbn3K25uk6e5LP8a6PKn3K6\nEGR7DyL7us1ss5mdCcp6N/4viwVB+cr1GUi1/OX8DIxnGfBgAa6TrUKVv1ifgZyU38yW4QffrMV/\n6W6N+SIu1s8fCncPcvoMlFtAEu/mZ8X8gGwLGV3FlfPr5Eihyp9xugLI9h5EJmjsjbNvG758jTm4\nTr4UqvxjlNEzMEbwuqLCzL6Zz+vkSKHKP0aRPAO5LH81/rk3oC2YhT4f18m1Qt2DMbJ5BsotIIn8\nJ1oZZ1+iiDEhM+vE/3UYiSjzcp0cKFT5s0qXZ9neg0THR/bV5uA6+VKo8sdVJs9APPFeV5TrMxBP\nvPLHVQTPQNbld85VOOeO4Btv3o1/BXIaWBH1KqJYP38o3D2IK9NnoFwDkuo4+yLbjmRx7lNRP+fr\nOtkoVPlzkS5fsroHZtaX4PjIuU9le508KlT5k+WhZJ+BcYx5XZGn6+RCocqfLA9hPQO5KH8bMNfM\n7gMwP5BYAzCAfxUxI0fXyZdC3YNkeUjrGSirgCT4z7QfP6hTrHp8ldPuDE9fSfAh5/k6GStU+XOU\nLi9ydA+6xzk+Eu0fL/NnIGn5kxxfDs/AsMjrCqAzn9fJlUKVP4nQnoEclb+BmB4qZnYC35MEoLZY\nP38o3D1Icnz6z0A2fYaLccF3R4rXd/og8OsMz1mJ73EwPZ/XKaXyZ5Ou2O8BfjTDQeDWmO0riRpo\nqFyfgRTKn2icirJ4BmKO2UrMuAv5uE4plr9Yn4Ec/BvYTcyItMH2lcBgsX/+hbwHuXwGQrtZef4g\nniRqUCdgQfDBRA+BW4Ef5Odw1La5wbbd+KqqSLrdwIcyuU45lj/d+1RK9yBq34HgHBVR244Dnynn\nZyDV8k+EZyAqzSngrmyuU47lL/ZnIJvyB2UbBO6J2X4E+EIpfP6FuAe5fgbKdrZf59wW/Luy0/h+\n0WvN7JGYNE8CXWb24eD3yOAu9fjqqL34KqcNNv5gQUmvE4Z8lj+T+xSGTO5BzL4N+DL2Bud50OL0\nMiinZyBm37jln0DPwHx8cFaVqFxl/AyMW/5SeAayKb9zbg5+bA0Y+Tdw0HwvkrSvE5Z83oNcPwNl\nG5CIiIhI6SirRq0iIiJSmhSQiIiISOgUkIiIiEjoFJCIiIhI6BSQiIiISOgUkIiIiEjoFJCIiIhI\n6BSQiIiISOgUkIiUGOdcTdh5EIlHz6ZkQwGJSAE452qcc83Oua1xtp9yzi1P4fitzrlT5GgG0VSv\nXWjOubCmbA+Fc64++GznJElXrJ9Xzp9NmZgUkIhEcc4tds4dd84NBcvheF8AzrkVUekOJ/oyCeYD\naQ2WhpjdlfjJqOJNEz7M/HTirUH6XEnp2oXknFsJzB3vfgafz+6oz+ekc+4e59yMqDQVzrmNUWn2\nO+dujXOu+uBcx4Mv+sGoY4acc/vzWNRIHlYAHcAK/DwhiRTd5wV5ezZlAtJcNiIxnHOz8BNEGdBi\nZpvHSbcYP7HULDN7KYXzDuEnsHp7zPYZqU5ENd45UjiuBoa/PDK6diEEtSNzgb1mdkeCdKfwX85L\n4k14GKRpBlaa2bUx2yuAncBiYCvQZmYngn1zgD34icI6zOy2rAuVhHNuI9AMNJrZoSRpi+rzipbp\nsykSoRoSkRjBl1Mb4IBVCZI24me1TBqMJLleIb5gDuK/wMO4dkqcc3PxU907YEl0rUccp4L1QII0\n3fjAMt72RcAKM/tEJBgBMLNjQQBzlOQ1FrlyMtWExfR5ieSaAhKR+DYE69oEr2OW4WtIippzbg9+\n2vFitwrYyMg9XZmDc/ZH/+Kca8Xfiy4zuy/BcUtzcG0RSYMCEpE4zGwA2Iv/a3197H7n3BLgcOQv\nVufcWufcrqDtwoHgdUFSQZuIA/HaKwRtIbYG7Rx2jXfORNcOXivNDX5tDdpTzEnh2pXBtbcEaQ4E\nbWGi87bSOXfEObfIOTc3+HnIObcrlbLHMT94ZdFK8tqpTDXjX8VtTJQoeLWVtHFtKp971OcYfS/n\njnO+FePdxySf19zgOTkStInZGLVvftBGJtLeaVbUvkhbmw2pnCtOmRI+myJpMTMtWrTEWfB/SQ8B\ng8CMmH1HgFuCn1uBwah984PjFsUcM4QPYiK/z8V/MQ4B+2PS1uJfS9wSta05zjmSXju4xiBwU4rX\nrg+uHZ1+RZB2Q9TxB4Lz7ge2ALcCu4J096R5r1dEzh11fweBW8dJfzzR/qh7sSvOvRnEt/vJ9vlI\n5d7XBnl9c9S2U9HPVNTnugtfMzcnuJ/D9zGFz2t/1O+3Rs4XtW1R7LZge0XM85TKuVJ6NrVoSXcJ\nPQNatBTzEvWlG/1lWQucjEnzYtTvFcF/zltizjXmP+yotLFfMntit8U7RyrXZiQgmZPitbvGufaR\n6PMQE6QE22rinTOF+3wEmB71e9wv0Kj9mQQki4NzDqaTtyTPRrJ7fwT4TMxxzcDrjA5IBoHlcc51\nOM622M/reJzPNnJ/ZsVui5OXRemcK9VnU4uWdBe9shFJLPL6ILo9w0pGtx1Zgg9SIualenLzr4ZG\nCXrELMY3RE0mH9eei2/4GWsb/l5Eer9EGpb2RJ0z0osn5QahwTVPWVTjYDN7CN/+I1nj1ozk6JwJ\n771zrhZf49AZvd3MNpnZxTa2gWpvVJoBfPkrY7aNErz6qcW/jjsQvJI7AJwOzhedv23BMYuitt0R\n3OuUzpXmsymSlovDzoBIMTOzTudcL1DjnFtuZjvxAcmsqDSRdiSLgQX4cSUg814atfh2DvF6iMTm\nL9fXTjTGxZEU0mRiHdDgnIvubeIY6RW0Eojb9TpN0fezFjiWzclSuPf1+M+xf+zRORN5VpZY8t5e\n2/EB9nrgoSC4OJzOuYJ2RCk9myLpUg2JSHKRWpKW4MvncPRft8652mD8jCozu9vM9mV5vdrgerXJ\nEubh2tF5iNUfs86VW81sZsxSja9xGK9xa+QLMdFgXJWM1OJgZkej9qVckzSeFO59pGdT0s8xC5Fz\n1yVLaCMNteuDYGQVQa1JGudK+dkUSZcCEpEkzGwH/ku4FtiBH0wr2kF8m5KdObpk5Ms2lQGmcn3t\nyKuaeLUgkS//n+foWtEjlY4RBBCRVwWxI61G7lGiL8Zaol4nBbaTux48ye59d3CtuF2IE3QnT0cv\no1+jxV5jccymDYyUf66ZRdcSpXKudJ5NkbQoIBFJTeQ/crOokUGDvzRHjfHhnMt2CO3Iq5Hx2k9U\nZnjtpK9xgjYg3cQff6UR355gR7LzpKGFsQFetEjX65aY7XtI8OUZuCM4PvZ6/fhagnG7qga1H1sS\n7E/l3kc+x5WxgYFzbje5ee0RCebWRnfLDq6xlZiALCrIW4tvlJvuuVJ6NkUyoYBEJDWRRqzbYrZH\nXgkscH7yvBX4Xi2G/9Jb7pybEfVlNeo/7Kjtw3/pB1XrkS/g7mAciblR40HUBuNGDKVybfwXiQOW\nBudZNN61A0vxX9rDgUeQdiW+J0jkddXM2DKNV854nHMLgmv3JUgWaTy5IHrsDjPrZOT1w+7IvmB8\njAXBq5R2ixqFNThuAF/70wtExg4ZPm9wf5rxXXBbE+Qr6ece/BypidkTjOux2/lh7/dH3cfZxH8N\nUhm9LcGzEsnnwaA8G4Pyn46pAYnYFuRtVGCZyrlSfTbz0RBZJoCwu/lo0VIqC35siFlxti/HD//9\nJCPjRmwMtm1g9Jgdg8G2Wfhuqbujtt9D1HgnwXmfZGSsj1nB759hpAtmwmtHnetwsG1L8Huya8/A\nfykfDsq9hajuoMHxke6gT+K76VbgazuGy5ngXi5mZDyO/cR0NY1KcyTqfE8ydmyXRcHxJ4M0J4Pf\nbxnv2jH3N96xy5Mdm+a9XxTcx8FgHT1+x4qo6x/Gj/sRex+3pPB53RP1rDwJfChBviuSfDZJz5XK\ns6lFS7qLJtcTERGR0OmVjYiIiIROAYmIiIiETgGJiIiIhE4BiYiIiIROAYmIiIiETgGJiIiIhE4B\niYiIiIROAYmIiIiETgGJiIiIhE4BiYiIiIROAYmIiIiETgGJiIiIhE4BiYiIiITu/wOeyrUzoDr7\nJQAAAABJRU5ErkJggg==\n", "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -150,7 +150,7 @@ "if plot_ratio: #plot ratio of curves\n", " plt.semilogy(mm,tt/ttp,label=\"measured\")\n", " plt.axhline(100,color=\"k\",linestyle=\"--\",label=\"ideal\")\n", - " plt.ylim([10,200])\n", + " plt.ylim([1,200])\n", " plt.xlabel('Validation AUC achieved',size=15)\n", " plt.ylabel('Speedup',size=fontsize)\n", "else: #plot both curves\n", @@ -162,11 +162,12 @@ " plt.ylabel('T [seconds]',size=fontsize)\n", " plt.xlabel('Validation AUC achieved',size=15)\n", "\n", - "plt.xlim([0.8,0.861])\n", + "plt.xlim([0.5,0.861])\n", "plt.legend(frameon=False,loc=\"center left\",fontsize=fontsize)\n", "plt.setp(plt.gca().get_yticklabels(),fontsize=fontsize)\n", "plt.setp(plt.gca().get_xticklabels(),fontsize=fontsize)\n", "\n", + "plt.savefig(\"speedup.png\",bbox_inches=\"tight\")\n", "plt.show()\n" ] }, From e61732d9acadd5e2bf333954b48d4ac85f1b4727 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Wed, 10 Jan 2018 21:03:32 -0500 Subject: [PATCH 007/272] changed bleed in to sample until num disruptive shots have been sampled --- plasma/preprocessor/preprocess.py | 53 ++++++++++++++++++++----------- plasma/primitives/shots.py | 4 +++ 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/plasma/preprocessor/preprocess.py b/plasma/preprocessor/preprocess.py index f6dfde53..d8600bd6 100644 --- a/plasma/preprocessor/preprocess.py +++ b/plasma/preprocessor/preprocess.py @@ -150,28 +150,43 @@ def apply_bleed_in(conf,shot_list_train,shot_list_validate,shot_list_test): num = conf['data']['bleed_in'] new_shots = [] if num > 0: - print('applying bleed in with {} shots\n'.format(num)) + print('applying bleed in with {} disruptive shots\n'.format(num)) num_total = len(shot_list_test) num_d = shot_list_test.num_disruptive() num_nd = num_total - num_d - if num_d > 0: - for i in range(num): - s = shot_list_test.sample_single_class(True) - shot_list_train.append(s) - shot_list_validate.append(s) - if conf['data']['bleed_in_remove_from_test']: - shot_list_test.remove(s) - else: - print('No disruptive shots in test set, omitting bleed in') - if num_nd > 0: - for i in range(num): - s = shot_list_test.sample_single_class(False) - shot_list_train.append(s) - shot_list_validate.append(s) - if conf['data']['bleed_in_remove_from_test']: - shot_list_test.remove(s) - else: - print('No nondisruptive shots in test set, omitting bleed in') + assert(num_d >= num), "Not enough disruptive shots {} to cover bleed in {}".format(num_d,num) + num_sampled_d = 0 + num_sampled_nd = 0 + while num_sampled_d < num: + s = shot_list_test.sample_shot() + shot_list_train.append(s) + shot_list_validate.append(s) + if conf['data']['bleed_in_remove_from_test']: + shot_list_test.remove(s) + if s.is_disruptive: + num_sampled_d += 1 + else: + num_sampled_nd += 1 + print("Sampled {} shots, {} disruptive, {} nondisruptive".format(num_sampled_nd+num_sampled_d,num_sampled_d,num_sampled_nd)) + assert(num_sampled_d == num) + # if num_d > 0: + # for i in range(num): + # s = shot_list_test.sample_single_class(True) + # shot_list_train.append(s) + # shot_list_validate.append(s) + # if conf['data']['bleed_in_remove_from_test']: + # shot_list_test.remove(s) + # else: + # print('No disruptive shots in test set, omitting bleed in') + # if num_nd > 0: + # for i in range(num): + # s = shot_list_test.sample_single_class(False) + # shot_list_train.append(s) + # shot_list_validate.append(s) + # if conf['data']['bleed_in_remove_from_test']: + # shot_list_test.remove(s) + # else: + # print('No nondisruptive shots in test set, omitting bleed in') return shot_list_train,shot_list_validate,shot_list_test diff --git a/plasma/primitives/shots.py b/plasma/primitives/shots.py index 09685093..e2865d72 100644 --- a/plasma/primitives/shots.py +++ b/plasma/primitives/shots.py @@ -166,6 +166,10 @@ def sample_weighted_given_arr(self,p): idx = np.random.choice(range(len(self.shots)),p=p) return self.shots[idx] + def sample_shot(self): + idx = np.random.choice(range(len(self.shots))) + return self.shots[idx] + def sample_weighted(self): p = np.array([shot.weight for shot in self.shots]) return self.sample_weighted_given_arr(p) From 0f3d9045b09f1d23c484fd87c89087509d551cb1 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Wed, 10 Jan 2018 21:17:33 -0500 Subject: [PATCH 008/272] bleed in option of adding multiple times --- examples/conf.yaml | 1 + plasma/preprocessor/preprocess.py | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/examples/conf.yaml b/examples/conf.yaml index a71947fb..d86e4118 100644 --- a/examples/conf.yaml +++ b/examples/conf.yaml @@ -18,6 +18,7 @@ paths: data: bleed_in: 0 #how many shots from the test sit to use in training? bleed_in_remove_from_test: True + bleed_in_equalize_sets: True signal_to_augment: None #'plasma current' #or None augmentation_mode: 'none' augment_during_training: False diff --git a/plasma/preprocessor/preprocess.py b/plasma/preprocessor/preprocess.py index d8600bd6..364c96a0 100644 --- a/plasma/preprocessor/preprocess.py +++ b/plasma/preprocessor/preprocess.py @@ -150,6 +150,7 @@ def apply_bleed_in(conf,shot_list_train,shot_list_validate,shot_list_test): num = conf['data']['bleed_in'] new_shots = [] if num > 0: + shot_list_bleed = ShotList() print('applying bleed in with {} disruptive shots\n'.format(num)) num_total = len(shot_list_test) num_d = shot_list_test.num_disruptive() @@ -159,8 +160,7 @@ def apply_bleed_in(conf,shot_list_train,shot_list_validate,shot_list_test): num_sampled_nd = 0 while num_sampled_d < num: s = shot_list_test.sample_shot() - shot_list_train.append(s) - shot_list_validate.append(s) + shot_list_bleed.append(s) if conf['data']['bleed_in_remove_from_test']: shot_list_test.remove(s) if s.is_disruptive: @@ -168,7 +168,20 @@ def apply_bleed_in(conf,shot_list_train,shot_list_validate,shot_list_test): else: num_sampled_nd += 1 print("Sampled {} shots, {} disruptive, {} nondisruptive".format(num_sampled_nd+num_sampled_d,num_sampled_d,num_sampled_nd)) + print("Before adding: training shots: {} validation shots: {}".format(len(shot_list_train,shot_list_validate))) assert(num_sampled_d == num) + num_to_sample = len(shot_list_bleed) + if conf['data']['bleed_in_equalize_sets']:#add bleed-in shots to training and validation set repeatedly + for shot_list_curr in [shot_list_train,shot_list_validate]: + for i in range(len(shot_list_curr)): + s = shot_list_bleed.sample_shot() + shot_list_curr.append(s) + else: #add each shot only once + for s in shot_list_bleed: + shot_list_train.append(s) + shot_list_validate.append(s) + print("After adding: training shots: {} validation shots: {}".format(len(shot_list_train,shot_list_validate))) + print("Added bleed in shots to training and validation sets") # if num_d > 0: # for i in range(num): # s = shot_list_test.sample_single_class(True) From ec4ed030434c3455b45ba41c462c16b55b604ed8 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Wed, 10 Jan 2018 21:21:23 -0500 Subject: [PATCH 009/272] bleed in --- plasma/preprocessor/preprocess.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plasma/preprocessor/preprocess.py b/plasma/preprocessor/preprocess.py index 364c96a0..be3808c5 100644 --- a/plasma/preprocessor/preprocess.py +++ b/plasma/preprocessor/preprocess.py @@ -168,7 +168,7 @@ def apply_bleed_in(conf,shot_list_train,shot_list_validate,shot_list_test): else: num_sampled_nd += 1 print("Sampled {} shots, {} disruptive, {} nondisruptive".format(num_sampled_nd+num_sampled_d,num_sampled_d,num_sampled_nd)) - print("Before adding: training shots: {} validation shots: {}".format(len(shot_list_train,shot_list_validate))) + print("Before adding: training shots: {} validation shots: {}".format(len(shot_list_train),len(shot_list_validate))) assert(num_sampled_d == num) num_to_sample = len(shot_list_bleed) if conf['data']['bleed_in_equalize_sets']:#add bleed-in shots to training and validation set repeatedly @@ -180,7 +180,7 @@ def apply_bleed_in(conf,shot_list_train,shot_list_validate,shot_list_test): for s in shot_list_bleed: shot_list_train.append(s) shot_list_validate.append(s) - print("After adding: training shots: {} validation shots: {}".format(len(shot_list_train,shot_list_validate))) + print("After adding: training shots: {} validation shots: {}".format(len(shot_list_train),len(shot_list_validate))) print("Added bleed in shots to training and validation sets") # if num_d > 0: # for i in range(num): From 7d6fcb1cfbf368a8c255b0dbab4b1f0ec4d2a48c Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Fri, 12 Jan 2018 15:06:37 -0500 Subject: [PATCH 010/272] Option to apply batch norm between (most) linear and non-linear layers, bidirectional wrapper for lstm --- plasma/models/builder.py | 85 +++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 35 deletions(-) diff --git a/plasma/models/builder.py b/plasma/models/builder.py index 0b223356..da3274d7 100644 --- a/plasma/models/builder.py +++ b/plasma/models/builder.py @@ -2,8 +2,8 @@ import keras from keras.models import Sequential, Model from keras.layers import Input -from keras.layers.core import Dense, Activation, Dropout, Lambda, Reshape, Flatten, Permute -from keras.layers.recurrent import LSTM, SimpleRNN +from keras.layers.core import Dense, Activation, Dropout, Lambda, Reshape, Flatten, Permute, RepeatVector +from keras.layers import LSTM, SimpleRNN, Bidirectional, BatchNormalization from keras.layers.convolutional import Convolution1D from keras.layers.pooling import MaxPooling1D from keras.utils.data_utils import get_file @@ -12,7 +12,6 @@ from keras.callbacks import Callback from keras.regularizers import l1,l2,l1_l2 - import keras.backend as K import dill @@ -73,10 +72,12 @@ def get_0D_1D_indices(self): def build_model(self,predict,custom_batch_size=None): conf = self.conf model_conf = conf['model'] + use_bidirectional = model_conf['use_bidirectional'] rnn_size = model_conf['rnn_size'] rnn_type = model_conf['rnn_type'] regularization = model_conf['regularization'] dense_regularization = model_conf['dense_regularization'] + use_batch_norm = model_conf['use_batch_norm'] dropout_prob = model_conf['dropout_prob'] length = model_conf['length'] @@ -118,6 +119,7 @@ def build_model(self,predict,custom_batch_size=None): batch_shape_non_temporal=(batch_size,num_signals) indices_0d,indices_1d,num_0D,num_1D = self.get_0D_1D_indices() + def slicer(x,indices): return x[:,indices] @@ -130,10 +132,6 @@ def slicer_output_shape(input_shape,indices): pre_rnn_input = Input(shape=(num_signals,)) if num_1D > 0: - #pre_rnn_0D = Lambda(lambda x: slicer(x,indices_0d),lambda s: slicer_output_shape(s,indices_0d))(pre_rnn_input) - #pre_rnn_1D = Lambda(lambda x: slicer(x,indices_1d),lambda s: slicer_output_shape(s,indices_1d))(pre_rnn_input) - #idx0D_tensor = K.variable(indices_0d) - #idx1D_tensor = K.variable(indices_1d) pre_rnn_1D = Lambda(lambda x: x[:,len(indices_0d):],output_shape=(len(indices_1d),))(pre_rnn_input) pre_rnn_0D = Lambda(lambda x: x[:,:len(indices_0d)],output_shape=(len(indices_0d),))(pre_rnn_input)# slicer(x,indices_0d),lambda s: slicer_output_shape(s,indices_0d))(pre_rnn_input) pre_rnn_1D = Reshape((num_1D,len(indices_1d)//num_1D)) (pre_rnn_1D) @@ -141,12 +139,40 @@ def slicer_output_shape(input_shape,indices): for i in range(model_conf['num_conv_layers']): div_fac = 2**i - pre_rnn_1D = Convolution1D(num_conv_filters//div_fac,size_conv_filters,padding='valid',activation='relu') (pre_rnn_1D) - pre_rnn_1D = Convolution1D(num_conv_filters//div_fac,1,padding='valid',activation='relu') (pre_rnn_1D) + '''The first conv layer learns `num_conv_filters//div_fac` filters (aka kernels), + each of size `(size_conv_filters, num1D)``. Its output will have shape + (None, len(indices_1d)//num_1D - size_conv_filters + 1, num_conv_filters//div_fac), + i.e., for each position in the input spatial series (direction along radius), + the activation of each filter at that position.''' + + '''For i=1 first conv layer would get: + (None, (len(indices_1d)//num_1D - size_conv_filters + 1)/pool_size-size_conv_filters+1,num_conv_filters//div_fac)''' + pre_rnn_1D = Convolution1D(num_conv_filters//div_fac,size_conv_filters,padding='valid') (pre_rnn_1D) + if use_batch_norm: pre_rnn_1D = BatchNormalization()(pre_rnn_1D) + pre_rnn_1D = Activation('relu')(pre_rnn_1D) + + '''The output of the second conv layer will have shape + (None, len(indices_1d)//num_1D - size_conv_filters + 1, num_conv_filters//div_fac), + i.e., for each position in the input spatial series (direction along radius), + the activation of each filter at that position. + + for i=1 second layer would output + (None, (len(indices_1d)//num_1D - size_conv_filters + 1)/pool_size-size_conv_filters+1,num_conv_filters//div_fac)''' + pre_rnn_1D = Convolution1D(num_conv_filters//div_fac,1,padding='valid') (pre_rnn_1D) + if use_batch_norm: pre_rnn_1D = BatchNormalization()(pre_rnn_1D) + pre_rnn_1D = Activation('relu')(pre_rnn_1D) + '''Outputs (None, (len(indices_1d)//num_1D - size_conv_filters + 1)/pool_size, num_conv_filters//div_fac) + + for i=1 pooling layer would output: + (None,((len(indices_1d)//num_1D- size_conv_filters + 1)/pool_size-size_conv_filters+1)/pool_size,num_conv_filters//div_fac)''' pre_rnn_1D = MaxPooling1D(pool_size) (pre_rnn_1D) pre_rnn_1D = Flatten() (pre_rnn_1D) - pre_rnn_1D = Dense(dense_size,activation='relu',kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn_1D) - pre_rnn_1D = Dense(dense_size//4,activation='relu',kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn_1D) + pre_rnn_1D = Dense(dense_size,kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn_1D) + if use_batch_norm: pre_rnn_1D = BatchNormalization()(pre_rnn_1D) + pre_rnn_1D = Activation('relu')(pre_rnn_1D) + pre_rnn_1D = Dense(dense_size//4,kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn_1D) + if use_batch_norm: pre_rnn_1D = BatchNormalization()(pre_rnn_1D) + pre_rnn_1D = Activation('relu')(pre_rnn_1D) pre_rnn = Concatenate() ([pre_rnn_0D,pre_rnn_1D]) else: pre_rnn = pre_rnn_input @@ -157,33 +183,22 @@ def slicer_output_shape(input_shape,indices): pre_rnn = Dense(dense_size//4,activation='relu',kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn) pre_rnn_model = Model(inputs = pre_rnn_input,outputs=pre_rnn) + pre_rnn_model.summary() x_input = Input(batch_shape = batch_input_shape) x_in = TimeDistributed(pre_rnn_model) (x_input) -# x_input = Input(batch_shape=batch_input_shape) -# if num_1D > 0: -# x_0D = Lambda(lambda x: slicer(x,indices_0d),lambda s: slicer_output_shape(s,indices_0d)) (x_input) -# x_1D = Lambda(lambda x: slicer(x,indices_1d),lambda s: slicer_output_shape(s,indices_1d)) (x_input) -# -# x_1D = TimeDistributed(Reshape((num_1D,len(indices_1d)/num_1D))) (x_1D) -# for i in range(model_conf['num_conv_layers']): -# x_1D = TimeDistributed(Conv1D(num_conv_filters,size_conv_filters,activation='relu')) (x_1D) -# x_1D = TimeDistributed(MaxPooling1D(pool_size)) (x_1D) -# x_1D = TimeDistributed(Flatten()) (x_1D) -# x_in = TimeDistributed(Concatenate) ([x_0D,x_1D]) -# -# else: -# x_in = x_input - #x_in = TimeDistributed(Dense(100,activation='tanh')) (x_in) - #x_in = TimeDistributed(Dense(30,activation='tanh')) (x_in) - #x_in = TimeDistributed(Dense(2*(num_0D+num_1D)),activation='relu') (x_in) - # x = TimeDistributed(Dense(2*(num_0D+num_1D))) - # model.add(TimeDistributed(Dense(num_density_channels,bias=True),batch_input_shape=batch_input_shape)) - for _ in range(model_conf['rnn_layers']): - x_in = rnn_model(rnn_size, return_sequences=return_sequences,#batch_input_shape=batch_input_shape, - stateful=stateful,kernel_regularizer=l2(regularization),recurrent_regularizer=l2(regularization), - bias_regularizer=l2(regularization),dropout=dropout_prob,recurrent_dropout=dropout_prob) (x_in) - x_in = Dropout(dropout_prob) (x_in) + if use_bidirectional: + for _ in range(model_conf['rnn_layers']): + x_in = Bidirectional(rnn_model(rnn_size, return_sequences=return_sequences, + stateful=stateful,kernel_regularizer=l2(regularization),recurrent_regularizer=l2(regularization), + bias_regularizer=l2(regularization),dropout=dropout_prob,recurrent_dropout=dropout_prob)) (x_in) + x_in = Dropout(dropout_prob) (x_in) + else: + for _ in range(model_conf['rnn_layers']): + x_in = rnn_model(rnn_size, return_sequences=return_sequences,#batch_input_shape=batch_input_shape, + stateful=stateful,kernel_regularizer=l2(regularization),recurrent_regularizer=l2(regularization), + bias_regularizer=l2(regularization),dropout=dropout_prob,recurrent_dropout=dropout_prob) (x_in) + x_in = Dropout(dropout_prob) (x_in) if return_sequences: #x_out = TimeDistributed(Dense(100,activation='tanh')) (x_in) x_out = TimeDistributed(Dense(1,activation=output_activation)) (x_in) From e69eb5ccf19c769eff7b350152d230654f9e2ae9 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Fri, 12 Jan 2018 15:06:51 -0500 Subject: [PATCH 011/272] Batch norm and bidirectional flags --- examples/conf.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/conf.yaml b/examples/conf.yaml index d86e4118..f599ff51 100644 --- a/examples/conf.yaml +++ b/examples/conf.yaml @@ -52,6 +52,8 @@ data: floatx: 'float32' model: + use_bidirectional: false + use_batch_norm: false shallow: False shallow_model: num_samples: 1000000 #1000000 #the number of samples to use for training From 40bfabcfca9e77d53e90304bbdfacde8f84ef8d1 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Fri, 12 Jan 2018 15:07:04 -0500 Subject: [PATCH 012/272] Any Keras after 2.0.8 is good --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 59e70b94..ab3be99f 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ download_url = "https://github.com/PPPLDeepLearning/plasma-python", #license = "Apache Software License v2", test_suite = "tests", - install_requires = ['keras==2.0.6','pathos','matplotlib==2.0.2','hyperopt','mpi4py','xgboost'], + install_requires = ['keras>2.0.8','pathos','matplotlib==2.0.2','hyperopt','mpi4py','xgboost'], tests_require = [], classifiers = ["Development Status :: 3 - Alpha", "Environment :: Console", From 2e58e488723caabc2e0f1747573363f0a2421f7f Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 16 Jan 2018 12:05:08 -0500 Subject: [PATCH 013/272] Make etemp and edens not available for JET until understood missing data beyond run 73337 --- data/signals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/signals.py b/data/signals.py index 13264366..b7af6c98 100644 --- a/data/signals.py +++ b/data/signals.py @@ -137,8 +137,8 @@ def fetch_nstx_data(signal_path,shot_num,c): profile_num_channels = 64 #ZIPFIT comes from actual measurements -etemp_profile = ProfileSignal("Electron temperature profile",["ppf/hrts/te","ZIPFIT01/PROFILES.ETEMPFIT"],[jet,d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) -edens_profile = ProfileSignal("Electron density profile",["ppf/hrts/ne","ZIPFIT01/PROFILES.EDENSFIT"],[jet,d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) +etemp_profile = ProfileSignal("Electron temperature profile",["ppf/hrts/te","ZIPFIT01/PROFILES.ETEMPFIT"],[d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) +edens_profile = ProfileSignal("Electron density profile",["ppf/hrts/ne","ZIPFIT01/PROFILES.EDENSFIT"],[d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) itemp_profile = ProfileSignal("Ion temperature profile",["ZIPFIT01/PROFILES.ITEMPFIT"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) zdens_profile = ProfileSignal("Impurity density profile",["ZIPFIT01/PROFILES.ZDENSFIT"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) trot_profile = ProfileSignal("Rotation profile",["ZIPFIT01/PROFILES.TROTFIT"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) From da6021cd25850b61f49a90db4c06499e299122c1 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Tue, 16 Jan 2018 14:14:49 -0500 Subject: [PATCH 014/272] Keep only paths profile is defined on --- data/signals.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/data/signals.py b/data/signals.py index b7af6c98..3c3e6313 100644 --- a/data/signals.py +++ b/data/signals.py @@ -137,8 +137,11 @@ def fetch_nstx_data(signal_path,shot_num,c): profile_num_channels = 64 #ZIPFIT comes from actual measurements -etemp_profile = ProfileSignal("Electron temperature profile",["ppf/hrts/te","ZIPFIT01/PROFILES.ETEMPFIT"],[d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) -edens_profile = ProfileSignal("Electron density profile",["ppf/hrts/ne","ZIPFIT01/PROFILES.EDENSFIT"],[d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) +#etemp_profile = ProfileSignal("Electron temperature profile",["ppf/hrts/te","ZIPFIT01/PROFILES.ETEMPFIT"],[d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) +#edens_profile = ProfileSignal("Electron density profile",["ppf/hrts/ne","ZIPFIT01/PROFILES.EDENSFIT"],[d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) + +etemp_profile = ProfileSignal("Electron temperature profile",["ZIPFIT01/PROFILES.ETEMPFIT"],[d3d],mapping_paths=[None],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) +edens_profile = ProfileSignal("Electron density profile",["ZIPFIT01/PROFILES.EDENSFIT"],[d3d],mapping_paths=[None],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) itemp_profile = ProfileSignal("Ion temperature profile",["ZIPFIT01/PROFILES.ITEMPFIT"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) zdens_profile = ProfileSignal("Impurity density profile",["ZIPFIT01/PROFILES.ZDENSFIT"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) trot_profile = ProfileSignal("Rotation profile",["ZIPFIT01/PROFILES.TROTFIT"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) From aa3430d54034125b821871d7ed16e9e6ad2487e0 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Tue, 16 Jan 2018 14:18:42 -0500 Subject: [PATCH 015/272] Update signals.py --- data/signals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/signals.py b/data/signals.py index 3c3e6313..68fde894 100644 --- a/data/signals.py +++ b/data/signals.py @@ -137,8 +137,8 @@ def fetch_nstx_data(signal_path,shot_num,c): profile_num_channels = 64 #ZIPFIT comes from actual measurements -#etemp_profile = ProfileSignal("Electron temperature profile",["ppf/hrts/te","ZIPFIT01/PROFILES.ETEMPFIT"],[d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) -#edens_profile = ProfileSignal("Electron density profile",["ppf/hrts/ne","ZIPFIT01/PROFILES.EDENSFIT"],[d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) +#etemp_profile = ProfileSignal("Electron temperature profile",["ppf/hrts/te","ZIPFIT01/PROFILES.ETEMPFIT"],[jet,d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) +#edens_profile = ProfileSignal("Electron density profile",["ppf/hrts/ne","ZIPFIT01/PROFILES.EDENSFIT"],[jet,d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) etemp_profile = ProfileSignal("Electron temperature profile",["ZIPFIT01/PROFILES.ETEMPFIT"],[d3d],mapping_paths=[None],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) edens_profile = ProfileSignal("Electron density profile",["ZIPFIT01/PROFILES.EDENSFIT"],[d3d],mapping_paths=[None],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) From dae8bf0c58c6b994ab305b7bf317ead95f6e8b15 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Wed, 24 Jan 2018 02:48:00 -0500 Subject: [PATCH 016/272] added option for partial multiplication factor in bleed in --- examples/conf.yaml | 15 ++++++++------- plasma/preprocessor/preprocess.py | 13 +++++++++++-- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/examples/conf.yaml b/examples/conf.yaml index d86e4118..1d48aa12 100644 --- a/examples/conf.yaml +++ b/examples/conf.yaml @@ -10,15 +10,16 @@ paths: signal_prepath: '/signal_data/' #/signal_data/jet/ shot_list_dir: '/shot_lists/' tensorboard_save_path: '/Graph/' - data: jet_data #'d3d_to_jet_data' #'d3d_to_jet_data' # 'jet_to_d3d_data' #jet_data + data: jet_to_d3d_data #'d3d_to_jet_data' #'d3d_to_jet_data' # 'jet_to_d3d_data' #jet_data specific_signals: [] #['q95','li','ip','betan','energy','lm','pradcore','pradedge','pradtot','pin','torquein','tmamp1','tmamp2','tmfreq1','tmfreq2','pechin','energydt','ipdirect','etemp_profile','edens_profile'] #if left empty will use all valid signals defined on a machine. Only use if need a custom set executable: "mpi_learn.py" shallow_executable: "learn.py" data: - bleed_in: 0 #how many shots from the test sit to use in training? + bleed_in: 5 #how many shots from the test sit to use in training? + bleed_in_repeat_fac: 10 bleed_in_remove_from_test: True - bleed_in_equalize_sets: True + bleed_in_equalize_sets: False signal_to_augment: None #'plasma current' #or None augmentation_mode: 'none' augment_during_training: False @@ -52,10 +53,10 @@ data: floatx: 'float32' model: - shallow: False + shallow: True shallow_model: num_samples: 1000000 #1000000 #the number of samples to use for training - type: "mlp" #"xgboost" #"xgboost" #"random_forest" "xgboost" + type: "xgboost" #"xgboost" #"xgboost" #"random_forest" "xgboost" n_estimators: 100 #for random forest max_depth: 3 #for random forest and xgboost (def = 3) C: 1.0 #for svm @@ -89,8 +90,8 @@ model: #have not found a difference yet optimizer: 'adam' clipnorm: 10.0 - regularization: 0.0 - dense_regularization: 0.01 + regularization: 0.001 + dense_regularization: 0.001 #1e-4 is too high, 5e-7 is too low. 5e-5 seems best at 256 batch size, full dataset and ~10 epochs, and lr decay of 0.90. 1e-4 also works well if we decay a lot (i.e ~0.7 or more) lr: 0.00002 #0.00001 #0.0005 #for adam plots 0.0000001 #0.00005 #0.00005 #0.00005 lr_decay: 0.97 #0.98 #0.9 diff --git a/plasma/preprocessor/preprocess.py b/plasma/preprocessor/preprocess.py index be3808c5..1d12f6c3 100644 --- a/plasma/preprocessor/preprocess.py +++ b/plasma/preprocessor/preprocess.py @@ -146,7 +146,7 @@ def save_shotlists(self,shot_list_train,shot_list_validate,shot_list_test): def apply_bleed_in(conf,shot_list_train,shot_list_validate,shot_list_test): - np.random.seed(1) + np.random.seed(2) num = conf['data']['bleed_in'] new_shots = [] if num > 0: @@ -170,13 +170,22 @@ def apply_bleed_in(conf,shot_list_train,shot_list_validate,shot_list_test): print("Sampled {} shots, {} disruptive, {} nondisruptive".format(num_sampled_nd+num_sampled_d,num_sampled_d,num_sampled_nd)) print("Before adding: training shots: {} validation shots: {}".format(len(shot_list_train),len(shot_list_validate))) assert(num_sampled_d == num) - num_to_sample = len(shot_list_bleed) if conf['data']['bleed_in_equalize_sets']:#add bleed-in shots to training and validation set repeatedly + print("Applying equalized bleed in") for shot_list_curr in [shot_list_train,shot_list_validate]: for i in range(len(shot_list_curr)): s = shot_list_bleed.sample_shot() shot_list_curr.append(s) + elif conf['data']['bleed_in_repeat_fac'] > 1: + repeat_fac = conf['data']['bleed_in_repeat_fac'] + print("Applying bleed in with repeat factor {}".format(repeat_fac)) + num_to_sample = int(round(repeat_fac*len(shot_list_bleed))) + for i in range(num_to_sample): + s = shot_list_bleed.sample_shot() + shot_list_train.append(s) + shot_list_validate.append(s) else: #add each shot only once + print("Applying bleed in without repetition") for s in shot_list_bleed: shot_list_train.append(s) shot_list_validate.append(s) From f6a770c9e88a36d78fd3568b09d2fa253579dad1 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Wed, 24 Jan 2018 02:48:34 -0500 Subject: [PATCH 017/272] added option for only fully defined signals --- plasma/conf_parser.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index 15bf4dbe..8e7375c7 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -105,6 +105,17 @@ def parameters(input_file): params['paths']['shot_files'] = [jenkins_jet_carbon_wall] params['paths']['shot_files_test'] = [jenkins_jet_iterlike_wall] params['paths']['use_signals_dict'] = jet_signals + elif params['paths']['data'] == 'jet_data_fully_defined': #jet data but with fully defined signals + params['paths']['shot_files'] = [jet_carbon_wall] + params['paths']['shot_files_test'] = [jet_iterlike_wall] + params['paths']['use_signals_dict'] = fully_defined_signals + elif params['paths']['data'] == 'jet_data_fully_defined_0D': #jet data but with fully defined signals + params['paths']['shot_files'] = [jet_carbon_wall] + params['paths']['shot_files_test'] = [jet_iterlike_wall] + params['paths']['use_signals_dict'] = fully_defined_signals_0D + + + elif params['paths']['data'] == 'd3d_data': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] @@ -131,7 +142,14 @@ def parameters(input_file): params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = {'q95':q95,'li':li,'ip':ip,'lm':lm,'betan':betan,'energy':energy,'dens':dens,'pradcore':pradcore,'pradedge':pradedge,'pin':pin,'torquein':torquein,'ipdirect':ipdirect,'iptarget':iptarget,'iperr':iperr, 'etemp_profile':etemp_profile ,'edens_profile':edens_profile} - + elif params['paths']['data'] == 'd3d_data_fully_defined': #jet data but with fully defined signals + params['paths']['shot_files'] = [d3d_full] + params['paths']['shot_files_test'] = [] + params['paths']['use_signals_dict'] = fully_defined_signals + elif params['paths']['data'] == 'd3d_data_fully_defined_0D': #jet data but with fully defined signals + params['paths']['shot_files'] = [d3d_full] + params['paths']['shot_files_test'] = [] + params['paths']['use_signals_dict'] = fully_defined_signals_0D #cross-machine elif params['paths']['data'] == 'jet_to_d3d_data': From a3b081f32e04f05592aba928d61bc072345e1d59 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Wed, 24 Jan 2018 21:13:53 -0500 Subject: [PATCH 018/272] changed extrapolation value in profiles to boundary value, as it can lead to unphysical values if only part of the x regime is defined --- plasma/primitives/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plasma/primitives/data.py b/plasma/primitives/data.py index 1354f506..fb19f466 100644 --- a/plasma/primitives/data.py +++ b/plasma/primitives/data.py @@ -227,7 +227,7 @@ def load_data(self,prepath,shot,dtype='float32'): for i in range(timesteps): _,order = np.unique(mapping[i,:],return_index=True) #make sure the mapping is ordered and unique if sig[i,order].shape[0] > 2: - f = UnivariateSpline(mapping[i,order],sig[i,order],s=0,k=1,ext=0) + f = UnivariateSpline(mapping[i,order],sig[i,order],s=0,k=1,ext=3) #ext = 0 is extrapolation, ext = 3 is boundary value. sig_interp[i,:] = f(remapping) else: print('Signal {}, shot {} has not enough points for linear interpolation. dfitpack.error: (m>k) failed for hidden m: fpcurf0:m=1'.format(self.description,shot.number)) From c3d9a4d8b6662ce1bd51b0c2c9c424059a5b43e4 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Wed, 24 Jan 2018 21:22:36 -0500 Subject: [PATCH 019/272] use full jet data (as opposed to just carbon wall) for cross-machine prediction. This gives more disruptive shots --- plasma/conf_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index 8e7375c7..a131598d 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -153,7 +153,7 @@ def parameters(input_file): #cross-machine elif params['paths']['data'] == 'jet_to_d3d_data': - params['paths']['shot_files'] = [jet_carbon_wall] + params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] params['paths']['use_signals_dict'] = fully_defined_signals elif params['paths']['data'] == 'd3d_to_jet_data': @@ -161,7 +161,7 @@ def parameters(input_file): params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = fully_defined_signals elif params['paths']['data'] == 'jet_to_d3d_data_0D': - params['paths']['shot_files'] = [jet_carbon_wall] + params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] params['paths']['use_signals_dict'] = fully_defined_signals_0D elif params['paths']['data'] == 'd3d_to_jet_data_0D': From 61dbe21a19089aa1d22a2f13da0263837bf3641f Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Thu, 25 Jan 2018 23:01:09 -0500 Subject: [PATCH 020/272] longer batch jobs by default --- plasma/utils/batch_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plasma/utils/batch_jobs.py b/plasma/utils/batch_jobs.py index 56203b97..4ae45732 100644 --- a/plasma/utils/batch_jobs.py +++ b/plasma/utils/batch_jobs.py @@ -112,7 +112,7 @@ def create_slurm_header(num_nodes,use_mpi,idx): assert(num_nodes == 1) lines = [] lines.append('#!/bin/bash\n') - lines.append('#SBATCH -t 06:00:00\n') + lines.append('#SBATCH -t 20:00:00\n') lines.append('#SBATCH -N '+str(num_nodes)+'\n') if use_mpi: lines.append('#SBATCH --ntasks-per-node=4\n') From 68f881dbce982afb4ad18d6e9db4851adb15faee Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Sun, 28 Jan 2018 19:48:30 -0500 Subject: [PATCH 021/272] added jet 1D signals only option --- data/signals.py | 7 ++++++- plasma/conf_parser.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/data/signals.py b/data/signals.py index 13264366..e0b79363 100644 --- a/data/signals.py +++ b/data/signals.py @@ -237,10 +237,15 @@ def fetch_nstx_data(signal_path,shot_num,c): fully_defined_signals = {sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if sig.is_defined_on_machines(all_machines)} fully_defined_signals_0D = {sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if ( sig.is_defined_on_machines(all_machines) and sig.num_channels == 1) } +fully_defined_signals_1D = {sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if ( sig.is_defined_on_machines(all_machines) and sig.num_channels > 1) } + d3d_signals = {sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if sig.is_defined_on_machine(d3d)} +d3d_signals_0D = {sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if (sig.is_defined_on_machine(d3d) and sig.num_channels == 1)} +d3d_signals_1D = {sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if (sig.is_defined_on_machine(d3d) and sig.num_channels > 1)} + jet_signals = {sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if sig.is_defined_on_machine(jet)} jet_signals_0D = {sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if (sig.is_defined_on_machine(jet) and sig.num_channels == 1)} - +jet_signals_1D = {sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if (sig.is_defined_on_machine(jet) and sig.num_channels > 1)} #['pcechpwrf'] #Total ECH Power Not always on! ### 0D EFIT signals ### diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index a131598d..1dea38b3 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -93,6 +93,10 @@ def parameters(input_file): params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = jet_signals_0D + elif params['paths']['data'] == 'jet_data_1D': + params['paths']['shot_files'] = [jet_carbon_wall] + params['paths']['shot_files_test'] = [jet_iterlike_wall] + params['paths']['use_signals_dict'] = jet_signals_1D elif params['paths']['data'] == 'jet_carbon_data': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [] @@ -168,6 +172,14 @@ def parameters(input_file): params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = fully_defined_signals_0D + elif params['paths']['data'] == 'jet_to_d3d_data_1D': + params['paths']['shot_files'] = [jet_full] + params['paths']['shot_files_test'] = [d3d_full] + params['paths']['use_signals_dict'] = fully_defined_signals_1D + elif params['paths']['data'] == 'd3d_to_jet_data_1D': + params['paths']['shot_files'] = [d3d_full] + params['paths']['shot_files_test'] = [jet_iterlike_wall] + params['paths']['use_signals_dict'] = fully_defined_signals_1D From 77e41a5d39a969615b3cb06f7a2ef1ea32e119a3 Mon Sep 17 00:00:00 2001 From: Alexey Svyatkovskiy Date: Tue, 30 Jan 2018 12:55:55 -0500 Subject: [PATCH 022/272] Roll back to Keras 2.0.6. Need to investigate changes to model.stop_training in Keras 2.1.x --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ab3be99f..59e70b94 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ download_url = "https://github.com/PPPLDeepLearning/plasma-python", #license = "Apache Software License v2", test_suite = "tests", - install_requires = ['keras>2.0.8','pathos','matplotlib==2.0.2','hyperopt','mpi4py','xgboost'], + install_requires = ['keras==2.0.6','pathos','matplotlib==2.0.2','hyperopt','mpi4py','xgboost'], tests_require = [], classifiers = ["Development Status :: 3 - Alpha", "Environment :: Console", From e22b32ebfba888f99ebb7d5328b43aa1be153152 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Thu, 1 Feb 2018 17:25:12 -0500 Subject: [PATCH 023/272] higher numbers of conv filters in tune hyperparams --- examples/tune_hyperparams.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/examples/tune_hyperparams.py b/examples/tune_hyperparams.py index e8efd61e..56b8eb69 100644 --- a/examples/tune_hyperparams.py +++ b/examples/tune_hyperparams.py @@ -7,8 +7,8 @@ tunables = [] shallow = False -num_nodes = 2 -num_trials = 50 +num_nodes = 1 +num_trials = 20 t_warn = CategoricalHyperparam(['data','T_warning'],[0.256,1.024,10.024]) cut_ends = CategoricalHyperparam(['data','cut_shot_ends'],[False,True]) @@ -34,14 +34,20 @@ lr_decay = CategoricalHyperparam(['model','lr_decay'],[0.97,0.985,1.0]) fac = CategoricalHyperparam(['data','positive_example_penalty'],[1.0,4.0,16.0]) target = CategoricalHyperparam(['target'],['maxhinge','hinge','ttdinv','ttd']) - batch_size = CategoricalHyperparam(['training','batch_size'],[64,256,1024]) - dropout_prob = CategoricalHyperparam(['model','dropout_prob'],[0.1,0.3,0.5]) - conv_filters = CategoricalHyperparam(['model','num_conv_filters'],[5,10]) + #target = CategoricalHyperparam(['target'],['hinge','ttdinv','ttd']) + batch_size = CategoricalHyperparam(['training','batch_size'],[128,256]) + dropout_prob = CategoricalHyperparam(['model','dropout_prob'],[0.01,0.05,0.1]) + conv_filters = CategoricalHyperparam(['model','num_conv_filters'],[128,256]) conv_layers = IntegerHyperparam(['model','num_conv_layers'],2,4) - rnn_layers = IntegerHyperparam(['model','rnn_layers'],1,4) - rnn_size = CategoricalHyperparam(['model','rnn_size'],[100,200,300]) - tunables = [lr,lr_decay,fac,target,batch_size,dropout_prob] - tunables += [conv_filters,conv_layers,rnn_layers,rnn_size] + rnn_layers = IntegerHyperparam(['model','rnn_layers'],1,3) + rnn_size = CategoricalHyperparam(['model','rnn_size'],[128,256]) + dense_size = CategoricalHyperparam(['model','dense_size'],[128,256]) + extra_dense_input = CategoricalHyperparam(['model','extra_dense_input'],[False,True]) + equalize_classes = CategoricalHyperparam(['data','equalize_classes'],[False,True]) + #rnn_length = CategoricalHyperparam(['model','length'],[32,128]) + #tunables = [lr,lr_decay,fac,target,batch_size,dropout_prob] + tunables = [lr,lr_decay,fac,target,batch_size,equalize_classes,dropout_prob] + tunables += [conv_filters,conv_layers,rnn_layers,rnn_size,dense_size,extra_dense_input] tunables += [cut_ends,t_warn] From 8f4c7d1d85c3c109b5970ce484b8ea616fe132a3 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Mon, 5 Feb 2018 15:40:34 -0500 Subject: [PATCH 024/272] latest version of scaling plot notebook --- examples/notebooks/FRNN_scaling.ipynb | 450 ++++++++++++++------------ 1 file changed, 245 insertions(+), 205 deletions(-) diff --git a/examples/notebooks/FRNN_scaling.ipynb b/examples/notebooks/FRNN_scaling.ipynb index 52c09f0e..40109fd0 100644 --- a/examples/notebooks/FRNN_scaling.ipynb +++ b/examples/notebooks/FRNN_scaling.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 4, + "execution_count": 29, "metadata": { "collapsed": false }, @@ -19,21 +19,33 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 13, + "execution_count": 212, "metadata": { - "collapsed": true + "collapsed": false }, "outputs": [], "source": [ + "import re\n", + "from os import listdir\n", + "from os.path import isfile, join\n", + "def plot_loss_vs_step(raw_path,ending='.out',save_path=False):\n", + " files = [join(raw_path,f) for f in listdir(raw_path) if (isfile(join(raw_path, f)) and ending in f)]\n", + " loss_arrays = [get_losses(open(f).read()) for f in files]\n", + " node_counts = np.array([get_num_gpus(open(f).read()) for f in files])\n", + " for (i,loss_arr) in enumerate(loss_arrays):\n", + " plt.semilogy(np.array(range(len(loss_arr)))*node_counts[i],loss_arr,label=r\"$N_{{GPU}} = {{{}}}$\".format(node_counts[i]))\n", + " plt.legend(loc=(1,0))\n", + " if save_path:\n", + " plt.savefig(save_path)\n", + "\n", + "def get_from_csv(f):\n", + " dat = np.genfromtxt(f,delimiter=',',names=True)\n", + " epochs = dat['epoch']\n", + " train_loss = dat['train_loss']\n", + " val_loss = dat['val_loss']\n", + " val_roc = dat['val_roc']\n", + " return epochs,train_loss,val_loss,val_roc\n", + "\n", "#regexes for parsing the log files\n", "def get_num_gpus(text):\n", " p = re.compile('\\[batch = \\d+ = \\d+\\*\\d+\\]')\n", @@ -54,6 +66,30 @@ " nums = [float(re.findall(r'\\d+.\\d+',match)[-1]) for match in matches]\n", " return nums[-1]\n", "\n", + "def get_effective_epoch(text):\n", + " p = re.compile(' \\[\\d+.\\d+\\/\\d+\\]')\n", + " matches = p.findall(text)\n", + " nums = [float(re.findall(r'\\d+.\\d+',match)[0]) for match in matches]\n", + " return nums\n", + "\n", + "def get_effective_epoch_and_loss(text):\n", + " p = re.compile(' \\[\\d+.\\d+\\/\\d+\\], ' + 'loss: \\d+\\.\\d+ \\[\\d+\\.\\d+\\]')\n", + " matches = p.findall(text)\n", + " e_eff = [float(re.findall(r'\\d+.\\d+',match)[0]) for match in matches]\n", + " e_eff = np.linspace(0,e_eff[-1],len(e_eff))\n", + " loss = np.array([float(re.findall(r'\\d+.\\d+',match)[-1]) for match in matches])\n", + " return e_eff,loss\n", + "\n", + "\n", + " \n", + "def get_epoch_size(text):\n", + " p = re.compile(' \\[\\d+.\\d+\\/\\d+\\]')\n", + " matches = p.findall(text)\n", + " nums = [float(re.findall(r'\\d+.\\d+',match)[0]) for match in matches]\n", + " epoch_size = float(re.findall(r'\\d+.\\d+',matches[0])[1])\n", + " return epoch_size\n", + " \n", + "\n", "def get_losses(text):\n", " p = re.compile('loss: \\d+\\.\\d+ \\[\\d+\\.\\d+\\]')\n", " #'loss: \\d+\\.\\d+ \\[\\d+\\.\\d+\\]'\n", @@ -67,7 +103,7 @@ "def get_execution_time(text):\n", " p = re.compile('Epoch \\d+.\\d+ finished \\(\\d+.\\d+ epochs passed\\) in \\d+.\\d+ seconds')\n", " #p = re.compile('Epoch 2 finished in \\d+.\\d+ seconds')\n", - " match = p.findall(text)[0]\n", + " match = p.findall(text)[1]\n", " execution_time = float(re.findall(r'\\d+\\.\\d',match)[-1])\n", " effective_epochs_passed = float(re.findall(r'\\d+\\.\\d',match)[-2])\n", " return execution_time/effective_epochs_passed\n", @@ -77,16 +113,161 @@ " return arr/arr[0]" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot from CSV log files" + ] + }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 249, "metadata": { "collapsed": false }, "outputs": [], "source": [ - "raw_path = './newtest/'\n", - "data_size = 'full' #'Titan' #full, large, medium\n", + "#only works if in the same order as the GPU counts!\n", + "csvfiles = [join(raw_path,f) for f in listdir(raw_path) if (isfile(join(raw_path, f)) and 'callback' in f)]\n", + "csvfiles = list(zip(*sorted(zip(node_counts,csvfiles)))[1])\n", + "f = csvfiles[0]\n", + "\n", + "\n", + "by_epoch_stats = [get_from_csv(f) for f in csvfiles]" + ] + }, + { + "cell_type": "code", + "execution_count": 251, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAisAAAF5CAYAAABeAGpJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzsnXt8znX/x5+fazaZrVFji+V8Jltb2JBE6MAUdTt1m/Nd\nN/UjEXeIW90OFSrcKUMlk+4iUdqdCsOKzTGGIse4EYU5bPP5/fHdrnbt2vHaNdd3X+/n4/F9rO/n\n+H5ta9fb5/P+vD9Ka40gCIIgCIJZsXnaAEEQBEEQhPwQZ0UQBEEQBFMjzoogCIIgCKZGnBVBEARB\nEEyNOCuCIAiCIJgacVYEQRAEQTA14qwIgiAIgmBqxFkRBEEQBMHUiLMiCIIgCIKpEWdFEARBEART\nYxpnRSk1VCl1SCl1WSmVqJRqVkD7tkqpJKXUFaXUfqVUTD5teyqlriulPi3uvIIgCIIg3FhM4awo\npXoArwMvAXcDO4CvlFKBebSvAawC1gKhwBvAfKVUhzzavgqsL+68giAIgiDceJQZLjJUSiUC32ut\n/y/zXQFHgTe11tNzaT8NeEhr3TRbWRwQoLV+OFuZDcNJiQXaZNZ3c3VeQRAEQRBuPB5fWVFKeQMR\nGKskAGjDg/oaiMqjW2RmfXa+yqX9S8AprfVCN80rCIIgCMINpoynDQACAS/gVI7yU0D9PPoE59H+\nVqVUWa31VaVUa6A/xjaRu+YVBEEQBOEGYwZnxe0opfyA94HBWutzbhz3dqAT8AtwxV3jCoIgCMJN\nwC1ADeArrfXZonQ0g7NyBsgAgnKUBwEn8+hzMo/2f2SuqjQAqgOfZ8ahQOaWl1LqGsbKyTEX5u0E\nfJivGkEQBEEQ8qMPsKQoHTzurGit05RSSUB7YCXYA13bA2/m0W0z8FCOso6Z5QApwF056l8B/IBn\ngaNa63QX5v0FYPHixTRs2LAw8kzPiBEjmDlzpqfNcBtW0mMlLSB6zIyVtIDoMSt79+7lySefhMzP\n0qLgcWclkxnAokzn4QdgBOALLAJQSk0Bqmits3KpvA0MzTwVtADDwXgceBhAa30V2JN9AqXUeaNK\n7y3svLlwBaBhw4aEh4cXQ655CAgIsIwWsJYeK2kB0WNmrKQFRE8poMhhFKZwVrTWyzJzm/wTYxtm\nO9BJa306s0kwcGe29r8opR4BZmKslBwDBmqtc54QKu68lufkybx2vEonVtJjJS0gesyMlbSA6LEi\npnBWALTWc4G5edT1z6VsPcbR48KO7zRGQfPeDBw/ftzTJrgVK+mxkhYQPWbGSlpA9FgRj+dZETxL\nRESh/b1SgZX0WEkLiB4zYyUtIHqsiDgrLpCe7mkL3EevXr08bYJbsZIeK2kB0WNmrKQFRI8VMUW6\n/dKCUiocSPr44yQef9xSwU6CIAgAHDlyhDNnznjaDKGUEhgYSLVq1XKtS05OzlolitBaJxdlXNPE\nrJQm9u3ztAWCIAju58iRIzRs2JDU1FRPmyKUUnx9fdm7d2+eDouriLPiAvv3e9oC99G/f38WLnS6\nOqnUYiU9VtICosfMZGk5c+YMqamplsolJdw4svKonDlzRpwVM2AlZ6Vjx46eNsGtWEmPlbSA6DEz\nObVYKZeUYA0kwNYFUlLAKqE+VgvcspIeK2kB0WNmrKRFsCbirLjA+fMgOXoEQRAE4cYgzoqLbN/u\naQsEQRAE4eZAnBUXKF8eduzwtBXuISEhwdMmuBUr6bGSFhA9ZsZKWgRrIs6KC9SrZ52VlenTp3va\nBLdiJT1W0gKix8xYSYtgTcRZcQErOStLly71tAluxUp6rKQFRI+ZsZIWwZqIs+IC9esbx5cvXfK0\nJcXH19fX0ya4FSvpsZIWED1mxkpaCkNsbCydO3fGx8eH6Ohoh7qDBw/Svn17bDYboaGhzJ1buu+5\nHTJkCJs3b/a0GcVG8qy4QL16xtHlXbsgMtLT1giCIAhFYeDAgTRq1IiGDRsya9Ysjh07RkhICAC1\natVi7dq19O7dmyVLlnjY0uLx3XffMX/+fJ588klPm1JsZGXFBWrVgjJlrBNkKwiCcLORmJjIhAkT\nCAoKYv78+Q51p0+fplmzZh6yzD1cvXqVb775BqWUp01xC+KsuEDZstCggTXiVkaNGuVpE9yKlfRY\nSQuIHjNjJS2F5eLFi/j7+9OvXz9iY2PJfqlvQkICrVq18qB1xWf27NkMGzYMq1xWLM6Ki4SFWcNZ\ncff9DZ7GSnqspAVEj5mxkpbCkrXiMGDAAE6cOMGqVavsdVu3bs26HbhUsnv3boKCgqhcubKnTXEb\nErPiImFh8OmnkJEBXl6etsZ1nnnmGU+b4FaspMdKWkD0mBlXtaSmGtePlCQNGoC74393795No0aN\nACNGpW3btrzzzjt06dIFgLS0NLwK8Yc9PT2dMmXM9TGqtebDDz9kypQpnjbFrZjru1yKCAsz/kf9\n+Wcj4FYQBOFmIyUFSnoBIikJ3H2n4oYNG+jWrZv9feDAgcTExHDs2DECAgLw9/d36rNs2TL27NlD\nlSpV8PX1xcfHh02bNjFr1iwA1q9fz+jRo6levTqdOnXi8uXLbNu2jdGjR7N69WrGjBnDtGnT6N27\nN5UrV2by5Ml8/PHHvPjii/To0QOA69ev0717d65evQrgtIWTtRqktaZChQrExcU52RkbG8vAgQPd\n840yEeKsuEhoqPF1+3ZxVgRBuDlp0MBwJkp6Dndz7NgxgoKC7O/du3dn2LBhzJ8/n6ioKKKioux1\n169fp3///rRu3ZqJEyfay19//XWqVq1qf2/Tpg0ZGRmMHDmS5s2bAzBnzhzGjRvHM888Q+vWrRk+\nfLi9fc+ePWnVqhXt2rWzl9lsNpYvX14sXZcvX6ZOnTouj2FWxFlxkcBAqFrVcFb+8hdPW+M6KSkp\nNCiJvwYewkp6rKQFRI+ZcVWLr6/7Vz08QdmyZenTpw+xsbGkp6czduxYe9348ePx8/Nj8ODBDn2i\noqKoUKGC/f3ChQvs27eP8GzfkCNHjuDv788333zj4JQAbNq0iZ49e7pVx5o1a/jhhx8YMGAAgH2F\nZurUqaxcuZLXXnvNrfPdSMRZKQZWCLIdPXo0K1eu9LQZbsNKeqykBUSPmbGSloI4fPhwrgHFgwYN\nYs6cOWzatIny5csDcPbsWWbMmMFPP/3k1L5ly5YO7wkJCYSFhdljWE6ePMn69etZunQpMTExTJ06\n1aF9amoqZcuWdSjLuQ2UF3ltAw0aNIhBgwY5aI2Li2Ps2LHce++9+Y5pdsRZKQZhYbBggaetKB6z\nZ8/2tAluxUp6rKQFRI+ZsZKWglixYgUtWrRwKg8NDSU8PJzQrD1+jNiWmjVrOmz3fPbZZ6xdu5a0\ntDRGjhxp33JZt24dgYGBrFy5kitXrnDp0iXi4+Px9vZmz5499q2hLGw258O4xd0Gykl6ejoAGRkZ\nbhvTU8jR5WIQFga//gr/+5+nLXEdqx1ZtJIeK2kB0WNmrKQlL5KTk+nYsSOjRo2iX79+LF682KnN\nkCFDuO++++zvNpuNihUrOrTp2rUrv//+Oy1atHCIDVm3bh2DBw8mOjqav/zlL/Tv3x9/f392795N\nkyZNHJyTjRs3OsTFlASvvPIKjz32GEophgwZUupPr4mzUgyyHHDJZCsIgmBuwsPDiY+P59q1a6Sk\npOSagn7IkCE8+uij9vd27dpx8uRJDh8+bC/LWjHJnjTu4sWL7Nq1y8HRyaJy5cr4+PjY3y9fvsyW\nLVto2rSpu6TlyosvvsjOnTvJyMhg//79vPXWWyU6X0kj20DFoHZtKF/eiFvp0MHT1giCIAjuxM/P\nj88//5wJEyYQGhrK7bffTlpaGlOnTqVu3bqAsWKzYMECAgIC+PLLL+nevbvDGNWqVaNXr15MmTLF\nnqRt6NChN1xLqUdrLU8hHyAc0ElJSTqLli217t1bl1qmTp3qaRPcipX0WEmL1qLHzGRpSUpK0jn/\nxglCYSno9yerHgjXRfz8lW2gYlLaTwSlpqZ62gS3YiU9VtICosfMWEmLYE3EWSkmYWGwbx9cvuxp\nS1xj0qRJnjbBrVhJj5W0gOgxM1bSIlgTcVaKSWiocT/Qjz962hJBEARBsCbirBSTJk3AZivdW0GC\nIAiCYGbEWSkmvr5Qv37pdVbOnDnjaRPcipX0WEkLiB4zYyUtgjURZ8UNlOYg26w7JKyClfRYSQuI\nHjNjJS2CNTGNs6KUGqqUOqSUuqyUSlRKNSugfVulVJJS6opSar9SKiZH/WNKqS1KqXNKqYtKqW1K\nqSdztHlJKXU9x7OnqLaHhsLOnXD9elF7ep7st4haASvpsZIWED1mxkpaBGtiCmdFKdUDeB14Cbgb\n2AF8pZQKzKN9DWAVsBYIBd4A5iulsqdmOwu8DEQCdwELgYU52gDsBoKA4MyndVHtDwuDCxfg0KGi\n9vQ84Va4MjUbVtJjJS0gesyMlbQI1sQUzgowApintX5fa50CPAWkAnmtTT4NHNRaj9Za79NazwH+\nkzkOAFrr9VrrzzLrD2mt3wR24uyMpGutT2ut/5f5/FZU48PCjK+ldStIEARBEMyMx50VpZQ3EIGx\nSgKA1loDXwN53fQUmVmfna/yaY9Sqj1QD1iXo6quUuq4UupnpdRipdSdRZRAUBAEB4uzIgiCIAgl\ngcedFSAQ8AJO5Sg/hbEtkxvBebS/VSlVNqtAKXWrUuqCUuoa8DnwjNb6m2x9EoF+QCeM1ZyawHql\nVPmiiggLK50XGsbGxnraBLdiJT1W0gKix8xYSYtgTczgrJQkFzBiWu4BXgRmKqXaZFVqrb/SWn+i\ntd6ttf4v8DBQEfhLUScKDS2dKyvJycmeNsGtWEmPlbSA6DEzVtIiWBMzOCtngAyMINfsBAEn8+hz\nMo/2f2itr2YVZN6ddFBrvVNrPRMjrmVsXoZorX8H9gN18jP44YcfJjo62uH55JMojh5dwdmzf7aL\nj48nOjraqf/QoUOd/iWTnJxMdHS0U76Dl156iWnTpjmUHTlyhOjoaFJSUhzK33rrLUaNGuVQlpqa\nSnR0NAkJCQ7lcXFx9O/fnzlz5jiU9+jRgxUrVjiUlQYdWWTpKe06srRYQQcYP48OOa4mL606sn4e\nWb9rpV1Hlpb4+HiGDx/u1NaKxMbG0rlzZ3x8fJx+dgcPHqR9+/bYbDZCQ0OZO3euh6x0jdTUVCZO\nnMizzz5LmzZtiImJ4dSpnBsRMGXKFJ5//nkmTpxITEwMFy5ccKsdcXFxREdHExUVRXBwMNHR0YwY\nMaLgjnlR1JsPS+LB2I55I9u7Ao4Co/JoPxXYkaNsCfBFAfPEAt/kU+8H/AYMy6Pe6dblLPbu1Rq0\nXrvWqUoQBKFUcDPdurxp0yb9/PPP6zJlyuijR4861ffq1csDVhWfMWPG6BMnTtjfO3XqpBs1aqSv\nXbtmL5s9e7Z+8MEH7e9TpkzRXbt2LfbcN8OtyzOAwUqpvkqpBsDbgC+wCEApNUUp9V629m8DtZRS\n05RS9ZVSfwcezxyHzD5jlFIPKKVqKqUaKKVGAk8CH2Rr86pSqo1SqrpSqiWwHEgD4ooqoG5dKFeu\ndG4FCYIg3GwkJiYyYcIEgoKCmD9/vkPd6dOnadYs31RfpuTq1avMnj3bYUVv5MiR7N27l5UrV9rL\npk+fTkzMn6nJ+vbty8qVK/npp59uqL1FwRTOitZ6GfA88E9gG9AU6KS1Pp3ZJBi4M1v7X4BHgAeA\n7RhHlgdqrbOfECoPzMHIo5IAPAb00VovzNYmBGNFJgVYCpwGIrXW2TZzCoeXFzRtWjqDbAVBEG42\nLl68iL+/P/369SM2NjZr9RyAhIQEWrVq5UHrXCMjI4PAwEAuXbpkL6tevToAP//8MwAHDhzg6NGj\nNGrUyN6mSpUqBAQE8O23395Yg4uAKZwVAK31XK11Da11Oa11lNZ6a7a6/lrrdjnar9daR2S2r6u1\n/iBH/XitdX2tdXmtdaDWurXW+j852vTSWodkjlFNa91ba+1yarfSGGSb2157acZKeqykBUSPmbGS\nlsKilAKMqwZOnDjBqlWr7HVbt24lIiLCU6a5jK+vL4cOHWLKlCn2sl9++QWAWrVqAYbTopTi1ltv\ndejr7+/PkSNHbpitRaWMpw2wEmFhsGABXL0KZcsW3N4MDBs2zNMmuBUr6bGSFhA9ZsZVLalpqaSc\nSSm4YTFoENgAX29ft465e/du+8pCrVq1aNu2Le+88w5dunQBIC0tDS8vrwLHSU9Pp0wZc3+MxsXF\nUb9+fbp27QrAuXPnAChf3jFDh5+fn73OjJj7u1zKCAuD9HTYswfuvtvT1hSOjh07etoEt2IlPVbS\nAqLHzLiqJeVMChHvlOwKRNKQJMLvcO91ABs2bKBbt27294EDBxITE8OxY8cICAjA39/fqc+yZcvY\ns2cPVapUwdfXFx8fHzZt2sSsWbMAWL9+PaNHj6Z69ep06tSJy5cvs23bNkaPHs3q1asZM2YM06ZN\no3fv3lSuXJnJkyfz8ccf8+KLL9KjRw8Arl+/Tvfu3bl61TjUmn1rCv5cDdJaU6FCBeLi8g+v3LFj\nBytWrOC///0v3t7eAHYnLKczlpaWRnp6eqG/hzcacVbcyF13gVLGVlBpcVYEQRBcpUFgA5KGJJX4\nHO7m2LFjBAX9mf2ie/fuDBs2jPnz5xMVFUVU1J/J0K9fv07//v1p3bq1w4WPr7/+OlWrVrW/t2nT\nhoyMDEaOHEnz5s0B40j4uHHjeOaZZ2jdurXD0fCePXvSqlUr2rX7M8LBZrOxfPlyt2i8ePEiQ4YM\n4dNPP+Wee+6xl1eqVMmuKzuXLl0iICDALXOXBOKsuMDV9Ku5lvv5QZ06EmQrCMLNga+3r9tXPTxB\n2bJl6dOnD7GxsaSnpzN27J/puMaPH4+fnx+DBw926BMVFUWFChXs7xcuXGDfvn0Ol0IeOXIEf39/\nvvnmGwenBGDTpk307NmzhBTB008/zWuvvca9994LwKFDh6hZsyY1a9YE4NSpUwQGGncFa605f/68\nPa7FjIiz4gL7z+4nKo9riMLCSleQ7YoVK3j00Uc9bYbbsJIeK2kB0WNmrKSlIA4fPky1atWcygcN\nGsScOXPYtGmTPZ7j7NmzzJgxI9cjvS1btnR4T0hIICwszB7DcvLkSdavX8/SpUuJiYlh6tSpDu1T\nU1MpmyO4Mec2UF4UtA30yiuvEBMTY3dUjhw5wrp166hZsyY1atSgTp067Nu3j8aNGwOQkpLC1atX\nnRwqMyHOigvsPLUzz7qwMJg+HbQ2toTMTlxcnKX+SFlJj5W0gOgxM1bSUhArVqygRYsWTuWhoaGE\nh4cTGhpqL9uwYQM1a9Z02O757LPPWLt2LWlpaYwcOZI6dYyE5+vWrSMwMJCVK1dy5coVLl26RHx8\nPN7e3uzZs8e+NZSFzeZ8GNcd20AfffQR3377LWXKlCEpydii+/HHH3nqqafsbfr27cv7779vj9tZ\ntGgR0dHR1K1bt1hzlyhFzSJ3Mz9kZrDtMK2DzovVq41MtocO5dlEEATBlFg5g21SUpLu0KGD9vb2\n1vXr19cffPCBU5t58+bp5cuX298/++wz3bJlS6d2ffv21QsXLnQoi4yM1F988YVT2y1btuj777/f\noSwhIUHv2LHDRSV5c/bsWe3r66ttNpvD4+XlpU+fPm1vl5aWpp9//nn97LPP6vHjx+u+ffvq8+fP\nF3v+ksxgKysrLrDr1K4868LCjK87dkCNGjfGHkEQBCF/wsPDiY+Pz7fNkCFDHN7btWvHiBEjOHz4\nsD25WtaKybhx4+ztLl68yK5du7jvvvucxqxcuTI+Pj7298uXL7Nly5YSuYfptttuc0gIlxdlypTh\n1Vdfdfv8JYk4Ky5w8uJJTlw4QRX/Kk51d9wBgYFG3ErmsXZBEAShFOLn58fnn3/OhAkTCA0N5fbb\nbyctLY2pU6fat0ySk5NZsGABAQEBfPnll3Tv3t1hjGrVqtGrVy+mTJlC5cqVAeOSS6FoiLPiIonH\nEunWsJtTuVKlL8hWEARByJ1GjRrx3nvv5VkfHh5OeHg4s2fPzrNN9nt4BNcwTbr90kSQXxCJxxLz\nrC9Nzkpu18eXZqykx0paQPSYGStpEayJOCsucFfluwp0Vn75Bc6fv3E2uYqVsnCCtfRYSQuIHjNj\nJS2CNRFnxQXuCrqLrSe2kpaRlmt9VpDtzrxPOJuGXr16edoEt2IlPVbSAqLHzFhJi2BNxFlxgaaV\nm3I5/XKe+Vbq1zcuMiwtW0GCIAiCYGbEWXGB+oH18bZ557kVVKYMNGkizoogCIIguANxVlygbJmy\n3H3H3SQeL/1BtgkJCZ42wa1YSY+VtIDoMTNW0iJYE3FWXCSyamSBQbY//ghpuYe1mIbp06d72gS3\nYiU9VtICosfMWEmLYE3EWXGRqDuj+Om3nzh96XSu9aGhcO0apKTcYMOKyNKlSz1tgluxkh4raQHR\nY2aspEWwJuKsuEhkSCQA3x//Ptf6pk2Nr2bfCvL19fW0CW7FSnqspAVEj5mxkhbBmoiz4iLVA6oT\nVD7v5HABAVCrlvmdFUEQBEEwO+KsuIhSisiQguNWxFkRBEEQhOIhzkoxiAqJ4vvj35NxPSPX+rAw\n4/ZlrW+wYUVg1KhRnjbBrVhJj5W0gOgxM1bSIlgTcVaKQWRIJBevXWTP6T251oeGwtmzcPz4DTas\nCFSrVs3TJrgVK+mxkhYQPWbGSloKQ2xsLJ07d8bHx4fo6GiHuoMHD9K+fXtsNhuhoaHMnTvXQ1YW\nj7Vr19K2bds866dMmcLzzz/PxIkTiYmJ4cKFC0Wqv+ForeUp5AOEAzopKUlrrfXFqxe1bZJNv7P1\nHZ0bhw9rDVp//nmu1YIgCKYiKSlJZ/8bZ2U2bdqkn3/+eV2mTBl99OhRp/pevXp5wKris2zZMt2v\nXz/9+OOP65o1a+baZvbs2frBBx+0v0+ZMkV37dq10PV5UdDvT1Y9EK6L+PkrKyvFoLxPeZoGNc0z\nbuXOO6FiRYlbEQRBMBuJiYlMmDCBoKAg5s+f71B3+vRpmjVr5iHLiscTTzzBwoUL6dy5c55tpk+f\nTkxMjP29b9++rFy5kp9++qlQ9Z5AnJViEhUSlWcmW6UkyFYQBMGMXLx4EX9/f/r160dsbGzW6jlg\nZPRt1aqVB60rOQ4cOMDRo0dp1KiRvaxKlSoEBATw7bffFljvKcRZKSaRIZHsOb2H81fO51qfFWRr\nVlLMnrWuiFhJj5W0gOgxM1bSUliUUgAMGDCAEydOsGrVKnvd1q1biYiI8JRpJcrPP/+MUopbb73V\nodzf358jR44UWO8pynhsZouQlRzuh+M/0LF2R6f60FCYORMuXAB//xttXcGMHj2alStXetoMt2El\nPVbSAqLHzLisJTW15NN0N2gAbk5at3v3bvvKQa1atWjbti3vvPMOXbp0ASAtLQ0vL68Cx0lPT6dM\nmdL1MXru3DkAypcv71Du5+fHuXPnCqz3FKXru2xC6t5Wl4q3VCTxWGKuzkpYmPF1504w46ri7Nmz\nPW2CW7GSHitpAdFjZlzWkpICJb0CkZQE4eFuHXLDhg1069bN/j5w4EBiYmI4duwYAQEB+OfyL8tl\ny5axZ88eqlSpgq+vLz4+PmzatIlZs2YBsH79ekaPHk316tXp1KkTly9fZtu2bYwePZrVq1czZswY\npk2bRu/evalcuTKTJ0/m448/5sUXX6RHjx4AXL9+ne7du3P16lUAh60p+HM1SGtNhQoViIuLK7L2\nLCcspzOWlpZGenp6gfWeQpyVYlJQcriGDcHb24hbMaOzYrUji1bSYyUtIHrMjMtaGjQwnImSpEED\ntw957NgxgoKC7O/du3dn2LBhzJ8/n6ioKKKioux1169fp3///rRu3ZqJEyfay19//XWqVq1qf2/T\npg0ZGRmMHDmS5s2bAzBnzhzGjRvHM888Q+vWrRk+fLi9fc+ePWnVqhXt2rWzl9lsNpYvX+52vdmp\nVKmSXVd2Ll26REBAQIH1nkKcFTcQFRLFzMSZXNfXsSnHMCAfH2jcWIJsBUGwIL6+bl/18ARly5al\nT58+xMbGkp6eztixY+1148ePx8/Pj8GDBzv0iYqKokKFCvb3CxcusG/fPsKzfT+OHDmCv78/33zz\njYNTArBp0yZ69uxZQorypmbNmgCcOnWKwMBAwFipOX/+PLVr16ZmzZporXOtr1Wr1g23NwtxVtxA\nZEgk566c48DZA9QPrO9UHxpq7iBbQRCEm4XDhw/nupI0aNAg5syZw6ZNm+zxGmfPnmXGjBm5Htlt\n2bKlw3tCQgJhYWH2GJaTJ0+yfv16li5dSkxMDFOnTnVon5qaStmyZR3Kcm4D5UVxtoFq1KhBnTp1\n2LdvH40bNwaMAOurV6/Srl27Aus9hZwGcgPNqzZHofLcCgoLg127wIPbfXkybdo0T5vgVqykx0pa\nQPSYGStpKYgVK1YQGhrqVB4aGkp4eLhD3YYNG6hZs6bDds9nn33Gs88+y9NPP+3gxKxbt47AwEBW\nrlzJsmXL+PLLL4mPjycoKIg9e/bYt4aysNmcP36ztoG++OKLfJ8vv/yyQEclIyPDaSsni759+/L+\n++/b3xctWkR0dDR16tQBICYmJtf6unXr5jtnSSIrK24g4JYAGlZqSOKxRGLCYpzqw8LgyhXYvx+y\nHV03BampqZ42wa1YSY+VtIDoMTNW0pIXycnJjBkzhu+++45atWoxbtw4nnzySYc2Q4YMoXLlyvZ3\nm81GxYoVHdp07dqVTz/9lPvvv9/+4Q6GszJhwgQeeughh/Zbt26lSZMmDs7Jxo0bHeJi3MmaNWuY\nP38+CQkJnD59mnvvvZcGDRrw7rvv2tu88MILjB07lv/7v/8jICCAkydPsmjRokLXe4SiprwtqQcY\nChwCLgOJQLMC2rcFkoArwH4gJkf9Y8AW4BxwEdgGPFmcecmRbj87Az8bqEP/HZpriuHffjPS7n/4\nYa7VgiAIpuBmSrdfGC5cuKBr1aqlf/nlF3vZxYsXdXBwsN6/f79Du/Lly+tLly45jXH48GHdqVMn\n+3tqaqqUWvbRAAAgAElEQVSeOXNmyRruIUoy3b4pVlaUUj2A14EhwA/ACOArpVQ9rfWZXNrXAFYB\nc4HewAPAfKXUCa31fzObnQVeBlKAa0AXYKFS6lRWm6LOmx+RIZEs3L6Qi9cu4ufj51BXsSJUr27E\nrfTuXZRRBUEQBE/h5+fH559/zoQJEwgNDeX2228nLS2NqVOn2rdEkpOTWbBgAQEBAXz55Zd0797d\nYYxq1arRq1cvpkyZYl+1GTp06A3XUtoxhbOC4STM01q/D6CUegp4BBgATM+l/dPAQa316Mz3fUqp\n1pnj/BdAa70+R583lVIxQOusNi7MmyeRIZFc19fZemIrbWu0daoPDZUTQYIgCKWNRo0a8d577+VZ\nHx4eTnh4eL65arLfsyO4hscDbJVS3kAEsDarTGutga+BvDb1IjPrs/NVPu1RSrUH6gHrijFvnjQM\nbIi/j3++QbbbtkGOHD8e58yZIi0gmR4r6bGSFhA9ZsZKWgRr4nFnBQgEvIBTOcpPAcF59AnOo/2t\nSin7WTCl1K1KqQtKqWvA58AzWutvijFvnnjZvGgR0oLNxzbnWh8WBqdPw8mTRR25ZBkwYICnTXAr\nVtJjJS0gesyMlbQI1sQMzkpJcgEIBe4BXgRmKqXaFHfQhx9+mOjoaIcnKiqKioeMtPs6c/kkPj6e\n6Oho4M+0+9u3G/uVsbGxDmMmJycTHR3t9C+cl156yelY4ZEjR4iOjna6fOytt95i1KhRDmWpqalE\nR0eTkJDgUB4XF0f//v0dMjIC9OjRgxUrVjiUZdeRHTPpyCJLT2nXkaXFCjrA+Hncd999ltCR9fPI\n+l0r7TqytMTHxztkWBWE4hAXF2f/bAwODiY6OpoRI0a4PJ7SHt6XyNyOSQW6a61XZitfBARorR/L\npc86IElr/Vy2sn7ATK11xZzts7V5FwjRWj/k4rzhQFJSUpJDlsIsVu9fTee4zhx89iA1K9Z0qNMa\nKlSAsWNhzJi8LBQEQfAcycnJREREkNffOEHIj4J+f7LqgQitdXJRxvb4yorWOg3jCHL7rDJl3NbU\nHtiUR7fN2dtn0jGzPD9sQNlizJsvLUJaAOQat6KUBNkKgiAIgit43FnJZAYwWCnVVynVAHgb8AUW\nASilpiilsodjvw3UUkpNU0rVV0r9HXg8cxwy+4xRSj2glKqplGqglBoJPAl8UNh5i0qgbyB1b6ub\nb9yKOCuCIAiCUDRM4axorZcBzwP/xEje1hTopLU+ndkkGLgzW/tfMI4YPwBsxziCPFBrnf2EUHlg\nDrAbSMBIEtdHa72wCPMWmfxuYA4LM7LYXrrk6ujuJ+eeemnHSnqspAVEj5mxkhbBmpjCWQHQWs/V\nWtfQWpfTWkdprbdmq+uvtW6Xo/16rXVEZvu6WusPctSP11rX11qX11oHaq1ba63/U5R5XSEyJJJt\nJ7dxOe2yU11YmBG7smtXcWZwL8nJRdo2ND1W0mMlLSB6zIyVtAjWxDTOilWIDIkk/Xo6205uc6pr\n1Ai8vMx1A/OcOXM8bYJbsZIeK2kB0WNmrKRFsCbirLiZpkFNKVemHJuPOset3HILNGwocSuCIAiC\nUBTEWXEzZWxlaFa1GYnH845bEWdFEARBEAqPOCslQGTV/INsd+6EjIwbbJQgCILgVkJCQhgyZIj9\nfe3atdhsNjZtKjj7RevWrenYsaNb7Rk3bhze3t5uHdMsiLNSAkSGRHLsj2Mc++OYU11YGKSmwk8/\necCwXMgt82Zpxkp6rKQFRI+ZsZKW3OjatSvly5fnUj5HMfv06UPZsmU5d+5cocc1UnMVXFbYvoXh\n0qVLTJo0ySlbcdaYNps1P9atqcrDRIZEArknhwsNNb6aJch22LBhnjbBrVhJj5W0gOgxM1bSkht9\n+vThypUrLF++PNf6y5cvs3LlSh5++GEqVswzCXqBtG/fnsuXL9OyZUuXxyiIixcvMmnSJNavX+9U\nN2nSJC5evFhic3sScVZKgDv876B6QPVcnZXAQKha1TxxK+5ehvQ0VtJjJS0gesyMlbTkRnR0NH5+\nfixZsiTX+hUrVpCamkqfPn2KPZePj0+xx8iP/K7Isdlssg0kFI2CksOZxVkRBEGwOrfccgvdunVj\n7dq1TpdIAixZsgR/f3+6dOkCwLRp02jVqhW33347vr6+NGvWzOlC0dzIK2bl3//+N7Vr18bX15eo\nqKhcY1quXr3K+PHjiYiIoEKFCvj5+dG2bVs2bNhgb/Pzzz9TpUoVlFKMGzcOm82GzWbjX//6F5B7\nzEp6ejqTJk2idu3a3HLLLdSqVYsJEyaQlpbm0C4kJIRu3bqxfv16mjdvTrly5ahTp06eDt6NRpyV\nEiIyJJKkX5O4lnHNqU6cFUEQhBtLnz59SEtLY9myZQ7l586dIz4+nm7dulG2bFkA3nzzTSIiInj5\n5ZeZMmUKNpuN7t27Ex8fX+A8OWNR5s2bx9ChQ7nzzjt59dVXiYqKokuXLpw4ccKh3fnz51m0aBHt\n27dn+vTpTJw4kZMnT9KxY0d+/PFHAIKDg5kzZw5aa5544gkWL17M4sWLefTRR+1z55y/X79+TJo0\niRYtWjBz5kzuvfdeXn75ZZ588kknu/ft20fPnj158MEHmTFjBgEBAcTExHDgwIFCfIdLGK21PIV8\ngHBAJyUl6YJIPJqomYj+4dgPTnUff6w1aH3qVIHDlDjLly/3tAluxUp6rKRFa9FjZrK0JCUl6cL+\njcvixIkTOikpKc/nxx9/LHCMH3/8Mde+J06ccFlTTjIyMnSVKlV0q1atHMrffvttbbPZ9Ndff20v\nu3LlikObtLQ03ahRI/3ggw86lIeEhOjBgwfb37/++mtts9n0xo0btdZaX7t2TQcGBurmzZvr9PR0\nhzmVUrpDhw4O9qWlpTmMf/78eV2pUiX91FNP2ctOnjyplVL6lVdecdI4btw47e3tbX9PSkrSSik9\ndOhQh3YjRozQNptNJyQkOGix2Ww6MTHRYS4fHx89duxYp7lyo6Dfn6x6IFwX8fNXVlZKiLDgMHy8\nfEwfZBsXF+dpE9yKlfRYSQuIHjNTHC3z5s0jIiIiz+eJJ54ocIwnnngi177z5s1z2a6c2Gw2evbs\nyebNmzly5Ii9fMmSJQQFBdGu3Z83umStsICx4nH+/Hlat25d5GsJvv/+e86ePcvTTz+Nl5eXvXzA\ngAH4+/s72VemTBnAWEQ4d+4caWlp3HPPPS5fh/DFF1+glGLEiBEO5SNHjkRrzerVqx3KmzZtSosW\nLezvQUFB1K1bl4MHD7o0vzsp42kDrErZMmUJvyOcxOOJPMMzDnW1a0P58sZWUIcOHjIwk48++siz\nBrgZK+mxkhYQPWamOFr+9re/5Xv0+ZZbbilwjI8//pgrV644ld9xxx0u25Ubffr0YebMmSxZsoQx\nY8Zw/PhxEhISGD58uMP2ycqVK/nXv/7Fjh07uHr1qr28qMGzhw8fRilFnTp1HMq9vb2pUaOGU/uF\nCxcyY8YM9u3bR3p6ur28Xr16RZo3+/xlypShdu3aDuVVq1bF39+fw4cPO5RXq1bNaYyKFSsW6Th3\nSSHOSgkSWTWSlftXOpXbbMbqisStCIJQ2rnjjjuK7VQ0atTITdbkT3h4OA0aNCAuLo4xY8bYg0d7\n9+5tb/Ptt9/y2GOP0a5dO95++22Cg4Px9vbm3Xff5ZNPPikx2xYtWsTAgQN5/PHHGTt2LJUqVcLL\ny4vJkydz/PjxEps3O9lXf7Kj8zmBdKOQbaASJOrOKA6eO8j/Lv3PqU6CbAVBEG48ffr0Yffu3eza\ntYu4uDjq1q1LRESEvf7TTz+lfPnyrFmzhpiYGDp16kS7du24fv16keeqXr06WmunANW0tDR++eUX\nh7JPPvmE+vXrs2zZMnr37k2HDh1o164dly9fdmhXlGRy1atXJz09nZ9//tmh/MSJE1y4cIHq1asX\nTZAHEWelBCkoOdy+fZDj91AQBEEoQfr06YPWmgkTJrB9+3anUzFeXl7YbDYyst2JcvDgQT7//PMi\nz9WiRQtuu+023n77bYfx5s+fz4ULF5zmzcnGjRvZsmWLQ1n58uUBI5amIB5++GG01syaNcuh/PXX\nX0cpxSOPPFJoLZ5GnJUS5M5b7+QOvztydVbCwoz7gTJPpHmM/v37e9YAN2MlPVbSAqLHzFhJS0HU\nqFGDli1b8tlnn6GUctgCAnjkkUf4448/6NSpE++88w6TJk0iKiqK+vXrF2r87Fsm3t7eTJ48ma1b\nt3L//fczZ84cRowYwbhx46hVq5ZDv86dO7Nv3z66devGu+++y5gxY+jcubPTFln58uWpV68ecXFx\nvP3223z00Ufs3bs3V1vCw8Pp06cPc+fOpXfv3vz73/+mb9++zJw5kyeeeIJWrVoVSpMZEGelBFFK\n5ZkcrkkTI3bF01tBVstcaSU9VtICosfMWElLYejTpw9KKVq0aOHkNHTo0IF3332XEydOMHz4cD7+\n+GNef/11Onfu7DRObnlNcr4//fTTzJ49m+PHjzNq1Ci+//57Vq1aZU/ulsWgQYN4+eWX2bZtG8OH\nD2ft2rUsXbqUsLAwpzEXLFhAcHAwI0aMoHfv3g7XCORsu2jRIl566SW+//57RowYwYYNGxg/fjyL\nFy8uUEteY3oCZYbAmdKCUiocSEpKSiI8PLxQfV7d+CqT1k3i/JjzlLE5xjM3agTt2sHs2SVgrCAI\nQhFJTk4mIiKCovyNE4QsCvr9yaoHIrTWRTqPLSsrJUxkSCSX0i7x4/+c93skyFYQBEEQCkaclRIm\nokoEXsorzyDbnTvBhSBzQRAEQbhpEGelhPH19iU0OJTE47kH2V64AIcOecCwTBISEjw3eQlgJT1W\n0gKix8xYSYtgTcRZuQFEVs09yDYszPjqya2g6dOne27yEsBKeqykBUSPmbGSFsGaiLNyA4i6M4qU\nMyn8dvk3h/KgIAgO9qyzsnTpUs9NXgJYSY+VtIDoMTNW0iJYE3FWbgBZyeF+OP6DU11YmGcvNPT1\n9fXc5CWAlfRYSQuIHjNjJS2CNRFnxQUuXSpa+9oVa3N7udvzDLKVE0GCIAiCkDfirLjAY4/BwoWF\nP8WTX3K4sDA4ehTOnnWzkYIgCIJgEcRZcYF77oEBA6BZM9iwoXB9okKiSDyWyHXt6OFkBdl6aito\n1KhRnpm4hLCSHitpAdFjZqykRbAm4qy4wL/+BRs3Guny27SBHj3g8OH8+0SGRPL71d/Zd2afQ3nd\nulCunOe2gqpVq+aZiUsIK+mxkhYQPWbGSloEayLOiiu89x4tW2Tw/fewaJGxutKgAbzySt5dmlVt\nhkI5bQV5ecFdd3luZeWZZ57xzMQlhJX0WEkLiB4zYyUtgjURZ8UV3nwT7r0X24F9xMTA/v3w3HP5\nx7DcWvZWGldunGfcigTZCoIgWIN9+/Zhs9lYtmyZp02xDOKsuEJsLJw+bXgZM2fiVy6DV16B8ePz\n7xYVEsXmY5udysPCYM8euHq1hOwVBEG4ibHZbAU+Xl5erF+/3m1zmuGmYitRpuAmghNZyVFefBFG\njoRPPzWOB9Wpk2+3yJBI5ifP58LVC/iX9XcYLj3dcFjuvrukjXckJSWFBg0a3NhJSxAr6bGSFhA9\nZsZKWnJj8eLFDu/vvfceX3/9NYsXL0ZrbS9v2LChW+arX78+ly9fxsfHxy3jCbKy4jq+vjBzJnz3\nHZw4AU2bwltv5bsXdE9wJBrNlhNbHMrvuguU8sxW0OjRo2/8pCWIlfRYSQuIHjNjJS250bt3b4en\nXr16APTq1cuhvFKlSrn2v3LlSpHnFEfFvZjGWVFKDVVKHVJKXVZKJSqlmhXQvq1SKkkpdUUptV8p\nFZOjfpBSar1S6rfM5785x1RKvaSUup7j2VMkw9u0Ma5OHjgQnn0W2rWDgwedmm3fDp0jG+BrC2Dz\nUce4FT8/Y1HGE0G2s2fPvvGTliBW0mMlLSB6zIyVtBSXr776CpvNxvLly3nhhReoWrUqfn5+XLt2\njTNnzjBixAiaNGmCn58fFSpUoEuXLuzZ4/ixkVvMSs+ePalUqRJHjx6lc+fO+Pv7ExQUxIsvvnij\nJZZKTOGsKKV6AK8DLwF3AzuAr5RSgXm0rwGsAtYCocAbwHylVIdsze4DlgBtgUjgKBCvlLojx3C7\ngSAgOPNpXWQB5csbqyrffGOcYW7aFP79b4dVlgoV4J4IG6n7W/DaR5v5IUfmfU8F2VrtyKKV9FhJ\nC4geM2MlLe5i/PjxfPfdd7zwwgtMnjwZLy8v9u3bx5o1a3jssceYNWsWI0eOJDk5mbZt23LmzJl8\nx1NKkZaWRocOHQgJCeG1116jZcuWTJ06lffee+8GqSq9mCVmZQQwT2v9PoBS6ingEWAAkNt1oE8D\nB7XWWWuX+5RSrTPH+S+A1vqv2TsopQYB3YH2QPYNzHSt9Wm3qLj/fmOVZfRo+Pvf4ZNPjGDc6tWp\nUcMIbYlZGMmHB+bSooXmr39VTJkCVasazsr06aC1sSUkCIIgeA6tNRs3bqRMmT8/Jps1a8bevXsd\n2vXq1YvGjRvz3nvvMXLkyHzHvHDhAhMmTOC5554D4G9/+xtNmjQhNjaWmJiYfPve7HjcWVFKeQMR\nwL+yyrTWWin1NRCVR7dI4OscZV8BM/OZqjzgDfyWo7yuUuo4cAXYDIzVWh8tvIIc+Psbqyrduhlb\nQ02awIwZMGgQKEXP1pG8f+SfvDznIG9MrM0nn8DYsdC4Mfz+u7EwU6OGy7MLgiDcMFJTU0lJSSnR\nORo0aOCRixYHDBjg4KiAYxxKRkYGv//+OxUqVKBmzZokJycXatwhQ4Y4vLdu3ZpVq1YV32CL43Fn\nBQgEvIBTOcpPAfXz6BOcR/tblVJltda5HQKeBhzH0clJBPoB+4A7gInAeqVUE611Ea8rzEGHDrB7\nt3FaaMgQ+M9/YP58WoS0AKBG60QOHKjN5MkwZQp88YXRbceOG+usTJs2jRdeeOHGTVjCWEmPlbSA\n6DEzrmpJSUkhIiKiBCz6k6SkJMLDw0t0jtyokcsf4uvXr/Paa68xb948Dh8+zPXMrX6lFHUKOA0K\nUKFCBfz8/BzKKlasyLlz59xis5UxRcxKSaOUGgP8BXhUa30tq1xr/ZXW+hOt9W6t9X+Bh4GKmW3z\n5OGHHyY6OtrhiYqKYsWKFQ7t4hMTiT51Cr78En78EZo04ba4FVT4OoDY2FgCAuC114yLDP38kvHx\niWbjRsd9z5deeolp06Y5lB05coTo6Ginf9G89dZbTnd8pKamEh0dTUJCgkN5XFwc/fv3JzU11aG8\nR48ezjri44mOjnb6PgwdOpTY2FiHsuTkZKKjo532b0taR/Z2VtCR1dYKOsD4eeT8l2dp1ZH188j6\nXSvtOrL6x8fHM3z4cKe2+dGgQQOSkpJK9PHUkepy5co5lU2YMIExY8bQqVMn4uLiiI+P5+uvv6ZO\nnTp2xyU/vLy8ci3PfnzaKsTFxdk/G4ODg4mOjmbEiBGuD6i19uiDsTWTBkTnKF8ELM+jzzpgRo6y\nfsC5XNo+j7H1c3ch7fkBeCWPunBAJyUl6SJz7pzW/fppDXp7eFX94NS7nJo88IDWjz5a9KEFQRDc\nQVJSknb5b1wpYtiwYdpms+Vat2bNGq2U0qtXr3aqa9CggX7kkUecygMDA/VDDz1kf09JSdFKKf3R\nRx/Zy3r27KkrVark1HfMmDG6XLlyrsgwHQX9/mTVA+G6iL6Cx1dWtNZpQBJG4CsAykj91x7YlEe3\nzdnbZ9Ixs9yOUmo08CLQSWu9rSBblFJ+QB3g18LaX2gqVDASx61aRZ0jF4mbuIurC941ImozkbT7\ngiAIniev7LNeXl5OqyAffPABZ8+evRFm3dR43FnJZAYwWCnVVynVAHgb8MVYXUEpNUUplf1s19tA\nLaXUNKVUfaXU34HHM8chs88LwD8xThQdUUoFZT7ls7V5VSnVRilVXSnVEliOscoTV2JKH3mEgxtW\nsrIelB04BLp2hV8N3ygsDH75Bc6fd+zy2mvGwSILrhQKgiCYjpwOSRadO3dmzZo1DBkyhHfffZdh\nw4bx3HPP5RrfIrgXUzgrWutlGNs1/wS2AU0xVkOyjhQHA3dma/8LxtHmB4DtGEeWB2qtswfPPoWx\nxfQf4ES2J/vZshCMXCwpwFLgNBCptS5RN7lhvZb8vUd5lk+JgR9+MI4CffghoU2N/0F27vyzrdaw\ncSM8/rhxMtrdKy8F5QYobVhJj5W0gOgxM1bSUljyu7snr7qJEyfy7LPPsnr1ap577jn27NlDfHw8\nwcHBTn1yGyOvceUeoUJQ1H2jm/mhODErOWi7qK3u9lE3rc+c0bpXL61BZ0Q/qu/0OanfeMO5/Zdf\nat2ggdZKaT1okNYnTxbbBK211l26dHHPQCbBSnqspEVr0WNmsrTcLDErQslg6ZiVm5XIqpFsProZ\nfdttsGQJ/Oc/2DZvZGdGY7w//cip/YMPGisub7xhbAnVrQuvvlr8m5onTpxYvAFMhpX0WEkLiB4z\nYyUtgjURZ8VDRIZE8uvFXzn2xzGjoHt3+PFHDlRrx9PresJf/gKnHRPrenvDM8/AgQMQE2Mkkxsw\noHh2eCJ/QUliJT1W0gKix8xYSYtgTcRZ8RCRIZEAbD6W7QBTpUp8/9wyent9hP7mGyOW5ZNPnPre\nfrtxFdGOHfCPf9woiwVBEATBM4iz4iGC/IKoWaEmicccb2AODYW4jL+Q8p8foXVrI7K2Vy/I5Whc\n48bGIwiCIAhWRpwVDxIZEunkrDRtanzdejTIWFX58EP46ivDK/nsM7fbkDPDZmnHSnqspAVEj5mx\nkhbBmrjkrCilHsy85TjrfahSartSaolSqqL7zLM2kSGRJP+azNX0P6NkAwKgVq3MI8pKQe/eRqr+\n5s3h0Ufhr3+F33LexZg7qalGpv/88rMU9vKt0oKV9FhJC4geM2MlLYI1cXVl5VXgVgCl1F3A68AX\nQE2yJWazKpeuFe+OwyyiQqK4mnGV7Scdk6c4ZbK94w5jVeX992HVKuMm50Lc0vnRR/Dww8ZJoj17\ncm8zZ86cYigwH1bSYyUtIHrMjJW0CNbEVWelJpD18dcdWKW1/gcwFHjIHYaZmY4fdGTgZwONo8fF\nSCsbGhxKWa+yTltBWc6Kw9BKGasqu3fD3XdDly7Qr59zutts9OsHK1bAzz8b20vPPJNr6IsgCIIg\nmJoyLva7hpEOH4wssu9n/vdvZK64WJl+Yf348tCXLNi+gMaVGjMofBB/bfpXbve9vUjj+Hj5EFEl\ngsTjifwf/2cvDw01dnqOH4eQkBydqlY1VlUWLYLhw+Hrr+Hdd+EhZx9RKSOb/4MPwptvwuTJRgjM\npEnw1FPGUWhBEISc7N2719MmCKWQEv29KWoWucyVhJXAGmA8huNSNbO8I7DflTFLw0O2DLYZ1zP0\nVz99pZ9Y9oT2/qe39pnso3v+p6c++NvBfDP85eS5Nc/pGrNqOJQdPqw1aP355wV0PnJE644djcYD\nB2p9/ny+zU+e1HrwYCMLbsOGRndBEIQsDh8+rH19fbOyjMojT5EfX19fffjw4Vx/v4qTwdbVlZVh\nwFyMywOf1lofzyx/CMOJsTw2ZaNj7Y50rN2R05dO88HOD1i0fRG+3r4Fd85GZEgkMxJncPLiSYL9\nggG4806oWNHYCurcOZ/Od94Ja9bA/PkwciTEx0NsLHTokGvzoCB45x34+9/h7behShWIjo5m5cqV\nRbLZzFhJj5W0gOgxM1laqlWrxt69e0v9XUHDhw9n1qxZnjbDbZQmPYGBgVSrVs39AxfVu7mZH9x4\nN1AWR38/qpmIXr53uUP5/fdr3b17EQb65Ret27c3Vln+9jet//ijUN2++uqrPOsKOYSpyE9PacNK\nWrQWPWbGSlq0Fj1mpTgrK0obH8JFQikVDqRprXdlvncF+mME3U7UWl9zlzNlJjJ1JyUlJbmcnvr4\nH8epemtVh7KQGSE82fRJpj4w1V42YoQRmnLgQBEG1xrmzYPnn4fAQFiwANq1c8nO8+eNIRo3hvvu\ngzZtjKdyZZeGEwRBEG5ykpOTiYiIAIjQWhfpvLyrp4HmAfUAlFK1gKVAKvAEMN3FMS3P3tN7uXPm\nnbR/vz1xu+K4kn4FyD05XFgY/PQTXLhQhAmUMiJnd+2CmjWhfXsYNgwuXiyyrWXKGDtKERHwxRfw\nxBPGNlLDhsYUS5YYeVwEQRAEoaRx1VmpB2RlAnkCWK+17g30wzjKLORCjQo1eO/R90jLSKP3p72p\nOqMqw9cMp0aFGmw5sYX06+n2tmFhxtedO12YqGZNWLvWuEBo4ULjeNH69UUaws/PuCxxwQLDaTp6\n1DhJdN99sG4d9O0L6ekFjyMIgiAIxcVVZ0Vl6/sARkI4gKNAYHGNsirlvMvx19C/sr7/elKGpjDw\n7oEs2bWE1ze/TmpaKlMT/twGatjQOFq8fXs+A+aHzWasquzYYUTS3nefcdQ5x3LIihUrCjVcSIiR\nTPftt2HvXvjf/+DWAg6p/+9/xs7UjaSwekoDVtICosfMWEkLiB4r4qqzshUYp5T6K3AfsDqzvCZw\nyh2GWZ36gfWZ3mE6x547xpJuS1Aovjjwhb3ex8eIF3HZWcmiTh1jKWTmTCOeJTQUNm60V8fFxbk0\n7G23FdymeXPDT+rRA+bONfLZXb/u0nSFxlU9ZsRKWkD0mBkraQHRY0VcDbBtCnwIVANmaK0nZZa/\nBdyeuSVkOdwRYJsXzd5tRsPAhrz/2Pv2sn79jDT5P/zgpkn27zcGTUw0InhffhnKlXPT4I5obdy/\nuG6dsQO1ZQukpcHtt/8ZrPuXvxjOjCAIgmB9bniArdZ6p9b6Lq11QJajkskoIMaVMW92IqvmHmS7\na9efsSEXr13EFefSTr16sGEDTJ8Oc+YYEyQmFtzPBZQyMudOmWIs5Jw/byTbHTrUyM47ZgwcOVIi\nU3nhQd0AACAASURBVAuCIAgWw9VtIACUUhFKqSczn3Ct9RWtdZq7jLuZiLozigO/HeBs6p+X94SF\nwZUrxoIIwIg1I6g3ux7TEqZx8uJJ1yby8jKONm/bBhUqQKtW8MILxkQliK+vcThp0iT47jv4/Xdo\n1iz/Pr/+Clev5t9GEARBsD4uOStKqcpKqW+BLcCbmc9WpdRapVQldxp4sxAZEgnA98e/t5eFhhpf\ns+JW+oX1IyokionrJhIyI4THPnqM1ftXk3E9o+gTNmxoLHn8618waxaEhxt7NTeIsmUNvyk/nnoK\nAgKgbVt46SXjgJMclxYEQbj5cHVl5S3AD2istb5Na30b0ATjEsM33WXczUTNCjWp5FvJYSuoYkWo\nXv1PZ6VVtVa8/9j7/DryV9548A0OnTtE57jOVJ9VnQnfTij6akuZMvRPSYHkZGPpIyoKXnzRNMsZ\nkycb20gVK8Ls2fDAA38uBo0dm3vwcf/+/W+8oSWElbSA6DEzVtICoseKuOqsPAj8XWttv2JRa70H\nGIpxP5BQRJRSuSaHCw01Th9np8ItFRjafCjb/raNrYO30rleZ2YlzuJMatHv8+jYsaNx7GjzZmOP\n5tVX4Z57DAfGwzRtasQBL18Op08b8TszZxrHqBcuzD3wuGPHjjfe0BLCSlpA9JgZK2kB0WNFXD0N\ndAG4V2u9PUf53cA6rXUBGThKJyV5GghgyoYpTN04lXMvnMOmDD/ypZfg3/+GU6eMoNW8uJx2mXLe\nbjjZs3OnkQ1u1y74xz9g3DjjHLXJ0NoIPPb2zrvNTz/B8ePGDpe//42zTRAEQXDGE+n2vwHeUErZ\nD54qpaoCMzPrBBeIDInkj6t/sPe0fcGKsDBjVeFkATs8hXFU0jIKEfvctKmxZDF+vLEH07y589KO\nCVAqf0cFYPFiI94lIACaNIH+/Q3Hb+tWuGbJ26sEQRCsiavOyjCM+JRflFI/K6V+Bg4B/pl1ggvc\nU+UebMrmsBWUlXa/uMnhTl86TZUZVXh61dMknUjKv7G3t7Gk88MPRha3e+4xAkjSStdBr/HjjQWi\n+fPh3nuNRaNnnzVOIfn7GylnBEEQBPPjap6Vo0A48AgwK/N5GOgKTHCbdTcZ/mX9aVK5iYOzUqOG\nkda+uIsbSimeiniKlftXcs+79xA+L5y5W+ay6utVeXe6+25jGWLMGCOeJTLS+PQ3MQkJCfb/9vIy\nVlQGDDBWVJKSjIshN2+G114zVl3yQ+sbf11AdrJrsQKix7xYSQuIHkuitXbbA4QCGe4c00wPhoOm\nk5KSdEnxt8//phvPaexQdu+9Wvfo4Z7x0zLS9Kp9q3TXuK7aa5KXph663lv19NOrns6/45YtWjdq\npLW3t9avvKJ1Wpp7DHIzXbp0cdtYR49qHRysdXS01pMna71mjdZnz7pt+AJxpxYzIHrMi5W0aC16\nzEpSUpIGNBCui/j561KAbV4opUKBZK11ARk0SiclHWALsGj7IgZ8NoBzL5wj4JYAwNi6iI+HlBT3\nznXy4km++PELtp3dRobOYO4jc/PvcPUqTJxoZMAND4f33oNGjdxrVDFJTU3F19fXLWP9+qtxZHrL\nFuM5f94or13bCOVp1gyGDIHy5d0ynRPu1GIGRI95sZIWED1mxRMBtkIJERkSiUaz5cSfCdrCwows\ntpcuuXeuYL9gBrQYwFsPv1Wgo3JdX6fvF4OZGV2ZXcvncf3iBWObaPp0yHAhKV0J4c7/oe+4A155\nxXAUf/vN+BksXgydO8Phw/DPfxYc5FscrPDHKTuix7xYSQuIHisizorJqHd7PSrcUsEpyFZrz4aL\n/Hb5Nw6dP8Q/vvkHTbcNJuCJn/ng/tu4PuYFzoQ34NgPa4t3b5HJUQrq1oU+fYyEvxs3wpkzBZ/q\nnjcPliyBAwc8G/8iCIJQmilTlMZKqU8LaFKhGLYIgE3ZiAyJZPOxzfayRo2MYNEdO4wYV08Q6BvI\nhv4bSMtIY+epnSQeSyT+7kQ+b/gtL7//E3e2eoAz41+g0ouvFJxH3yIURuaiRX/eFVmxonGwqnnz\nP7eR7rijRE0UBEGwBEVdWfm9gOcw8L47DbwZybqBOWul4pZbjKt8int8OTdGjRpVpPbeXt5EVIlg\naPOhfPDYByybeYyKKb9w8slHCZw4Hdq0MZYR/p+9M49vqkr///ukTdu0pXuhtGUXEVnKIpi6gCOK\nKx23EXUcN9zFbUaR+c4oOs4o7rujjijqqLiNftFRUL/zE8ShIBREVpVCoWWHlm5J2yTn98dJ0nRJ\nuqVNcjnv1+u+cnNzz7nPpzdpnjznOc9phYO1B3FJVzDM9ktH9XQ3y5erCMyiRaoab1ycmkr9619D\ndja89Zb/tuGmpatoPeGLkbSA1mNEOhRZkVJ22wIFQohbgLuALOAH4FYppd+V9YQQpwBPACOAHcDf\npJRv+Lx+LXAFas0igNXA/zTvs6PX7QmsuVbuX3I/vxz6haHpQwE1FNQdzkr//v273Edm5gAyX/8Y\nrvlWVV7Ly1MF5W69FUyN/vB5753H+n3rOT7neKy5Vqy5VibmTCTNktZlGzwEQ0+wSU+HM85QG6jh\noNJSVcbm+OP9t+vfvz8//ww7d0Lv3mpLT4/cwFU43puuYCQ9RtICWo8RCepsoE4bIcR04A3gemAl\ncCfwG+BoKWWLBW+EEAOB9cCLwDzgNNy1XqSUX7nPeQv4DvgvYAdmA+cDx0opd3fyut0+Gwig3FZO\n2qNpvHnem/wu73cAPPEE3HcfVFaG+ZdVTY0q0//ssyrK8tpravoMsGT7EpbtWEZhWSGFpYXetYyG\npQ/Dmmvl+vHXc0K/E0Jpfdhx332qHp8HkwkyMhqdl9Gj1XpJGo1GE+50ZTZQhyIr3cidwMtSyjcB\nhBA3ogrOXQM82sr5NwHFUspZ7udbhBAnufv5CkBK+TvfBu5Iy4XAFOCfnbxuj5BqSWV4xnCWly73\nOitjxkBtrVrvZtiwUFnWDhIS4Jln4PzzVTW20aPVjKGbbmLywMlMHjgZUPV9isuLKSxVjkthWWHH\nV40+Apg9Wy3VtG9f47Z3b+N+e4oKT5oEdnujg9N8GzVK585oNJrwJuTOihDCDIwHHvIck1JKIcTX\nQL6fZlbg62bHFqPWJvJHAmAGDnXhuj1G8xWY8/LU4w8/hLmz4uGUU1R9+3vugZkz4aOPVJRl4EBA\nVdQdkjaEIWlD+O3o37ary2U7lrFq1yqsuVbGZo0lNjq2++wPE+LjVWDKHZzqFFOnwo4dyrnZtAmW\nLFH71dXq9RdegJtv9t9+xw747DPo06epk5OSEnhxTY1GowkW4TB1OQOIAvY2O74XlUfSGll+zk8S\nQvj7BnsEKKPRyenMdXsMa66VdXvXUVOviqtkZEBOTvDzVjYHu9KcL4mJ6pvw669h61b1E/7llzs9\nh3dl2Ur++H9/JH9ePklzk7C+auWORXewYP0CtldsR0rZvXp6mGBp+fOf4ZVX4JNPVMLv1q1q2YGa\nGti+HS69NHD7DRtUYcKLLlJRmmOOgbQ0iI1V78lx48BmC9yH09nN77UQYCQ9RtICWo8RCQdnpdsR\nQswGLgbOk1JGxHq71lwrTulk9e7GRQe7I8l21qxZbZ/UVaZMUUViLrsMbryx8ad+B/l9/u85PPsw\n31/3PU9MfYKj0o7is58+49KPLmXQM4O46IOLAuqRUvL+hvdZ9Msilu9czsb9GymtLKWqriosa8R0\n972Jj4cBA9SU6kCcdZZapfrgwcbIzAcfqHoz110H+flqllMgLrkERo+exXHHwcUXq+GtV15Rfmxx\nccStkQn00GenhzCSFtB6DElH6/MHe0MNzTQABc2Ozwc+9tNmCfBks2NXAeWtnHsXauhnbBCuOw6Q\nffr0kdOmTWuyWa1W+fHHHzdZB2Hx4sWtrulw8803y1dffbXJsdWrV8tp06bJ/fv3SymldDgdMvGh\nRDnlqily7ty5Ukop//QnKfv2lbKkpEROmzZNbtq0qUkfzz77rLzrrruaHKupqZHTpk2T3377bZPj\n77zzjrzqqqtkSUlJk+MXX3xxUHV4uO+++5SOxYulzM2VslcvWfLII0HRccnll8iFmxfKxb8s9upp\nTcenn38qORrJ/c22CUhRIGTK3BT5+U+fB9Qx8+6Zcvrt0+XCzQvlku1L5Jrda+S3676VZ559pvxx\nw49d0nHVVVc1OVZSUtL996PZ9YL1vmrOpEkXy/POe0XOmCHlqadKOXCglEIsljBNgpTXXBMZOnzv\nh+e9Fon3o/n7qqSkxBA6pFT347TTTjOEDs/98P0/HSk63nnnHe93o+c7c9KkSeGxNlBnEUIUAiuk\nlLe7nwvUdORnpZSPtXL+XOAsKWWez7F3gBQp5dk+x2YBfwSmylamI3fiuj0yG8jDlDenkBSbxMfT\nPwbUr9mLL1YJlr17d/vlu4/Dh+H3v1c5LGeeCf/4B+TmdvtlpZRU1Vdx2H6Yw3WHqayr9O57Hi8Y\nfgFHpR3lt49Xi17luk+v8/t674Te7L2r+chiUxZuWcjB2oMkxyWTFJtEcmwyyXHJ3sfYqFjEEZAM\n0tCgAmzFxWpKdqCPVGmpqk0zaBAMHtx069+/7UrCGo0m9BhhNtCTwHwhxGoapxDHo6IcCCEeBrKl\nlFe6z38JuEUI8QjwGmqGz0WAr6NyD/AAcCmwQwjRx/1StZTSs8pOwOuGGmuOldfWvqa8SiEYM0Yd\n/+EHOP300NrWJZKTYd48uPBCNY4wcqQaU7jyym7N2BRCkBSbRFJsEv3o16k+ZoydwW9H/dbr4FTW\nVTbZb3C1PZ7x/Mrn+ar4K7+v33zczbxwzgudsi+SMJs7ljx83HHKsfnoI7U2k2dJKpMJ+vWDpUuV\n46LRaIxHWDgrUsr3hRAZwF+APsBa4Awp5X73KVnQ+O0ipdwuhDgHNfvnNqAUmCGl9J0hdCNqqOfD\nZpd7wH2d9lw3pFhzrTy07CF2HN7BgJQBDBmiZgavXRvhzoqHs8+G9evhjjtUMbkPP1SJDNnZobbM\nL0IILGYLFrOFrMTO5WF/+bsvaXA2UFlX2cTZ8TwOSQv87b2rahd3LLqDkb1HMiJzBCN7j2RI2hCi\nTWHxce4WcnNVbrYHh0MVyysubtwyMwP38dprsHFj06jMgAEqUVij0YQ5HR03OpI33Dkrq1evbjFW\n1x3sq94nuR/57o/veo+dcIKUl10WvGs0H88MGQsXSpmVJWVKipRvvSWly9WpbsJGTxDwp2XDvg1y\n8uuTZfoj6d68m9gHY2Xe3/PkZR9dJh9a+pCsqqvqYWvbJtT35r77pBw6VEqzWUo1JU1KIaTs10/K\nyZOlfOqpjvUXaj3BxEhapNR6wpXVq1d3OmfFuD/FDEBmQiZDUodQWFrIJSMvAdSMoG++Cd41amtr\ng9dZV5g2rXGO7O9+p6IsL70EWR2LXoSNniDgT8uxmcfyzVXfIKVkX80+NuzfwIZ9G1i/bz0b9m/g\ny61f8ocT/tDD1rZNqO/NAw+ozemEsrKmUZni4sZhJX/U1anAnycqU1Vl/PdapKL1GI+wSLCNFHo6\nwRbg8n9dzi+HfqHwWlUg7pVXVAGvqiqwWHrEhJ7n44/VFGeHA55/Xs17PQISToOFdOc4BWL6h9PZ\neXindxhpZO+RjOg9gj4JfY6I5N7OsG2bqjFT7y5+EBur0q3Gjm3cxo/Xyb4ajT+MkGCr8YM118oH\nGz+gzlFHbHQsY8aoX4AbNqiEQ0Ny/vlw8smq8u1ll6koy9//HuFToHqO9jgbpw48lWU7l7Fq9yre\nWvcWdc46ANIt6YzoPYKZE2bymxG/6W5TI4pBg1Txu1271LIX69bBmjVqQcr585VvvW2bt0izRqMJ\nItpZCXOsuVbqnfUU7S4iv18+I0eq2Q9r1xrYWQFVsnfBAjVj6OabYcQIePFF+I3+Ag0GNxx3Azcc\ndwMATpeTreVbmwwlmUTgepG2Bhv1znqS45J7wtywwWRSyb65uWpFCQ91dSpXfMCAwO0//1z92Bg7\nVlX/1UEsjaZ9aGclzMnrk0dcdByFpYXk98snPl6tDRSsSrYHDhwgIyMjOJ11B7/5DUyerByWiy+G\n6dPV0JAfm8NeTwfoKS1RpiiOTj+ao9OP5vzh57erzRe/fMGF719Iv6R+jOg9gpGZahhpZO+RDM8Y\nTkJMQos2Rro30FRPbKwaAmqLZ56BL79U+5mZTYeQxo6Fo45SDlFPY+R7YwSMpqcz6JyVDhCKnBWA\nk18/mexe2bx30XuAGhnZsQOWLet63wUFBSxcuLDrHXU3UsJ778Ett0B0tEq+Pb/lF2vE6GkH4axl\nV9Uu/rPtP2zYt4EN+1VEZlvFNgAEghG9R7DuxnVNhqSa69lWvg2bw4bD5Wh1G5gykMGpg/3acLD2\nIB9s/MBve4fLwe3H305mgv85ze9veD9gHwNTBvLW+W+12taj54FvHsDusJOVmEWfxD70Sejj3U+N\nS23yN5BSTbles6ZxKypSRe8A7r0X/vKXdt2CoBLO77XOoPWEJzpnxeBYc6y8v/F97/O8PLUKrsvV\n9V9h999/f9c66CmEUIm2p5yikm8vuEB5bc8+q8qfuokYPe0gnLVk98rm8tGXNzlWXV/Npv2b2LB/\nA+W28ha5M831nPPOOWw6sMnvNeZMnsP9p9zv9/U91Xu49YtbiTZFE22KJkpEefc92xV5VwR0VmwN\nNirrKr3nx5vjm7Tvl+S/eKBHT9GeItbtXcfe6r3YHE1XdDSbzDw+9XFuO/42QL2N+/dX269/rc6p\nd9azY3cNxRtT6N8/8LjQ3r0qXyYvT60TGizC+b3WGbQe46EjKx0gVJGVjzZ+xEUfXETZ78vI7pXN\n4sWqSv0vv7S/+qehkBLefhtuvVWtoPfyy1BQEGqrNB1k9a7V1DnrWjgYni01LpVUSxurLIYR0r2c\nw97qveyt2cue6j3srd6LNdfK+Gz/Y0RLti/hlDdOISYqpklUJivB/ZiYxYyxM7CYLbz2GsyYoZye\no49uOYx0hI8UaMIcHVnpaWw29YXZQ9lx+f3yASgsLeSC4Rd4y+6vXXuEOitCwOWXw6mnwg03qJ+o\nV1yhSva3tYSwJmwI9AUeifgu5zA0fWi72w3PHM57F73H3mq3g1OjnJ21e9eyd6vav3bctYB6248b\n13QY6bPPoHrwW5D+M/1S+/Dkg26Hx+34JMYk6unomohHR1Y6gDeygiplS3y8qn+fkNC4f8op8Oij\ngTt6+21VjKG19gkJau2cZsUa+j/Vn0tGXsKjp6u++/aFa6+FBx/sBqGRhJTw5ptw++3qb/ePf6gy\n/hqNQWirbo7LBTPe/z0Lt37AYcdenDRdn8oSbeHKvCv5+7l/B9T06v79ISqq6TUO2Q6RHJds6GUb\nNKFFR1Z6mr/+VdX8qKmB2lr16Nlvz7o2V12lijL446231E8oH6y5VgpLVWE4VqzgPfkEDfPj4VAr\nzk58vKoC6/vfyA/z5s1jxowZbdscrgihFkCcMgWuu45555zDjGuugSefVE5fBBPx96YZWk/naCsq\nYjLB65c8CTyJlJJye3mTKM2e6j0MShkEqJWujzlG5ajn5TUOH2386UWeSpgJQKI5kRRLCilxTbd7\nTryHkb1HUlKipmm7XGqTUj3WOxsQ0kRsTBTnnRdY06efqirCvu19t7w8OO00/+1rauDhh1tv63LB\ntm3zePrpGYapeWO0z05n0M5KZzjrrMDr2bdFcwen+X4rcyCtuVb+/J8/0+BswFxfT9+4cg7vLoXv\nWmnvcKhhkUBccgl8+ilFLhcz/vrXls7OqafCHwKUbJcSFi1qbOfbPiFB5ZL0ZOg5Nxc+/5yiKVOY\n8cEHan7ovHkwdWrP2RBkioqKDPUPSuvpfoQQpFnSSLOkMTxzeCuvwxdfNA4hffONmljnkj/CsH9B\nXAW/+30FGbnlVNgrvFtJRQn1TlW699//VpPyWjDhH3DOLYi6JPptb+ns5PTK4aEpDwFqCvc33yhH\ny2RSdplMIEySKJNgxozAzkpdHfzzny3be/aLi4vYt884zko4vtd6Gj0M1AFClWALsHznck547QRW\nXbeK8dnjWbAALr0UDhxoMhlGUV/fds3vRYtg0yb/jlN+Psye7b99XZ1ySPwhhCqb75ny0BorVqgo\nUmvOTny8mu5w5pmBdbTGjh0qC/Hrr+G66+DxxyEpqeP9aDRHALW1amhISvVlP2CA+gj6o6YGKitb\nOgq/VGxm5Z7vqKyvwOZSTk65vdHpiTfH8+Xvvgxoy4mvnciGfRtItaS2cHZSYlOYOmQqZw09q01N\nnq+1QL+XHntM5fuMHKm2UaNU7Umd9tZ96GGgI4CxfcdiNpkpLC1kfPZ4b5LtDz+oIEgT2rM4yZln\nds4R8GA2q4IRgaJEeXmB+9i3D779tmVbz4pyFos6Hojf/Q6++66ls5OWBiecoPJZFi+G115TQ0XN\ncTpVpnJzhyk2VpcX1RwRxMerL+n24vmINGdi2jFMHHxMl2y54/g72F6xvTGqU6cetxzYQoW9gpyk\nnIDOytZDW5nwjwkkxiQSExWDOcpMTFSMdzObzMwrmMeQtCEMHqxy/5YsUWuueUbm0yd8TdKY/2PY\nUWZOtDa28/SRHp/OBcMvCKjjl0O/4JKuJu18bYkSUTrpuYNoZyVCiIuOY2zfsRSWFXILtzB0qPou\nX7u2FWelJ/DUHe8K06apzRcp1cB6TY2addUWZ5wB/fq1dHgOHlR93Xyz+iOddhrcdJNKfvYtUFFZ\n2fq6BSZTo/Py3nuqiq4/Vq1SsXV/CdO9euH1LjUajV+6uh5VUmwS95x4D9X11TS4Gqh31tPgdD+6\nn8dGxwJqJY8LL1Tt6uthyxaVi/P6hk1853qfA7X1FK1sbO/pY3jGcC4YfgFSqt860a18ixa8WxCw\nhtB9k+7jgV894Pf14vJipn843evsmKPMRJuiMZvM3v3HT3+cfsn+6wAt2b6E73Z+520XbYrGHGX2\n7qfHp3Pu0ecG/HtuPrAZh8vRantzlBlLtMX79+xutLMSQVhzrPz7538DKnd21CgVWTEUQqjIUExM\n++KxzRKRW8XlUgPzd9+thr9ee61xYZdevVQJ0UA5RP37B+5/40a1bpGnncvV9PX0dDVeF4hrr1X/\nKVtzduLj1cKO55zjv73TqVbY87SJidGRIc0RR2ZCJvecdE+H28XEqP+no0bBpdwK3NrqeVJKnFJF\nfktLYehQOPbYpkNJI0fC2xe8Q3V9ldfJ8XWW6p31jO4zOqA9sVGxjMsa1+hwuRpwuBw0OBuobail\nwdmAS7oC9rGibAVPFz6t2rkaaHCqPjz2j+w9sk1n5bwF57Hl4Ba/rz9wygPcN/k+v69vPrCZqW9N\n9TpYjtIAE0vaQOesdIBQ5qwALFi/gEs/upR9d+0jMyGTG26AwsKuOSxGKePsIaCerVvhmmtg6VJV\nUO7hhwMPzncGKdXPNF9np6FB/QcLxJNPqqW0fZylgjVrWJibq55ffjk84P+XGGVlTSNdUVEtk5/f\nfDNwhGftWpVH1Jqz5IkQtWe2mx+OqPdahGEkLdAzeg4eVFUofvxR/c5Yvx6qq9VrycnqI//BB2qo\nqasEU49LupTT4nJiMVsCnrt+33oVpXI2NHGYPPvDM4Yzorf/McQ91Xt4YeULXoepbEsZC25bADpn\nxdhYc62A8pjPPfpcxoxRQYK6OpVi0RlmzpwZRAtDT0A9Q4bA//t/aiHE2bPVErivv66iFsFCCHUz\nYmNV3kx7+f3vWxya+eWX7Z/NlJamhqIC5RClpATu4z//gbvuasxObE7//lBSEriPW29VCc6tODsz\ns7Nh+XKVvO0Pl0slD5jNYR8ZMtJnx0haoGf0pKfDbbc1Pne51Ft//fpGB6bF5IdmlJWpj2Vbv5mC\nqcckTMRExUDblS0Y2buNH1ltkJWYxYOnNhYDK8ooYgELOtWXjqx0gFBHVqSU9H2iL9eOu5a/nvpX\nli9XOaRFRapWgqYD/PwzXH01/Pe/cMcdqnZOfHyorQo9UirvtzVnB9p27O6+GzZv9j+kdscdgSsZ\n/vyzqiMfFdVydphn/6WXApdu3rhRJSD4ax8f374kdI2mmznzTFVlYfDgxiEkz+PQocpnNxJ6NtAR\nghCiSXG4UaPUj8+1a7Wz0mGGDlXTAJ55Bv70J1U8Yv78wL/6jwSEUFPS4+La/lnYGo891rXr9+4N\nb7zR0tnxfd6Wo/Hee4GXLh4+XDk0gZg9W8X5/Tk7Y8eqZAV/HD6songeWvtReNppamjNH+vXK8fP\nH8nJcPrpgXV8/jlUVfm3wzNf1x/l5aoPf+1BrcsVqDTAmjWwbp3/PlJTA5c4APjwQ/U39cf48YGH\nOA8ehI8+8m8DwPTpgaOPK1a465f76SM9XdWvCsSbb0JFhffpvNGSnamwezfs3gSfLs3nr4cmAupt\nfv/98Mc/+rTfv18VmHFTXS1Zu0b59lHREB0FO6ZcjUhL9ab+Wa3NIjfffqvyB9w01EukVO1NAkTv\nTPVDLhAvvwyHDvn/W0yaBCedFLiPjiKl1Fs7N1SVfbl69WoZKuZ+O1cmPpQoHU6HlFLKoUOlvP32\nkJljDDZtkvL446U0maS8+24pbbZQW6TpCna7lPv3S1lSIuXGjVJ+/72U33wj5eefS/nBB1J++mnb\nfVx2mZQTJkg5YoSUAwdKmZkpZUKClEJICVL+7W+B269dq84LtG3ZEriPP/0pcPvRo9vWMXRo4D4e\nfPDI0WEyNd2ioppubem4914pzeamW0xM4zZ+fNs6Ro2S0mJpusXHN25z58r9+9Xb9bnnpFy6tFn7\ndeuk7NXLuzkSeskKkppsR/FTkz/NTz816+Ovf5UyNdW71cSlygOkebel4mSZmKhe7tNHyqlTW9Ex\ncaKUGRnerTIuQ1bEZsrDsZnycFymfD//CXnrrVL+4Q9S/vGPSo+UUq5evVoCEhgnO/j9q4eBk2Sa\nHgAAIABJREFUOkCoh4GgcYXWdTeuY1SfUVx8sSpX8s03nevvk08+4by2amNHEJ3W43TCE0/Avfeq\nmOz8+XD88UG3ryPoexOGSKmm1JtMfLJokX89DkeTX9BAyxyclJTAS2LYbGpIzl8fJlPgyAyoSFTz\n//G+fZjNEBPj/964XIFtAPXz3WTyb4PT2XKGXPM+Wpv/29yO5m0C5DQZ4r3mQ3v0OJ0qt7++XuX0\ne/ZzcgIPJ61bp4J4vm18+8jIgBtvDGzf1Verslv++rjrLlVFoivDQCGPVkTSRhhEVqrrqqXpAZN8\nZdUrUkr1Ay85WUqXq3P9XXzxxUG0LvR0Wc+GDeoXtckk5ezZ6ld6iND3Jrwxkh4jaZFS6wlXdGSl\nhwiHyArA2JfHMi5rHPN+PY/PP1flN7ZtwzDrYIQch0PlXsyZo5I9589vvXCcRqPRaNpNVyIrAWJ3\nmnAlPzef5aXLgcaK9mvXhtAgoxEdrbLaVq9WU5CtVjU8VF8fass0Go3miEQ7KxGINdfKpgObqLBX\nkJ2txhQNV8k2HBg1SmXNz5kDc+fChAlqZoNGo9FoehTtrEQgnuJwK8tWIoSasacjK92E2ayiKt9/\nrxL6Jk5UlWQbGkJtmUaj0RwxaGclAhmaNpQ0S5q33kpXnJWr25pPH2F0m54xY2DlSvif/1FFzSZO\nbFo7ohvQ9ya8MZIeI2kBrceIaGclAvEUh/PkrYwZA9u3t5wp2R6mtrece4TQrXpiYlRUZeVKlYR7\n3HHwt781ri0fZPS9CW+MpMdIWkDrMSJ6NlAHCJfZQAAPLnmQpwqf4sCsA2zcYGLUKFWQddKkkJp1\n5FBXp6qkzp2rqpm+8UbgSqAajUZzhKNnAx2BWHOtlNvL+fngzwwbpiat6LyVHiQ2VkVVCgtVGfhx\n45Tj0k1RFo1GozmS0c5KhDIxZyICQWFpIWazWvhKOyshYMIEtZLkHXeoNYZOPBE2bQq1VRqNRmMo\ntLMSoSTHJXNs5rFN8lY646wsW7YsyJaFlpDoiYuDRx6B775Ti62NHQuPP67qX3cBfW/CGyPpMZIW\n0HqMSNg4K0KIW4QQ24QQNiFEoRBiQhvnnyKEWC2EsAshfhJCXNns9WOFEB+6+3QJIW5rpY857td8\ntzaWYw0ffFdgHjMGNmzo+IzaRx99tBssCx0h1WO1qjosM2fCrFlw8snw00+d7k7fm/DGSHqMpAW0\nHiMSFs6KEGI68AQwBxgL/AAsFkJk+Dl/IPAZ8H9AHvAM8KoQwne99HhgK3APsDvA5dcDfYAs9xbk\nda27D2uulR/3/Uh1fTV5earAaqAV5VtjwYIF3WNciAi5HotFRVW+/VYt556XB08/3XIht3YQci1B\nRusJX4ykBbQeIxIWzgpwJ/CylPJNKeVm4EagFrjGz/k3AcVSyllSyi1SyheAD939ACClXCWlvEdK\n+T4QqE66Q0q5X0q5z70dCo6k7seaa8UlXazatYrRo9Wxjg4FxcfHB9+wEBI2ek48UZUVvuEGuPNO\nOOUU+OWXDnURNlqChNYTvhhJC2g9RiTkzooQwgyMR0VJAJBqPvXXQL6fZlb3674sDnB+IIYKIcqE\nEFuFEP8UQvTrRB8h4djMY0mKTWL5zuUkJ8PgwTrJNqyIj1dRlW++gbIyGD0annuuU1EWjUajOZIJ\nubMCZABRwN5mx/eihmVaI8vP+UlCiNgOXLsQuAo4AxXNGQQsFUIkdKCPkGESJibmTKSwrOuVbDXd\nyOTJqtrtjBlw221w6qlQXBxqqzQajSZiCAdnJWRIKRdLKT+SUq6XUn4FnA2kAheH2LR2Y81RSbZS\nSvLylLPSkTp/d999d/cZFwLCVk9Cgoqq/Oc/UFKioix//3vAKEvYaukkWk/4YiQtoPUYkXBwVg4A\nTlSSqy99gD1+2uzxc36llLKus4ZIKQ8DPwFHBTrv7LPPpqCgoMmWn5/PJ5980uS8L7/8koKCghbt\nb7nlFubNm9fkWFFREQUFBRw4cKDJ8Tlz5vDII480ObZjxw4KCgrYvHkz1lwr+2r2sb1iO6Wlz3Ho\n0N2UlTWeW1tbS0FBQYupb++++y5XX301/fv3b3J8+vTpIdHhy3PPPdfiw9mWDg8ePWGr4/jjKRg+\nnGWnnQY33wxTp0JJSQsdHi1hq6Od98PD9OnTqWi2HkSk6vDcD897LdJ1eLQYQQeo+7Fo0SJD6PDc\nD9//05Gi49133/V+N2ZlZVFQUMCdd97Zok17CYty+0KIQmCFlPJ293MB7ACelVI+1sr5c4GzpJR5\nPsfeAVKklGe3cv424Ckp5bNt2JHovu59UsrnW3k9bMrtezhQe4DMxzJ5+4K3OSn5MgYMgE8/hXPP\nDbVlmjb56is1NFRRAU88Addeq1Z21mg0GgNihHL7TwLXCSGuEEIcA7yEmno8H0AI8bAQ4g2f818C\nBgshHhFCDBNC3Axc5O4HdxuzECJPCDEGiAFy3M+H+JzzmBBikhBigBDiBOBjoAF4t3vlBo+M+AyG\npg2lsLSQfv0gNVXnrUQMp58OP/4I06fD9dfDmWfCzp2htkqj0WjCjrBwVtzTi+8C/gKsAUYDZ0gp\n97tPyQL6+Zy/HTgHOA1Yi5qyPENK6TtDKNvd12p3+7uAIuAfPufkAu8Am4EFwH7AKqU8GFyF3Yun\nOJwQOsk24khOhn/8A774QlX1GzkSXn+9Y4lHGo1GY3DCwlkBkFK+KKUcKKW0SCnzpZSrfF67Wkp5\narPzl0opx7vPHyqlfKvZ6yVSSpOUMqrZdqrPOZdKKXPdffSXUl4mpdzW/WqDizXXypo9a7A12MjL\nU+U92kvzcctIJ2L1nHkmrF8PF1wA11wD557L5iVLQm1VUInYe+MHI+kxkhbQeoxI2Dgrms5jzbXi\ncDko2l3EmDGq9lhVVfvazpo1q3uN62EiWk9KioqqfPoprFnDrNNOg7feMkyUJaLvTSsYSY+RtIDW\nY0S0s2IARvcZjSXaQmFpIWPGqGPr1rWv7fPPt8gjjmgMoefcc2H9ep6fNg2uuAJ+/WvYHWjFiMjA\nEPfGByPpMZIW0HqMiHZWDEC0KZoJORMoLCtk+HAwm9uft9J86nKkYxg9aWn0/9e/4JNPYOVKGDEC\n3nknoqMshrk3boykx0haQOsxItpZMQie4nAxMep7TSfZGoRf/1ol3p55Jvz2t3DhhbC3efFmjUaj\nMTbaWTEI1lwrpZWllFaWdjjJVhPmpKerqMqHH8KyZcobff/9UFul0Wg0PYZ2VgyCNdcK4M1b+fFH\ncDjabte8umGkYyQ9LbRceKGKsvzqV6o2y8UXw/79rTcOQ4x0b8BYeoykBbQeI6KdFYPQt1dfBiQP\n8Dordjv89FPb7Wpra7vfuB7ESHpa1ZKZCR98AO+9p9YZGjECPvqo543rBEa6N2AsPUbSAlqPEQmL\ncvuRQjiW2/flkg8vobSylE/PX0ZaGrz9Nlx2Wait0nQbe/fCTTfBxx/DpZeqhRLT00NtlUaj0bSK\nEcrta4KANdfKql2rSEiqp39/nWRrePr0UVGVt9+GRYtUlOV//zfUVmk0Gk3Q0c6KgcjPzafOWccP\ne35gzBidZHtEIIQKn23YABMnwnnnqdos5eWhtkyj0WiChnZWDMSYrDHERMV481bWrGm7LEfzJcUj\nHSPp6ZCWvn1VVOWNN2DhQhVl+eyz7jOuExjp3oCx9BhJC2g9RkQ7KwYiNjqWcX3HUVimnJX9+2HP\nnsBtrrnmmp4xrocwkp4OaxFCRVU2bICxY2HaNLj6aqio6B4DO4iR7g0YS4+RtIDWY0S0s2IwrDlW\nlu9c7i2731beyv3339/tNvUkRtLTaS05OSqq8tpr8K9/qZWcFy0Kqm2dwUj3Boylx0haQOsxItpZ\nMRj5/fLZVrENS8ZekpLazlsJx1lNXcFIerqkRQgVVVm/Xg0JnXUWXHcdVFYGz8AOYqR7A8bSYyQt\noPUYEe2sGAxPcbiVu1aQl6dnBB3x9OunoiqvvAILFqgoy9dfh9oqjUaj6RDaWTEY/ZL60TexrzfJ\nVjsrGoRQUZX162HoUDj9dLjxRqiqCrVlGo1G0y60s2IwhBBYc61eZ+Wnn6Cmxv/58+bN6znjegAj\n6Qm6lgED4Kuv4MUX4Z//hFGjVBXcHsJI9waMpcdIWkDrMSLaWTEg+bn5rCxbycjRDqRU6wT5o6io\nQ0UEwx4j6ekWLSaTqnr7448waBBMmQIzZ0J1dfCv1Qwj3Rswlh4jaQGtx4jocvsdINzL7Xv4tuRb\nJs2fxIqr13LC4DxeeAFuuCHUVmnCDpdLRVnuuQeysuD112HSpFBbpdFoDIout69pwvjs8USJKNbs\nK2T4cJ23ovGDyaSiKj/8ANnZcMopcMcdoBdN02g0YYZ2VgxIvDmevKw8b3E47axoAnLUUfDNN/Dk\nk/DyyzBmDHz3Xait0mg0Gi/aWTEo+bn53uJw69aB0xlqizRhTVSUiqqsXQsZGXDyyXDXXWCzhdoy\njUaj0c6KUbHmWtlycAuDjj1EbS388kvr5xUUFPSsYd2MkfSERMuwYfDtt/Doo/D886psf2FhULo2\n0r0BY+kxkhbQeoyIdlYMiqc4XEPvlYD/SrYzZ87sKZN6BCPpCZmWqCgVVVmzBpKT4cQTYfZssNu7\n1K2R7g0YS4+RtIDWY0T0bKAOECmzgQCklPR+vDe3TLiFVy+/nyuugIceCrVVmojD4YDHH4c5c1Ru\ny/z5MGFCqK3SaDQRiJ4NpGmBpzjc8tLlOslW03mio1VUZfVqsFggPx/+9Ceoqwu1ZRqN5ghCOysG\nxppjZUXpCvLGuLSzoukaI0fC8uXwwAPw2GNw3HGgC1VpNJoeQjsrBsaaa+Vw3WEyjtnC7t2wb1/L\ncz755JOeN6wbMZKesNNiNquoyqpVKuIycaIaHqqvb1fzsNPTRYykx0haQOsxItpZMTATciYgENRl\nqNkcrSXZvvvuuz1sVfdiJD1hq2X0aFi5Eu69VyVCTZzoP4Pbh7DV00mMpMdIWkDrMSI6wbYDRFKC\nrYdRfx+FNSefdy97hTlz4O67Q22RxlCsWQNXXgmbNsF996n8FrM51FZpNJowRCfYavySn5vPirJC\n8vJ0kq2mGxg7Vg0LzZ6t8lmsVli/PtRWaTQag6GdFYNjzbWyft96jh1bpZ0VTfcQEwMPPqiKx9nt\nMG4cPPywmvas0Wg0QUA7KwbHmmtFIuk17Hu2bNHV0zXdiGeG0B/+AH/+M5xwAmzcGGqrNBqNAdDO\nisE5JuMYkmOTsaUvx+mEDRuavn711VeHxrBuwkh6IlJLbKyKqvz3v1BVpaIsjz0GTmdk6gmAkfQY\nSQtoPUYkbJwVIcQtQohtQgibEKJQCBGwTKYQ4hQhxGohhF0I8ZMQ4spmrx8rhPjQ3adLCHFbMK4b\naZiEieNzj6fEVYjJ1DJvZerUqaExrJswkp6I1nL88SrKcuutcM89cNJJTB05MtRWBZWIvj/NMJIW\n0HqMSFg4K0KI6cATwBxgLPADsFgIkeHn/IHAZ8D/AXnAM8CrQojTfU6LB7YC9wC7g3HdSMWaY+X7\n3YUcPUy2cFYuvfTS0BjVTRhJT8RrsVhUVGXZMjh4kEv//Gd48knDLAEe8ffHByNpAa3HiISFswLc\nCbwspXxTSrkZuBGoBa7xc/5NQLGUcpaUcouU8gXgQ3c/AEgpV0kp75FSvg/4q1rV0etGJNZcKwdq\nDzDkuGKdZKvpeU44QYX0brpJLZA4eTL8/HOordJoNBFEyJ0VIYQZGI+KkgAgVfGXr4F8P82s7td9\nWRzg/GBdNyI5Pvd4ABKPWc66deByhdggzZFHfLyKqixZAnv2QF4ePPusfjNqNJp2EXJnBcgAooC9\nzY7vBbL8tMnyc36SECK2G68bkaRZ0hiWPozatEKqqmDbtsbXli1bFjrDugEj6TGSFnDrOflkVe32\n2mvh9tvhV7+C4uJQm9YpjHR/jKQFtB4jEg7OSsRx9tlnU1BQ0GTLz89vsX7Dl19+SUFBQYv2t9xy\nC/PmzWtyrKioiIKCAg4cONDk+Jw5c3jkkUeaHNuxYwcFBQVs3ry5yfHnnnuOu5uVqK2traWgoIBB\nlYMocamy+2vXqvLNV199NY8++miT86dPnx7WOpp/aD06PHj0RLoOjxYj6AB1P+680z1Km5AAzz7L\nl48+SsH336vy/S+84I2yhLsOz/3wvNci9X74vq8effRRQ+gAdT8uuugiQ+jw3A/f/9ORouPdd9/1\nfjdmZWVRUFDQ+D+gE4S83L57OKYWuFBKudDn+HwgWUp5fittlgCrpZS/9zl2FfCUlDK1lfO3uV97\ntovXjbhy+x5eWvUSt35xK+mvHOa6q+J58EF1vLa2lvj4+NAaF0SMpMdIWsCPnupqNVvoxRdVlOW1\n12DgwJDY11GMdH+MpAW0nnAlosvtSykbgNXAFM8xIYRwP/+vn2bLfc93M9V9vDuvG7FYc604XA4G\nWFc3SbI1wgfAFyPpMZIW8KMnMVFFVb7+GrZuhVGj4OWXIQLWLDPS/TGSFtB6jEjInRU3TwLXCSGu\nEEIcA7yEmno8H0AI8bAQ4g2f818CBgshHhFCDBNC3Axc5O4HdxuzECJPCDEGiAFy3M+HtPe6RmJk\n75EkmBOIH1bYngVyNZqeZcoU+PFHuOwyuPFGOOMM2LEj1FZpNJowISycFff04ruAvwBrgNHAGVLK\n/e5TsoB+PudvB84BTgPWoqYgz5BS+s4Qynb3tdrd/i6gCPhHB65rGKJN0UzImUBtaiE7d8LBg6G2\nSKNpRlKSiqosWqRWcR45EubNi4goi0aj6V7CwlkBkFK+KKUcKKW0SCnzpZSrfF67Wkp5arPzl0op\nx7vPHyqlfKvZ6yVSSpOUMqrZ1rwfv9c1GtYcK9scywHpja40T6iKdIykx0haoAN6zjhDrdz8m9+o\nWUNnnw2lpd1rXCcw0v0xkhbQeoxI2Dgrmu7Hmmtlv303sb13evNW+vfvH1qjgoyR9BhJC3RQT3Ky\niqr8+9+wbp2KsrzxRlhFWYx0f4ykBbQeIxLy2UCRRCTPBgLYW72XrCeyGFL0HiemXMwbb7TdRqMJ\nOeXlcMcd8OabcO65aqgoOzvUVmk0mg4S0bOBND1Hn8Q+DEoZRPzRhbrsviZySE1VUZX//V9YtUpF\nWd5+O6yiLBqNpnvRzsoRhjXXSnVKIRs3Ql1dqK3RaDpAQYHKZTnrLLj8cjj/fFW6X6PRGB7trBxh\nWHOtlDpX45B1bNxIiyqGkY6R9BhJCwRJT3q6iqr861+wfDmMGAELFoQkymKk+2MkLaD1GBHtrBxh\n5Ofm0yDroe9a1q6FWbNmhdqkoGIkPUbSAkHWc/75sGEDnH46XHqpmjm0b1/w+m8HRro/RtICWo8R\n0c7KEUZeVh6xUbFkjFHF4Z5//vlQmxRUjKTHSFqgG/RkZKioyvvvq9WcR4yADz8M7jUCYKT7YyQt\noPUYEe2sHGHERMUwPns8lqEqydZoU+KMpMdIWqAb9fzmNyrKMmmS2r/kEmi2oFt3YKT7YyQtoPUY\nEe2sHIFYc6xUJS9n7Vo9oUJjEHr3VlGVd9+Fr75SUZaPPw61VRqNJkhoZ+UIJL9fPhWUcNi5m5KS\nUFuj0QQJIVRUZcMGyM+HCy5Qs4YOHQq1ZRqNpotoZ+UIxJprVTu5K7jvvkdCa0yQkFJSYa9g7ty5\noTYlaDzyiDHujYce05OVpaIqb72lKuCOGAGffhr0yxjp/hhJC2g9RiQ61AZoep7cpFxyeuVwaGgh\nO3bEhtqcNqlz1LGrahdlVWWUVZZ5H3dV7/I+31W1C7vDjmWZhe8Hf8/kAZOZNGASo3qPIsoUFWoJ\nnaK2tjbUJgSVHtUjhIqqnHoqXH+9qtFy5ZXw9NOQkhKUSxjp/hhJC2g9RkSX2+8AkV5u35eL3r+I\n/yvcj5i/hGHD1IK3SUnQq1fjfltbr15gNnfeBiklh2yHWjghZVVN9w/UNk2WjDfHk9Mrh5ykHPXo\n3s+Mz2Tj/o0sKVnCirIV1DvrSYlL4aT+JzGp/yQmD5zM2KyxmKO6YLQm8pBSVcC9/XZITIRXX1WF\n5TQaTY/SlXL7OrJyhJKfm89nqfdy/RUOaqqiqayEykq1uK1nv7ISqqoCJ+FaLK07OQlJdUSl7EL2\nKsMZX0Z93C5s0WVUizIqXGUccpRxoG4XdU67ty+BoHdCb68Tkp+b3+iQ+DwmxyYjhAioz+6ws6J0\nBUtLlrJ0x1LuX3I/tV/XkmBO4MT+J3qdlwnZE4iNDv/okqYLCAFXXQWnnQbXXadWcb7mGnjySbVg\nokajCXt0ZKUDGCmy8t2O7zjp9ZMour6IsX3H+j3P5YKaGl8HRlJ26BDbD5VRWlnG7uoy9trKOFhf\nRoWzjErKqI0qo97cbOpofTxU5UBljnqsym7cr8zBVJNDkqkvyYnmdkV0WjuenKweW6PeWU/R7iKW\nbF/C0h1LWbZjGZV1lcRGxWLNtXqHjfL75RNvjg/iX1oTVkgJr70Gd97ZuLLz1KmhtkqjOSLoSmRF\nOysdwEjOiq3BRtLcJB6yPsTdp98NNOaGtMgP8dn35IZ4aB4NaR4F8Y2G1NeLJlGb5hEcf68132pq\n/Ovq0+cA+fkZTJwIEybAcce1nqLgdDn5Ye8PXudlaclSDtkOYTaZOS77OK/zcmL/E0mK9eMBdTMH\nDhwgIyMjJNfuDsJKz44dMGMGfP21yml5/HHlBXeAsNLTRYykBbSecEU7Kz2EkZwVgAn/mMD6Z9Yz\nbOawVnNDLNGWlo5HMyekb2LfHs8BcTqhurqlE3PoENx3XwEDBizk++/VMYBhw2DixMYtLw9im438\nuKSLjfs3srRkKUtKlrBk+xL21uzFJEyMzRrLpAGTmDxgMif1P4n0+PQe0VlQUMDChQt75Fo9Qdjp\nkRJeeQX+8AdVDXfePJgypd3Nw05PFzCSFtB6whXtrPQQRnNW3l73Ni9/9jLHjj621YhISlxKm7kh\n4UZRURHjxo3D5YKffoKVKxu3tWuhoUElBY8Z09SBOfpoMPlM5JdS8vOhn5s4LzsrdwIwqvcor/Ny\n8oCTyUrM6lYtRiFs9WzbpqIs/+//wc03wyOPqETcNghbPZ3ASFpA6wlXtLPSQxjNWTnSqKuDH35o\ndF6+/x48i5kmJalhI18HJju7afuSihKWlCzxOjC/HPoFgGHpw5g0YJLXgemX3K+HlWm6jMsFf/87\nzJoFffrA66/D5MmhtkqjMRTaWekhtLNiPCoqYPXqRgdmxQrYvVu9lpPT6Lh48l98J4/sqtqlZhu5\nnZeN+zcCMDBloDfnZfKAyQxOHRxxEaojlq1b1UyhpUvh1lvh4YchISHUVmk0hkA7Kz2EdlaODMrK\nmg4fff+9SgAGOOaYptGX0aMb81/21+xn2Y5l3ujL2j1rkUiye2U3cV6OyTjGMM6LlBKbw8Zh+2Eq\n7BUcrnM/tvI8JiqGwamDGZw6mEGpgxiUMgiL2RJqCS1xueD552H2bBVemz8fTjop1FZpNBGPdlZ6\nCCM6K/PmzWPGjBmhNiNodIcelwu2bGnqwPzwg8p/iYlpmf8ydKjKf6mwV/Ddju+8zsuqXatwSieZ\n8ZmcPODkNqvs9sS9cbgcVNZV+nUwmjgfda2f0+BqaLVvkzCRFJtESlwKybHJ7Pl2D+XDy6l31nvP\n6ZvY1+vAeB2ZlEEMTh1M3159MYkQrgjy88+qPsvy5XDHHfC3v6nCQm6M9NkxkhbQesIVXRRO02mK\niooM8SHw0B16TCYYPlxtV16pjtntymH5/nvlvHz1lfoxDmqoSOW/pDBx4jncOfEcHj0dquurWb5z\nudd5ufuruwNW2W1Li5SS2oZar/PQZnSjlePV9dV++7dEW5SjEZfsdTgy4jMYkjqkxXHPc9/9xJjE\nJs7GLT/ewnN/eo5dVbsoLi9mW/k2isuLKa4oZmv5Vr4q/oo91Xu858dGxTIoVTkug1MGN+67HZpe\nsR2batxhhg5Vw0HPPAP/8z9qnaH589UiiRjrs2MkLaD1GBEdWekARoysaIJHRQWsWtU0/2WP+7s3\nN7dp9GX8eIiJb6yyu6RkCf/d+V9sDpu3yu64rHHYHXa/EY3DdYdxuByt2mISphaORJPn/o67nyfF\nJhETFdODfz1FbUMt2yu2Kyem2batYhu1DY1rpGTGZ3qHlAanNEZmBiYPpo8lF6cjioYGqK8n4GN7\nzkks3cyZ711F39LvWXb8H/jC+hfqTXFIiSG2mBgYMkTNihs2TG0DBkBUZC6rpQlT9DBQD6GdFU1H\nkLL1/JfqalUBvnn+yzEj6vnxwGrvEgHr960nMSaxqSMR69/B8H2eYE4IWl6My6W+tOvr1Yyqzu57\ntvY4B6091jdIbKZ91MQUY48rxm4ppj6xmIaEbTiTipG9SkG4/585o6FiIFQMgvLBLTd7+xYzjI5W\nX+RmM8SZncyse4K7q+6l1DyYZYlnUCcs1JviqDM1Pnr266N8XotqPNYQpY41RMXhMMUgTAIhCOlm\nt8Mvv6iRL7u75mNMDBx1VKPzMmxYozOT3jOlhjQGQzsrPYR2VjRdxelsPf/F4VBfDmPHNjovI0eq\nNl1xEDz7XenD0Xrwpt2YTCoJ2WxWGj1f/p7nHXkM9Jow11FlKqFCFHNIFnPItY39zmL2NRSzt34r\ntc4qr01J5lRyEgbRv9dgBiQN9kZmhqQNZnB6f+LjzKrP1vy9jRvhrrtUFVy7HWw2tXn2O4IQKg8m\nLq7pY2vHAr3W0T78rEDqcilZW7aoOkVbtjTu79jReF56etMojMeROeqolgUXNRoP2lnpIbSzoukO\n7HZVsM4TeVm5Un05tBdfJyA2Nrj7wegjHIYSPCt8e4aTmg8x7Ti8A6d0AmoIrV9SP7+JvxnxGf6j\nVlIqD6+5A9Ps0VVbg8tWi6u2Bul+Lm21SJtNPdrtYKsFex3YbAibDex1CLsdUVeHyWaBoDyJAAAS\nIUlEQVRH2Osw1dVjstcRVd96krM/XFEmHDHRNMRE44iJxhFrBosFkyWe6IRexCQkEZOYhIizeB2f\nhqg4Dtks7KuMY0+lhbKDFnbuj2PbXgvltjhsWKgXcaT0tZA1yELOkDj6HW1h0PA4hoy00HewBREd\nBm8GTY/jki52Ve1i0beLuO7c60A7K92LEZ0Vo5Rx9mAUPeXlSstTTy0M6AyYzU0r74Yz4XxvGpwN\n7Kzc2Zj060789ewfsh3ynpsYk8iglEHseGkHg24ehMPlwOlyqkepHts6Jgnu/13hglgnxDnA0gAW\nR+N+nEM97+WMIt5pIsFpwuIQJDhMWJwm4h3wepGN649yEtfQ2C7eIUgihiSXmURnFPEOE3EOiG1w\nEl3nIKq+AVMHnaQGommItuCKiUNYLEQlxGFOthCVENzoUsFNN7FwwYLG1yLkQ2Kzqc++73b4MDz9\ndAFz5y4kJQVSU9V6Z8nJapgyXLA12Lyfl63lW5s8bivfRp2zDnYBrwB6NpCmo8ycOTPUJgQVo+hJ\nTYV7753JcceF2pLgEc73xhxl9kZQptByfaDD9sNNIjLbK7az7cJtDMwdSJQpimhTNNGmaKKE2g+3\nY21NAR/55Zecetqp7KvZpxYyrSyjxL2oqWdhU8/+QVu5t53JBSnEMSguiwExvcmNySDXnE5fcypZ\npmTSSYJDCdTtslBeCofKbJTvtlO5z4bzoI24g3Ys2Eiz2Ondy0Zagp3UWBtJ5nISouzEumyIulaG\n2toYm5wJar0nDzExPTbUZpNxlNtVtKm8QrRwPny3Q4eaPq+r86/o9NNbHu3ViyYOTPPHQMcSEvwM\nc/pBSsm+mn2tOiNbD21ld/Vu77mWaIv383TmkDPdQ6xDsO2wcdErF7X/oj7oyEoHMGJkRaPRaDqC\n3WFnT/UeyiobHZhdVbvYVb2rybGq+qom7ZJjk8nule3dMmJyMNVm03Awm8qybPYX57BjYxa/bInx\npv6YzWqWUosk3yEOMhLtrTsyreURtec132E6mx1XtXoUNhuizoapzoZJujr2tyIWGxbsxGEXKtHa\nER2HM0ZFl2ScBREXh4i3YEq0EJ0Yh7mXhZikOGJTLMSlWrCkxtEQbaHaGUe1w0JlfRyHGyxUuB2i\nQzYLB2viOFBjYW+lhQOVMVQcVk5SRYX/NKro6EbnxePAJKXWE51egjN5K/WJxdTEbOWwqZgDrq3s\nrSvG5mxc8r5PQh+vEzI4xf2YOpghqUPISsxqdahU56z0ENpZ0Wg0mvZRVVfV1JlpJUqzq2qXGh7w\nITM+k8y4bBJlDtG12Tgqsqnelc2BbTns25oNVdlQk0lqSlSrSb5Dh6ogh93eMnLR3s0zI6o5veIa\nyEqxk5Vso3cvG5m97GQk2EiLt5NmsZESZyc5xkZyjI1Es51Ek42EKDsWYcPc0HnniY58Twuh/gDu\nSI+MVc5Rg1nlHdWKaCpxchgH5aKOw9RSHl1NZXQllbFV2KMlNjPYoqKw12Vgs/fFbsvGVtsfW80A\n7JWDsVUPQUSlE5OsnKm4VAuJqWZS00TAyE5paRHTpmlnpdvRzopGo9EEDykl5fbyFlGa5g7Nnuo9\n3gRoABNRJJJFjD0H5+FsavdkU3cgG6pyoCqbGEc69bUWaLCAw+fRacZiEaSm0qktLi4kfyQ1f7+d\nzo2zppqKw3s4dHAXlRV7qKzcT23lQezV5dRXH8ZU1+DNa0p0RZMsY+jlilY5SQ2SmAYXUe7E7Y7g\nxES9SUWQ7MRR47JQK9W+J7q0njruYgnonBVNR/nkk08477zzQm1G0DCSHiNpAa0nnAmVFiEEaZY0\n0ixpjOozyu95TpfTm0/TulPzHWWVuzhg2w9A/SZgeMt+TMKEKdpCvdlCZbSKNlRFW9hvtmCJtmDx\nfZQWLJUWLDYLln3NXmvnY1x0XNeXjBCCTz7/XN0f90qqVXVV7vypvSpvxFbMVvtWbz6Vw+WAeDAl\nmOh/bH/38Mwp3mGaHHc+Saol1f91pVRJNO2M/kTZ7VhsNiw+r7lq7dQftlFfZcdRZSN67x5Y38k/\nQ7hEVoQQtwB3AVnAD8CtUsrvA5x/CvAEMALYAfxNSvlGs3N+A/wFGAj8BMyWUn7h8/ocYE6zrjdL\nKY/1c03DRVby8/NZvnx5qM0IGkbSYyQtoPWEM0bRUueoY3f1bqZNmcbLn7yMrcGGzWFr/2M7zrE7\n/IwR+SE2Krb9Dk4rx2KjY3noyoeY/NfJ3mTW/bX7vf0nxiR6nZAmj2lD6J/cPySVqP0R8WsDCSGm\noxyP64GVwJ3AYiHE0VLKA62cPxD4DHgRuAw4DXhVCLFLSvmV+5wTgHeAe4B/A78FPhFCjJVSbvTp\nbj0wBfBkA3WxBFZkkZmZGWoTgoqR9BhJC2g94YxRtMRGxzIwZSCDcgZxQr8TuuUaLumizlHndWBq\nG2o75hC14hhVVlf6PdfusBMtotl5eCfD0odx9lFnNya2pg4mMz7TMKu4ByIsnBWUc/KylPJNACHE\njcA5wDXAo62cfxNQLKWc5X6+RQhxkrufr9zHbgO+kFI+6X5+nxDidNSstpt9+nJIKfej0Wg0Gk0b\nmIRJRT3MFrC0fX4wKFhVwMKrw7NGUU8R8ko5QggzMB74P88xqcamvgby/TSzul/3ZXGz8/PbcQ7A\nUCFEmRBiqxDin0KIfh2UoNFoNBqNphsJubMCZABRwN5mx/ei8ldaI8vP+UlCiNg2zvHtsxC4CjgD\nuBEYBCwVQiR0wH6NRqPRaDTdSLgMA4UEKeVin6frhRArgRLgYuD1VprEAWzatKkHrOsZVq5cSVFR\nh/Kcwhoj6TGSFtB6whkjaQGtJ1zx+e7s+CRwKWVIN8AMNAAFzY7PBz7202YJ8GSzY1cB5T7PS4Db\nmp1zP7CmDXtWomYWtfbaZYDUm970pje96U1vnd4u66ivEPLIipSyQQixGjUjZyGAUKnNU4Bn/TRb\nDpzV7NhU93Hfc5r3cXqzc5oghEgEjgLe9HPKYtSsou1Ax+avaTQajUZzZBOHKiWyuI3zWhAWdVaE\nEBejIik30jh1+SLgGCnlfiHEw0C2lPJK9/kDgR9RU5dfQzklTwNnSym/dp+TD3wD/BE1dflSYDYw\nzjN1WQjxGPApKgqTAzwAjAaOlVIe7GbZGo1Go9Fo2kHIIysAUsr3hRAZqAJufYC1wBk+U4qzgH4+\n528XQpwDPIWaolwKzPA4Ku5zlgshLgP+5t5+Bn7drMZKLqoWSzqwH1gGWLWjotFoNBpN+BAWkRWN\nRqPRaDQaf4TD1GWNRqPRaDQav2hnRaPRaDQaTVijnZUOIIS4RQixTQhhE0IUCiEmhNqmziCEOFkI\nsdBdudclhCgItU2dRQjxRyHESiFEpRBirxDiYyHE0aG2q7MIIW4UQvwghDjs3v4rhDgz1HYFAyHE\nbPf77cm2zw4/hBBz3Pb7bhvbbhm+CCGyhRBvCSEOCCFq3e+9iFyl1f2/ufn9cQkhngu1bR1FCGES\nQjwohCh235dfhBB/DrVdXUEIkSiEeFoIsd2taZkQ4rj2ttfOSjvxWWxxDjAWtTL0YndicKSRgEpi\nvhk15z2SORl4DjgetaClGfhSCNFDq3YEnZ2oxTfHoZah+A/wv0KIVha8jxzcjv31qM9NJLMeNQkg\ny72dFFpzOo8QIgX4DqhDVfEeDvwBKA+lXV3gOBrvSxaqVIUE3g+lUZ1kNnAD6n/0McAsYJYQYmZI\nreoa81Azd38LjESt4/e1EKJvexrrBNt2IoQoBFZIKW93PxeoL5ZnpZStLbYYEQghXMB5UkpDrJLl\ndh73AZOklMtCbU8wEEIcBO6SUr4eals6g7t+0WrUAqT3ogoz/j60VnUcIcQc1IzCiIw8NEcIMRfI\nl1JODrUt3YEQwlPOIuIirUKIT4E9UsrrfI59CNRKKa8InWWdQwgRB1QB06SUi3yOrwI+l1Le11Yf\nOrLSDjq52KImNKSgfk0dCrUhXcUdCr4EiCdAMcMI4AXgUynlf0JtSBAw0sKn04BVQoj33UOoRUKI\na0NtVDBw/8/+LerXfCTyX2CKEGIogBAiDzgR+DykVnWeaNQagHXNjttoZ3QyLOqsRACBFlsc1vPm\naFrDHe16GljWrJ5ORCGEGIlyTjy/Rs6XUm4OrVWdw+1sjUGF6CMdz8KnW4C+qOU7lgohRkopa0Jo\nV2cZjIp2PYGqRTUReFYIUSelfCuklnWd84Fk4I1QG9JJ5gJJwGYhhBMVWPiTlHJBaM3qHFLKaiHE\ncuBeIcRm1HfnZagf+z+3pw/trGiMxIvAsahfIJHMZiAP9c/2IuBNIcSkSHNYhBC5KOfxNCllQ6jt\n6SqdWPg03DEBK6WU97qf/+B2lG8EIt1ZuQb4Qkq5J9SGdJLpqC/zS4CNKIf/GSHErgh2JC9HVZwv\nAxxAEaoo6/j2NNbOSvs4ADhRiXW+9AEi9cNgKIQQzwNnAydLKXeH2p6uIKV0AMXup2uEEBOB21G/\ngiOJ8UAmUOSOeoGKUE5yJwrGyghOmpNSHhZC/IRaTywS2Q00X0J+E3BBCGwJGkKI/qhk+/NCbUsX\neBR4WEr5gfv5BvcyM38kQh1JKeU24FfuyQ9JUsq9QogFNP6vC4jOWWkH7l+FnsUWgSaLLf43VHZp\nFG5H5dfAr6SUO0JtTzdgAmJDbUQn+BoYhfpVmOfeVgH/BPIi2VGBJgufRqpz/B0th7GHoaJFkcw1\nqGGGSM3vAJWn5mx2zIUBvrOllDa3o5KKmoX2SXva6chK+3kSmC/UCtGexRbjUQswRhRCiATUP1nP\nr93B7gSuQ1LKnaGzrOMIIV5ELVJZANQIITzRr8NSyohbGVsI8RDwBbAD6IVKEpyMWlU8onDncTTJ\nHRJC1AAHpZTNf9GHPaL1hU8bgHdDaVcXeAr4TgjxR9T03uOBa4HrArYKY9w/Iq8C5kspXSE2pyt8\nCvxZCFEKbECVMrgTeDWkVnUBIcRU1HfOFmAoKnq0kXZ+h2pnpZ38//buJsTKKo7j+Pe3kgyqTRNG\ne6M3FxVEtEvKiKKowEWbhMBAiGhbORlk0CqzrEVIA4KQtCgwCnpZFEbkohfQhLBFGeQoGDoqpf8W\nzzNyu0wxlbd77vj9wDDnnuc8957DMPDjPOfcs4jDFifJTcDHdLtmim6BHXSL0daNq1P/0nq6MXwy\nVP8IMPO/9+a/m6L7O6wAjgFfA3cskZ00MNnf67OkDj6tqi+T3E+3mPNp4CDw+KQu4uytpjv0dhLX\nEA3aADxHt5NuCjgEbOvrJtWlwGa6oH8U2AU8VVXDM0gL8ntWJElS0yb++ZckSVraDCuSJKlphhVJ\nktQ0w4okSWqaYUWSJDXNsCJJkppmWJEkSU0zrEiSpKYZViRJUtMMK5IueEnOJrl33P2QtDDDiqSx\nSrK9Dwtn+t/z5Uk+NVfSeeRBhpJa8B7dabkZqDs9nq5Iao0zK5JacLqqDlfVLwM/x+DcI5r1SXYn\nmUvyfZIHBm9Ocl2SD/vrs0leT3LxUJt1Sb5NcirJT0m2DPXh8iRvJzmR5ECSe0Y8ZkmLZFiRNAk2\nAW8BNwA7gJ1JVgIkWQ68DxwBbgQeBFYDL8/fnOQxYCvwGnAtcDdwYOgzngF2AtcDu4EdSS4b3ZAk\nLVaqatx9kHQBS7IdeBg4NVBdwPNV9UKSs8CrVbVh4J49wN6q2pDkUWAzcFVVneqv3wW8C6yoqsNJ\nfgTeqKqNf9GHs8CmqpruXy8HjgNrquqD8zxkSf+Qa1YkteAjYD1/XrNydKD8+VD7PcCqvnw18NV8\nUOl9RjdzvDIJwJX9Z/ydb+YLVTWX5FdgarEDkDQ6hhVJLThRVQdH9N4nF9nut6HXhY/KpSb4jyhp\nEtyywOt9fXkfsCrJRQPXbwPOAPur6jjwA3D7qDspaTScWZHUgmVJrhiq+72qjvTlh5LsBT6lW99y\nM7Cuv7YDmAbeTPIs3aObLcBMVc32baaBbUkO022TvgS4taq2jmg8ks4jw4qkFqwBDg3VfQdc05c3\nAmuBV4CfgbVVtR+gqk4muRN4CfgCmAN2AU/Ov1FVzSRZBjwBvAjM9m3ONVmgT+4+kBrhbiBJTet3\n6txXVe+Muy+SxsM1K5IkqWmGFUmtc/pXusD5GEiSJDXNmRVJktQ0w4okSWqaYUWSJDXNsCJJkppm\nWJEkSU0zrEiSpKYZViRJUtMMK5IkqWl/AH6L3Z2xLaDUAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "colors = ['b','g','r']\n", + "\n", + "for (i,loss_arr) in enumerate(loss_arrays):\n", + " by_epoch = by_epoch_stats[i]\n", + " epoch,t_loss,val_loss,val_roc = by_epoch\n", + " alpha = 1.0\n", + " if node_counts[i] == 4:\n", + " alpha = 0.4\n", + "# plt.semilogy(effective_epochs[i],np.array(range(len(loss_arr)))*node_counts[i],loss_arr,alpha=alpha,label=r\"$N_{{GPU}} = {{{}}}$\".format(node_counts[i]))\n", + " plt.plot(epoch,t_loss,color=colors[i],label=r\"$N_{{GPU}} = {{{}}}$\".format(node_counts[i]))\n", + " plt.plot(epoch,val_loss,'--',color=colors[i])\n", + "\n", + " \n", + "plt.plot([],[],'--k',label=\"Validation\")\n", + "plt.plot([],[],'-k',label=\"Train\")\n", + "plt.xlim([0,9])\n", + "plt.xlabel(\"Epoch\")\n", + "plt.ylabel(\"Loss\")\n", + "plt.legend(loc=\"best\")\n", + "plt.grid()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot from CSV log files" + ] + }, + { + "cell_type": "code", + "execution_count": 246, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "a = effective_epochs_and_loss[0][1]\n", + "a\n", + "import pandas as pd\n", + "def get_rolling_mean(arr,N):\n", + " ret = np.zeros_like(arr)\n", + " cutoff=N-1\n", + " for i in range(cutoff):\n", + " ret[i] = np.mean(arr[:i])\n", + "# ret[:cutoff] = arr[:cutoff]\n", + " ret[cutoff:] = pd.rolling_mean(arr,N)[cutoff:]\n", + " return ret\n" + ] + }, + { + "cell_type": "code", + "execution_count": 250, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/julian/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py:10: FutureWarning: pd.rolling_mean is deprecated for ndarrays and will be removed in a future version\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 250, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAikAAAGACAYAAABoaPKlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3Xl4VOXZ+PHvMzNZCMQABlBQNlEBaaPBpQgiYou21Wjl\nrRJUkEV+LmilCGpFxFrL4gJtxb61RNGqAX1VDCiKgAIRqUpERRZlkbAIsgWyZ5bn98eZmcxkmUwm\nM2cymftzXbnCnHPmnGfuDJk797McpbVGCCGEEKK5sUS7AUIIIYQQdZEkRQghhBDNkiQpQgghhGiW\nJEkRQgghRLMkSYoQQgghmiVJUoQQQgjRLEmSIoQQQohmSZIUIYQQQjRLkqQIIYQQolmSJEUIIYQQ\nzZIkKUIIIYRoluImSVFKXaOU2qaU2q6UGhft9gghhBAiMBUPNxhUSlmBLcDlQAlQAFyitT4e1YYJ\nIYQQol7xUkm5GNistT6otS4B3gWGRblNQgghhAggXpKUzsB+n8f7gS5RaosQQgghgtDskxSl1GVK\nqTyl1H6llEsplVXHMXcrpXYrpcqVUhuUUhdFo61CCCGECJ9mn6QArYFNwF1ArQE0SqmbgKeBR4EL\ngK+AD5RS6T6HHQDO8Hncxb1NCCGEEM1UTA2cVUq5gOu11nk+2zYA/9Va/8H9WAF7gb9rree4t3kG\nzg4BioHPgUtl4KwQQgjRfNmi3YCmUEolAP2Bv3q2aa21UmolMMBnm1MpNRn4GFDA7EAJilLqVOAq\n4AegIiKNF0IIIVqmZKA78IHW+mhTThTTSQqQDliBQzW2HwLO9d2gtV4GLAvyvFcBrza5dUIIIUT8\nuhl4rSkniPUkJVJ+AHjlnHPok5sb5abEh0mTJjF37txoNyPuSNzNJzE3n8TcXFu3buWWW24B92dp\nU8R6knIEcAKdamzvBBxswnkrAPrYbGRmZjbhNCJYaWlpEusokLibT2JuPol51DR5uEQszO6pl9ba\nDmwErvRscw+cvRJY3+QLVFU1+RQiOAcPNiWnFKGSuJtPYm4+iXnsavaVFKVUa6AXxoBXgJ5KqQzg\nmNZ6L/AMsFAptRH4DJgEpAALm3rt+/bvp21WFtnZ2WRnZzf1dCKA/fv3N3yQCDuJu/kk5uaTmJsj\nNzeX3NxcTpw4EbZzNvskBbgQ+AhjjRSNsSYKwEvAWK316+41Uf6M0c2zCbhKa324qReem5ZG/7y8\nhg8UTda/f/9oNyEuSdzNJzE3n8TcHJ4/6AsKCsIW82afpGit19BAt5TW+jngubBf3G4P+ylF3aRS\nFR0Sd/NJzM0nMY9dMbWYm1mUUpnAxi/atKF/cXG0myOEEGFVWFjIkSNHot0MEaPS09Pp2rVrvft9\nKin9tdYFTblWs6+kRJVUUoQQLUxhYSF9+vShrKws2k0RMSolJYWtW7cGTFTCRZKUQKqqQGtQquFj\nRZOMGTOGF198MdrNiDsSd/NFO+ZHjhyhrKyMV155hT59+kStHSI2edZAOXLkiCQp0fZHrUm79lqy\nb75Z+jQjbNiwYdFuQlySuJuvucS8T58+snaICKtIzO6RMSl18IxJ2QhkFhdDmzbRbpIQQoSFZ7zA\nxo0bJUkRjRbM+yecY1JiejE3U1RWRrsFQgghRFySJKUhkqQIIYQQUSFJSkMqmnzrARGE/Pz8aDch\nLknczScxFyJ4kqQ0RCopppgzZ060mxCXJO7mk5gLETyZ3RPAJCDtjjvIvuMOmd0TYYsWLYp2E+KS\nxN18EnPRUkVido9UUgKYC+TNni0JiglSUlKi3YS4JHE3n8TcPDk5OVxzzTUkJiaSlZXlt2/Xrl1c\neeWVWCwWMjIyeO658N9ZxUwTJkzg008/jWobsrOzycvLY+7cuWE7p1RSGiLdPUIIEZPGjRtH3759\n6dOnD/PmzWPfvn2cccYZAPTs2ZNVq1YxcuRIXnvttSi3tGk+/vhjFixYwC233BLtpoSdVFIaIkmK\nEELErA0bNjB9+nQ6derEggUL/PYdPnyYiy66KEotC4/KykpWr16NaqEro0uS0hCZ3WOKKVOmRLsJ\ncUnibj6JublKSkpITU3ltttuIycnB98FTPPz8xk4cGAUW9d0zz77LBMnTqSlLswqSUpDpJJiCjPu\nASFqk7ibT2JuLk+FYezYsRw4cIBly5Z5933xxReelVFj0ubNm+nUqRMdO3aMdlMiRsakBDAJSJs5\nk2yHQwbPRtg999wT7SbEJYm7+WIt5g4HFBVF9hpt24ItAp9Gmzdvpm/fvoAxBmXIkCE8//zzXHvt\ntQDY7XasVmuD53E4HNgi0cAm0Frz6quvMnPmzGg3xSsSs3uaV9SbmblA5t13gyQoQog4VVQEb70V\n2WvccAOkp4f/vOvWreOGG27wPh43bhyjR49m3759pKWlkZqaWus5r7/+Olu2bKFz586kpKSQmJjI\n+vXrmTdvHgBr165l6tSpdOvWjauuuory8nK+/PJLpk6dyrvvvsuDDz7I7NmzGTlyJB07duTxxx/n\njTfe4OGHH+amm24CwOVyMXz4cCrdlfqaXTWe6o/WmrZt25Kbm1urnTk5OYwbNy48gQqT7OxssrOz\nfe/d02SSpATgtFplTIoQIq61bWskEZG+RiTs27ePTp06eR8PHz6ciRMnsmDBAgYMGMCAAQO8+1wu\nF2PGjGHQoEHMmDHDu/3pp5+mS5cu3seDBw/G6XQyefJkLr74YgDmz5/PtGnTuOeeexg0aBD33Xef\n9/gRI0YwcOBAhg4d6t1msVh4++23m/S6ysvL6dWrV8jniBWSpATgtNlkTIpJtm3bRu/evaPdjLgj\ncTdfrMXcZotMlSMakpKSuPnmm8nJycHhcPDQQw959z3yyCO0adOG22+/3e85AwYMoK1PFlVcXMz2\n7dv97gBcWFhIamoqq1ev9ktGANavX8+IESPC+jref/99PvvsM8aOHQvgrcjMmjWLvLw8nnrqqbBe\nL5okSQnAaZUkxSxTp04lLy8v2s2IOxJ380nMzbFnz546BymPHz+e+fPns379elq3bg3A0aNHeeaZ\nZ9ixY0et4y+99FK/x/n5+Zx//vneMSoHDx5k7dq1LFq0iNGjRzNr1iy/48vKykhKSvLbVrO7pz71\ndfeMHz+e8ePH+73W3NxcHnroIS677LKA54w1kqQE4LBJd49Znn322Wg3IS5J3M0nMTfHkiVLuOSS\nS2ptz8jIIDMzk4yMDO+2devW0aNHD79unXfeeYdVq1Zht9uZPHmyt2tlzZo1pKenk5eXR0VFBaWl\npaxYsYKEhAS2bNni7QLysFhqT6JtandPTQ6HAwCn0xm2czYXMgU5AIfNKpUUk8i0zOiQuJtPYh5Z\nBQUFDBs2jClTpnDbbbfxyiuv1DpmwoQJXH755d7HFouFdu3a+R1z3XXXceLECS655BK/sR9r1qzh\n9ttvJysrixtvvJExY8aQmprK5s2b6devn19S8sknn/iNe4mEJ554gt/97ncopZgwYULMzR5riFRS\nAnBaJUkRQohYkpmZyYoVKwIeM2HCBL/HQ4cOZdKkSezZs4du3boBeCsk06ZN8x5XUlLCN99845fg\neHTs2JHExETv4/Lycj7//HO/QbSR8PDDD/Pwww9H9BrRJElKAA6Z3SOEEC1emzZtWLp0KdOnTycj\nI4NTTz0Vu93OrFmzOPvsswGjQvPCCy+QlpbG8uXLGT58uN85unbtSnZ2NjNnzvQurnb33Xeb/lpa\nGklSAphRVMT/vvsu2bm5sphbhM2ePZsHHngg2s2IOxJ380nMm6e+ffvy0ksv1bs/MzOTzMzMgGOK\nRo8eHYmmxQxZzM1kD3TqwPCBl8libiYoKyuLdhPiksTdfBJz0VJFYjE3GTgbgMNike4ekzz22GPR\nbkJckribT2IuRPAkSQnAbrPIwFkhhBAiSiRJCcBulSRFCCGEiBZJUgKwWyRJMcuRI0ei3YS4JHE3\nn8RciOBJkhJAlU3JmBSTeO5BIcwlcTefxFyI4EmSEoDdqqSSYhLfu44K80jczScxFyJ4kqQEUGWR\nJMUsvncUFeaRuJtPYi5E8CRJCcBuQbp7hBBCiCiRxdwCeHnHfj4rs8uKs0IIIUQDIrHirFRSArih\n7xnkpaRIgmKCnJycaDchLknczScxFy1VdnY2eXl5zJ07N2znlCQlgEqLlu4ekxQUFES7CXFJ4m4+\nibkQwZMkJYAqGzJw1iTz58+PdhPiksTdfBJz8+Tk5HDNNdeQmJhIVlaW375du3Zx5ZVXYrFYyMjI\n4LnnnotSK0NTVlbGjBkzuPfeexk8eDCjR4/m0KFDtY6bOXMm999/PzNmzGD06NEUFxdHobWhkzEp\nAVRZNLhc4HCATUIlhBCxZNy4cfTt25c+ffowb9489u3bxxlnnAFAz549WbVqFSNHjuS1116Lcksb\n7/HHH+fee+/l9NNPB+Dqq69m6NChbNq0iYSEBMBIiNeuXcvy5csBmDVrFrfeeitLliyJWrsbSyop\nAVRZtPEPqaYIIURM2rBhA9OnT6dTp04sWLDAb9/hw4e56KKLotSy0FVWVvLss8/6jW+aPHkyW7du\nJS8vz7ttzpw5jB492vt41KhR5OXlsWPHDlPb2xSSpARQYXMnKTIuRQghYlJJSQmpqancdttt5OTk\noLX27svPz2fgwIFRbF1onE4n6enplJaWerd169YNgJ07dwLw/fffs3fvXvr27es9pnPnzqSlpfHR\nRx+Z2+AmkCQlgCqrVFLMUrO/WJhD4m4+ibm5lFKAcTuCAwcOsGzZMu++L774gv79+0eraSFLSUlh\n9+7dzJw507vthx9+AIxuLDCSFaUUp5xyit9zU1NTKSwsNK2tTSUDLQKotLjc/5AkJdImTpwY7SbE\nJYm7+WIt5mX2MrYd2RbRa/RO701KQkrYz7t582ZvJaFnz54MGTKE559/nmuvvRYAu92O1Wpt8DwO\nhwNbMx+XmJuby7nnnst1110HwPHjxwFo3bq133Ft2rTx7osFzTvqUVYp3T2mGTZsWLSbEJck7uaL\ntZhvO7KN/s9HttqwccJGMk8P/+0C1q1bxw033OB9PG7cOEaPHs2+fftIS0sjNTW11nNef/11tmzZ\nQufOnUlJSSExMZH169czb948ANauXcvUqVPp1q0bV111FeXl5Xz55ZdMnTqVd999lwcffJDZs2cz\ncuRIOnbsyOOPP84bb7zBww8/zE033QSAy+Vi+PDhVLr/APbtgoLq6o/WmrZt25KbmxvwdX711Vcs\nWbKEDz/80Dto1pN81UzC7HY7Docj6BhGmyQpAVRJJUUIEed6p/dm44SNEb9GJOzbt49OnTp5Hw8f\nPpyJEyeyYMECBgwYwIABA7z7XC4XY8aMYdCgQX43gXz66afp0qWL9/HgwYNxOp1MnjyZiy++GDBm\n0UybNo177rmHQYMGcd9993mPHzFiBAMHDmTo0KHebRaLhbfffjssr7GkpIQJEybw1ltvceGFF3q3\nd+jQwfu6fJWWlpKWlhaWa5tBkpQApLtHCBHvUhJSIlLliIakpCRuvvlmcnJycDgcPPTQQ959jzzy\nCG3atOH222/3e86AAQNo27at93FxcTHbt2/3u1FkYWEhqamprF692i8ZAVi/fj0jRoyI0CuCO++8\nk6eeeorLLrsMgN27d9OjRw969OgBwKFDh0hPTweMykxRUZF33EoskCQlgAqbO0mR7p6IW7JkCddf\nf320mxF3JO7mk5ibY8+ePXTt2rXW9vHjxzN//nzWr1/vHa9x9OhRnnnmmTqn5l566aV+j/Pz8zn/\n/PO9Y1QOHjzI2rVrWbRoEaNHj2bWrFl+x5eVlZGUlOS3rWZ3T30a6u554oknGD16tDdBKSwsZM2a\nNfTo0YPu3bvTq1cvtm/fznnnnQfAtm3bqKysrJVINWeSpASwa81RsoDs998n+/LLo92cFi03N1d+\ncUeBxN18EnNzLFmyhEsuuaTW9oyMDDIzM8nIyPBuW7duHT169PDr1nnnnXdYtWoVdrudyZMn06tX\nLwDWrFlDeno6eXl5VFRUUFpayooVK0hISGDLli3eLiAPi6X2JNpwdPcsXryYjz76CJvNxsaNRnfc\nt99+yx133OE9ZtSoUbz88svecTkLFy4kKyuLs88+u0nXrk8kbjCI1lq+anwBmYDuOipda9A6L08L\nIURLsHHjRg3ojRs3RrspEbFx40b9q1/9SickJOhzzz1X/+c//6l1zL/+9S/99ttvex+/8847+tJL\nL6113KhRo/SLL77ot+0Xv/iFfu+992od+/nnn+srrrjCb1t+fr7+6quvQnwl9Tt69KhOSUnRFovF\n78tqterDhw97j7Pb7fr+++/X9957r37kkUf0qFGjdFFRUZOuHcz7x3MMkKmb+HkslZQAKmxO9z+k\nu0cIIWJBZmYmK1asCHjMhAkT/B4PHTqUSZMmsWfPHu+iaJ4KybRp07zHlZSU8M0333B5HZX1jh07\nkpiY6H1cXl7O559/7jeINlzat2/vt5BbfWw2G08++WTYr28mSVIC8A6craqKbkOEEEJETJs2bVi6\ndCnTp08nIyODU089FbvdzqxZs7xdIwUFBbzwwgukpaWxfPlyhg8f7neOrl27kp2dzcyZM+nYsSMA\nd999t+mvpaWRJCWASot7LrnM7hFCiBatb9++vPTSS/Xuz8zMJDMzk2effbbeY3zvkyPCQ5bFD8Ch\nnGirVSopJhgzZky0mxCXJO7mk5gLETxJUgJwaQc6MUmSFBPE2iqcLYXE3XwScyGCJ0lKAC4c6IRE\n6e4xQXZ2drSbEJck7uaTmAsRPElSGqATEqWSIoQQQkSBJCkNcEmSIoQQQkSFJCkNcCUkSHePCfLz\n86PdhLgkcTefxFyI4EmS0gCppJhjzpw50W5CXJK4m09iLkTwJElpgNMmlRQzLFq0KNpNiEsSd/NJ\nzIUIniQpDXDZbFJJMUFKSkq0mxCXJO7mk5gLETxJUhrgTEiQJEUIIYSIAklSGuC02aS7RwghhIgC\nSVIa4LBJJcUMU6ZMiXYT4pLE3XwScyGCJ0lKAxw2q1RSTNC1a9doNyEuSdzNJzE3T05ODtdccw2J\niYlkZWX57du1axdXXnklFouFjIwMnnvuuSi1smlWrVrFkCFD6t0/c+ZM7r//fmbMmMHo0aMpLi5u\n1P5ok7sgB/I+jCn5hnGdT0MWso6se+65J9pNiEsSd/NJzM0zbtw4+vbtS58+fZg3bx779u3jjDPO\nAKBnz56sWrWKkSNH8tprr0W5pY33xhtv8N5771FSUkJhYWGdx8yfP5+1a9eyfPlyAGbNmsWtt97K\nkiVLgtrfWLm5ueTm5nLixImQnl8XqaQEcjX88/wLyW7fPtotEUIIEYINGzYwffp0OnXqxIIFC/z2\nHT58mIsuuihKLWua3//+97z44otcc8019R4zZ84cRo8e7X08atQo8vLy2LFjR1D7Gys7O5u8vDzm\nzp0b0vPrIklKA+xW6e4RQohYVVJSQmpqKrfddhs5OTlorb378vPzGThwYBRbFznff/89e/fupW/f\nvt5tnTt3Ji0tjY8++qjB/c2FJCkNsFutMnDWBNu2bYt2E+KSxN18EnNzKaUAGDt2LAcOHGDZsmXe\nfV988QX9+/ePVtMiaufOnSilOOWUU/y2p6amUlhY2OD+5kLGpDTAbpMkxQxTp04lLy8v2s2IOxJ3\n88VczMvKINKJVe/eEIFF7jZv3uytFPTs2ZMhQ4bw/PPPc+211wJgt9uxWq0NnsfhcGCzxdbH5fHj\nxwFo3bq13/Y2bdpw/PjxBvc3F7EV9SiwWy3S3WOCZ599NtpNiEsSd/PFXMy3bYNIVxs2boTMzLCf\ndt26ddxwww3ex+PGjWP06NHs27ePtLQ0UlNTaz3n9ddfZ8uWLXTu3JmUlBQSExNZv3498+bNA2Dt\n2rVMnTqVbt26cdVVV1FeXs6XX37J1KlTeffdd3nwwQeZPXs2I0eOpGPHjjz++OO88cYbPPzww9x0\n000AuFwuhg8fTqX7s8W3Cwqqqz9aa9q2bUtubm6jX7sn+aqZhNntdhwOR4P7mwtJUhpgt1mkkmIC\nmZYZHRJ388VczHv3NpKISF8jAvbt20enTp28j4cPH87EiRNZsGABAwYMYMCAAd59LpeLMWPGMGjQ\nIGbMmOHd/vTTT9OlSxfv48GDB+N0Opk8eTIXX3wxYMySmTZtGvfccw+DBg3ivvvu8x4/YsQIBg4c\nyNChQ73bLBYLb7/9diResleHDh28r8tXaWkpaWlpDe5vLiRJaUCVRSopQog4lpISkSpHNCQlJXHz\nzTeTk5ODw+HgoYce8u575JFHaNOmDbfffrvfcwYMGEDbtm29j4uLi9m+fTuZPjEpLCwkNTWV1atX\n+yUjAOvXr2fEiBERekX169GjBwCHDh0iPT0dMCozRUVFnHXWWfTo0QOtdZ37e/bsaXp76yNJSgOq\nrEoqKUIIEWP27NlTZ9Vq/PjxzJ8/n/Xr13vHYxw9epRnnnmmzqm3l156qd/j/Px8zj//fO8YlYMH\nD7J27VoWLVrE6NGjmTVrlt/xZWVlJCUl+W2r2d1Tn6Z093Tv3p1evXqxfft2zjvvPMAYtF1ZWcnQ\noUMb3N9cSJLSgCqrdPeYYfbs2TzwwAPRbkbckbibT2JujiVLlnDJJZfU2p6RkUFmZiYZGRnebevW\nraNHjx5+3TrvvPMOq1atwm63M3nyZHr16gXAmjVrSE9PJy8vj4qKCkpLS1mxYgUJCQls2bLF2wXk\nYbHUnkQbzu4ep9NZq8vGY9SoUbz88svecTkLFy4kKyvL+1pGjx5d5/6zzz47LG0LB0lSGlBlVdLd\nY4KysrJoNyEuSdzNJzGPrIKCAh588EE+/vhjevbsybRp07jlllv8jpkwYQIdO3b0PrZYLLRr187v\nmOuuu4633nqLK664wvuhDkaSMn36dH7961/7Hf/FF1/Qr18/v6Tkk08+8Rv3Ek7vv/8+CxYsID8/\nn8OHD3PZZZfRu3dv/v3vf3uPeeCBB3jooYf4wx/+QFpaGgcPHmThwoVB728WtNbyVeMLyAS05f9Z\n9X9uu1lr0Nrp1EIIEes2btyoAb1x48ZoN6XZKC4u1j179tQ//PCDd1tJSYk+7bTT9Hfffed3XOvW\nrXVpaWmtc+zZs0dfddVV3sdlZWV67ty5kW14FATz/vEcA2TqJn4eSyUlAIuyUml1Z8V2O9ToVxRC\nCBH72rRpw9KlS5k+fToZGRmceuqp2O12Zs2a5e36KCgo4IUXXiAtLY3ly5czfPhwv3N07dqV7Oxs\nZs6c6a3S3H333aa/lpZGkpQAbMpGpWcKeWWlJClCCNFC9e3bl5deeqne/ZmZmWRmZgZc58b3Pjgi\nPGRZ/ACsykalJ0IyeDaijhw5Eu0mxCWJu/kk5kIET5KUAGwqgUqLeyVAGTwbUWPHjo12E+KSxN18\nEnMhgifdPQFYLTa8qYlUUiLKd4VHYR6Ju/kk5kIET5KUAGwqgXLlrqRIkhJRmS1kRctYI3E3n8Rc\niOBJkhKAzWKjQrkXyZHuHiGEEMJUkqQEYLMkUO5JUqSSIoQQQphKBs4GYFMJlFudxgOppERUTk5O\ntJsQlyTu5pOYCxE8qaQEYLMkUKHcSYpUUiKqoKCAcePGRbsZcUfibr7mEvOtW7dGuwkiBpn9vpEk\nJQCbJYFyq3T3mGH+/PnRbkJckribL9oxT09PJyUlpdb9bIQIVkpKCunp6aZcK26SFKXUW8AQYKXW\n+sZgnpNgSaDC6jAeSHePEKIF6Nq1K1u3bpVF5UTI0tPT6dq1qynXipskBZgH5ABBr1ucYLFRLt09\nQogWpmvXrqZ9yAjRFHEzcFZrvRYoacxzbJYEymzuSookKUIIIYSp4iZJCUWCJYEKi3T3mCErKyva\nTYhLEnfzSczNJzGPXc0ySVFKXaaUylNK7VdKuZRStd5hSqm7lVK7lVLlSqkNSqmLwt0OmzWBKmVH\nW61SSYmwiRMnRrsJcUnibj6Jufkk5rGrWSYpQGtgE3AXoGvuVErdBDwNPApcAHwFfKCUSvc55i6l\n1JdKqQKlVFIojbCpBJxUQWKiVFIibNiwYdFuQlySuJtPYm4+iXnsCmuSopTqrJTq0dTzaK3f11pP\n11q/A6g6DpkE/Etr/bLWehtwB1AGjPU5x3Na6wu01plaa0+Goeo5X50SrAk4dKWRpEglRQghhDBV\nSEmKUupOpdTLNbb9E9gL7FBKrVdKnRqOBtZx7QSgP7DKs01rrYGVwIAAz/sQWAz8WilVqJS6pKFr\nJVhtOHQVJCZJkiKEEEKYLNRKyh3ASc8DpdQvgf8HvARMAfpidMVEQjpgBQ7V2H4IOK2+J2mtf6W1\n7qS1bqO17qq1/m9DF1rx1yUce6WQrOKTZL30EllZWQwYMIAlS5b4H7diRZ0Ds+6+++5aS2AXFBSQ\nlZVVa42CRx99lNmzZ/ttKywsJCsri23btvlt/8c//sGUKVP8tpWVlZGVlUV+fr7f9tzcXMaMGVOr\nbTfddFOzeh2etsT66/CIldfhe81Yfh2+mvvrGDVqVIt4HbH083j66adbxOtojj+P3Nxc72fjaaed\nRlZWFpMmTar1nJBprRv9BRQBd/o8/jewG1Dux38BdoVy7jqu5QKyfB6f7t52SY3jZgOfhumamYC+\n8ekx+tTHu2tnj7O0fvBBLSLnxhtvjHYT4pLE3XwSc/NJzM21ceNGjTGeNFM38fM41EqK1Z0oeAwD\nlmutPYNcd7mTiUg4AjiBTjW2dwIOhvNCCRYbLu1Ay8DZiFu8eHG0mxCXJO7mk5ibT2Ieu0JNUr4H\nsgCUUkOBM4D3ffZ3wai2hJ3W2g5sBK70bFNKKffj9eG8ls1ixYkdnSADZ4UQQgizhbos/jxgoVLq\nIJCGkbQs99k/BPg61EYppVoDvaieidNTKZUBHNNa7wWecV9/I/AZxmyfFGBhqNesi81iw6kdaBk4\nK4QQQpgupCRFa/2yUuo48BuMisk/3BUOlFLtgUqaljBcCHyE0aelMdZEAWNg7lit9evuNVH+jNHN\nswm4SmuBpLsWAAAgAElEQVR9uAnXrGXl8+9SXnWCRY4T3CrdPUIIIUS9cnNzyc3N5cSJE2E7Z8jr\npGitl2qt79RaP6S1PuCz/ZjW+jda69ebcO41WmuL1tpa46vmOijdtdattNYDtNZfhHq9+lxz9++w\njUzkxtO7SCUlwuoaQS4iT+JuPom5+STm5sjOziYvL4+5c+eG7Zxhuwuye1XX4UAS8IFv4hKrEqwJ\n7oGz0t0TabIiZHRI3M0nMTefxDx2hZSkuBdu+4XW+gL34wTgE4wl6hVwTCl1hdb6m7C1NAq8A2dt\nMrsn0rKzs6PdhLgkcTefxNx8EvPYFWp3zzDgXZ/HIzDWFhmPMZ7kGDCjSS1rBmxWGxqNS2b3CCGE\nEKYLtbvnNIzF2zyuBwq01i8CKKWeB/7YxLZF3Zt/WwylsDh1P2Nbh61nTAghhGhxmtPA2XIgFUAp\nZQWGAh/47C8C2jWtadF3y/23wkj4XfdeUkmJsJrLMQtzSNzNJzE3n8TcHJEYOBtqkrIJGK+U6oNx\nr55TgGU++3tS+946MSfBalRPHLYESVIibM6cOdFuQlySuJtPYm4+iXnsCrUP4xGMxds2YwyUzdNa\nf+qz/3rg07qeGEtsFisADptVBs5G2KJFi6LdhLgkcTefxNx8EvPYFepibp8qpfoCl2F07azw7FNK\ntcNYdG1VWFoYRTaLVFLMkpKSEu0mxCWJu/kk5uaTmMeukEeDutdBqXXXJq31cYw7Ese8BJsRHqdU\nUoQQQgjTNWnKilLqEuC3QDf3pj3Au1rr/za1Yc3B8088Dyfg7TO+Z5JUUoQQQoh6NZvZPUopm1Lq\nVYy7Dk8D/sf9NQ1Yr5R61T3rJ6bdM/0eGAlX9+4n3T0RNmXKlGg3IS5J3M0nMTefxNwczWl2zzQg\nG5gP9NBat9Zatwa6A8+6900LSwujyOrOs+xW6e6JtK5du0a7CXFJ4m4+ibn5JOaxS2mtG/8kpXYB\n67XWt9Sz/xXgUq11zya2LyqUUpnAxoXvLeS2z27ji+N/ov9zT0o1RQghhGhAQUEB/fv3B+ivtS5o\nyrlCraR0wbhXT30+ATqHeO5mw+qegmy32cBuB5cryi0SQggh4keoScp+YGCA/QOBmL8Lsl93D0gl\nRQghhDBRqEnKK0C2UmqeUsozswelVDel1FyMMSkvh6OB0WRRRniq3FORZVxK5Gzbti3aTYhLEnfz\nSczNJzGPXaEmKX8G3gDuBXYppcqVUuXALuAPwOvAX8LTxOh58tEn4TX4cOs3xgZJUiJm6tSp0W5C\nXJK4m09ibj6JuTlyc3PJyspi0qRJYTtnSANnvU9W6mLgN/ivk/Ke1vqzMLQtajwDZ99c+SbD84fz\njnqSrEenQGEhnHlmtJvXIhUWFsoI/CiQuJtPYm4+ibm5wjlwtkmLubmTkVoJiVLqHKCf1vqtppw/\n2jwDZ6s8Y1IqKqLYmpZNfoFEh8TdfBJz80nMY1eo3T0N+T1Gd1BM84xJ8Q6cle4eIYQQwjSRSlJa\nBE+SUmmTJEUIIYQwmyQpAXimIFdJJSXiZs9uEfekjDkSd/NJzM0nMY9dkqQEYLG4pyBb3WGKdJJS\nVQWvvAK7d0f2Os1QWVlZtJsQlyTu5pOYm09iHrskSQmgupJiQpLyxRdw3nlw660Qh1n/Y489Fu0m\nxCWJu/kk5uaTmMeuoGf3KKXuasR5LwmhLc3OIw88Aofhv4M+NzZEMkmZOhUSE+GKK+CrryJ3HSGE\nECICcnNzyc3N5cSJE2E7Z9DrpCilGnvjGq21tja+SdHnWSfl4/UfM2TFEB7t/CIzJoyBRYvgppvC\nf8Ht26F3b3j1VThwAB59FE6eBGtMhk8IIUQci9Y6KX2acqFY5FknpcIS4e6e55+HU0+F4cNh3Too\nK4OdO+GccyJzvWboyJEjpKenR7sZcUfibj6Jufkk5rEr6DEpWuvtjf2KZMPN4L13j0UZGyKRpPz4\nIyxcCGPGQFISZGQY2zdtCv+1mrGxY8dGuwlxSeJuPom5+STmsUsGzgbgGTjr0C4jgQh3kvK3v8FZ\nZxn/vss95KdDB+jc2X9cyv/+L7TwG2TNmDEj2k2ISxJ380nMzScxj12SpATg6e5xupzhT1IqK+FP\nf4IbbjC6dnr0qN53/vnVlZTXXoM774Tp08N37WYoMzMz2k2ISxJ380nMzScxj12SpATg6e6JSJKy\nYYMx9mTyZGjb1n+fJ0nZtcuosLRvD++9B+Xl4bu+EEII0cxJktIAq7Li1O4kJZw3GFy50hgs6xmD\n4isjw5jl07u3kcC89x6UlsKKFeG7vhBCCNHMSZLSAAvWyFRSVq6EK68ESx0/gqFD4brr4KmnoKAA\nLrnEWOjtrZi+qXRAOTk50W5CXJK4m09ibj6JeeySJKUBFmXFocOcpJw4AZ99Br/8Zd3709NhyRK4\n916jqweMsSt5ecbS+S1QQUGTptKLEEnczScxN5/EPHYFvZib35OUmtrAIRqoAPYB+VrrwyG0LWo8\ni7kNHjyYTw59wiW/yOaTr781Khr//GfTL7BkCfzud8Y9erp3D+45334L/frByy8bS+cLIYQQzYjv\nirNr166FMCzmFmqS4sJIRABUjd01t9uB+cBkHcrFosCTpGzcuJHLlw/lt6dMY1HuW8YYkRdeaPoF\npk2DF1+E/fsb97xrr4UdO2DzZlmNVgghRLMUzhVnQ+3u6QF8DSwGBgKd3F+DgNeBTUA/4FLgLeAP\nwANNaWi0WC1WHM4wd/ccOmSshdJYjzxirJfy5pvhaYcQQgjRjIWapDwN7NRaj9Raf6q1Puz+Wq+1\nzgZ2A49prTe4H68CxoSr0WayqggMnD182Fi0rbEuvhiGDYO//AVcjb2VkhBCCBFbQk1ShgErA+xf\n5T7GYxnQLcRrRZVVWXFpJzoxjFOQQ01SwOgq+uYbWLo0PG1pJrKysqLdhLgkcTefxNx8EvPYFWqS\nYgf6B9h/IeDweayA0hCvFVVWixUXTlyJYa6kdOwY2nMvuwwuvxwefxxiY4hPUCZOnBjtJsQlibv5\nJObmk5jHrlCTlMXAWKXUX5RSXTwblVJdlFJPALe5j/EYDMTkzWesykhSdEIz6O7xmDYNNm5sUYu7\nDRs2rOGDRNhJ3M0nMTefxDx2hZqk3I/RhfMnoFApVaaUKgMKgYeA99zHoJRKxkhQ/tL05prPajG6\ne1zhSlLsdigqalqScuWVxo0Jly9venuEEEKIZsoWypO01mXAdUqpAcDVVI832QN8oLVe73NsBfBw\nUxsaLYnWRJyuKlyJyRwrOsiHmxdzU7+bQj/hkSPG96YkKUrBBRf43ylZCCGEaGGatOKse2bPo1rr\n29xfj/omKC1BK1srqnQ520p2cehYISPeHNG0E/70k/G9KUkKVN+EsIWMS1myZEm0mxCXJO7mk5ib\nT2Ieu5q8LL5SKkEp1UEp1bHmVzgaGG3JCck4dAXv7V9OsnsosNPlDP2Eh92L7zY1ScnIMLqN9u5t\n2nmaidzc3Gg3IS5J3M0nMTefxDx2hZSkKKUSlVKPKqUKgXLgIPBjHV8xz1NJqbRBkjtJ6fRUp9BP\n6ElSQp3d43H++cb3TZuadp5mYvHixQ0fJMJO4m4+ibn5JOaxK6QxKcA/gPHA+8BzwPGwtaiZaZWQ\nzEldjkpKJslprJNytPxo6Cc8fBiSk6F166Y1rEsX4+aDX30FsgaAEEKIFijUJOX3wEKt9bhwNqa5\nmTRpEt+Vfoe1V0eKqSTZqai+NVGIPNOPVc1bHjWSUtXjUoQQQogo873BYLiEOibFAnwWtlY0U3Pn\nzmXon4Zi/bmNcqsmqQlDUbyaukaKr4wMSVKEEEI0C9nZ2eTl5TF37tywnTPUJGUZMCRsrWjGWtla\ncdJ1kAob2JwaS1NvmRPOJOX882HXLghj1hotY8bE5K2dYp7E3XwSc/NJzGNXqEnKQ8A5Sqm/K6XO\nU0qlKqVSan6Fs6HRkmxLptR1lEp3x5hn8KyuMfXX4YCqqiBO+NNPTR806zFokPF9ZaDbKMUGWREy\nOiTu5pOYm09iHrtCTVL2AOcDE4GvgSKguMbXyXA0MNpa2Vph15VUWo3Hni6fcke533FvvQULFwZx\nwnBWUnr2hJ//3Lh4jMvOzo52E+KSxN18EnPzScxjV6gDZ+fQ5BGksaFDayOh8FRSXv7182R9PIHj\n5cdJSaguFhUVBXnCcCYpADfcAM88Y5RxEhPDd14hhBAiykJdFv/BcDekubru3Ot4YOUD3krKuW26\nA7C7aDddTunCZ5+BzSeKLhdY6qtPORxw7Fj4k5QZM2D1arj66vCdVwghhIiyJq8429KdmnIqUF1J\nOTPZGE+y89hOwJhc88UX1ceXlAQ42VH3+irhTFL69TNuNvjmm+E7ZxTk5+dHuwlxSeJuPom5+STm\nsSuoSopSaipG985TWmvtftwQrbV+skmtawZSE1MBvJUUS7mFU1udyv7i/XUeX1ICp5xSz8k+/dT4\n3qNH+BqoFPz2t+hly5j8wR85/7TzGZUxKnznN8mcOXMY5BkILEwjcTefxNx8EvPYFWx3zyyMJOVv\nQJX7cUM0EPNJSqLVGOfhqaTs31VJh9YdOFx6uM7ZPPXO8NEa5swxZuT87GfhbeRFF6H+/nde/Ggu\nRa2IySRl0aJF0W5CXJK4m09ibj6JeewKNklpBaC1rvJ9HA+Ue2XYCnekVGUFHVI6cLjssPeGxr4c\njnpOlJ9vVFKWLQt/I/v3ByDzR1jdM/ynN0NKSouYsR5zJO7mk5ibT2Ieu4Iak6K1rtRaV9Z83NBX\n5JptPk93j7PMqKQcKjnMe+/VPs5ur+cETz9tjB/5zW+Cut6PP0JlsBE85xxKEqD/gSCPr8eWw1u4\n7MXLOF7eYm/FJIQQIobIwNkgebp7HKWVpLdK52hZ3TcZrLeS8u23xuybIO7Z43LB0qWwfn2QjbNa\n+fJ06N/E+06/+9275Bfmk7tZbmsuhBAi+kJOUpRSo5VSa5VS+5RSpUqpshpfpeFsaDTlXLmMiR0/\nAsBRVklqUirFlcUAdO/uf2y9lZSiImjXrtbm0lJjVrIvT6LzYyOSjo2nG5UUq7IG/6QaiquM13Sy\n0vx1+KZMmWL6NYXEPRok5uaTmMeukNZJUUo9ATwIbMW4j0+L7h+4ostv+biV8QHuKqskNTHV+EBv\nBd26wQ8/VB9bZyVFa+P+Om3b1tr1xhvGYNsJE2qfQzdiubxt3VK4779l/LbDwOCfVMO+k/sAKK0y\nP7/s2rWr6dcUEvdokJibT2Ieu0JdcXY8kKe1/l04G9NcdesGA4YkAcaYlNSkVErsRtLSqsYQYofD\nuOffoUMwYIB7Y3m5UWJxJynX5l7Lb3r9hjsvurPO2UCeaoyrETcz/LST8aQeu0PPF789/C0ApXYj\nSTlZeZKdx3ZywekXhHzOYN1zzz0Rv4aoTeJuPom5+STmsSvU7p4U4P1wNqQ5s1ig988SAHCVV5Ca\nmEqpvQSXdtWZpKxcCd9847PRs2Z+WhoAy75bxl3v3VXv9ertMqqHw+Vgczs7pQnQ84fQ7ojs0i6+\nOvgVUF1Juea1a8h8PjOk8wkhhBBNFWqSsgaI/J/XzYlSuJKS0eVGJQWgSpf6JSkJCf4Jhrfrx5Ok\ntG1Lmb3Mu3/v3upjfbt26h18W4+fSn/CZYFd6VY6Hypr+Al1WLR5EZVOYzqRp5KyrnBdSOcSQggh\nwiHUJOVOYIhS6o9KqTbhbFBzMmnSJLKyssjNNWa76IQkqKz0rkJboYtJTq4+PinJSDA89/LxLpHv\nk6R4xn0k0Iri4urnehKTrVurqzBBTAQC4Jcv/xKAfR2TOONQeQNH1+3mt272/tuTpHg4Xc6QztkY\n27Zti/g1RG0Sd/NJzM0nMTdHbm4uWVlZTJo0KWznDDVJKQBOw1hR9oRS6qhS6qcaX4fC1soomTt3\nLnl5ed7bfOtEI0lJT0kHoIzDfjcXTEkxKimeGwx61zk54e6CadvWO3XZTjknK6qTAac7D1i3Dnbv\nblw7tx7ZCkBhhyTO/Klpy9O0b9Xer9oDUOGoaNI5gzF1ajB3WhDhJnE3n8TcfBJzc2RnZ5OXl8fc\nuXPDds5QB86uwlj2Pq7opCQs9kq6nNIdgOPsBjK8+1u3NqYUe3i7fnwqKSd/3Ozdv+rAm3TAWMK+\nri6eYCspCZYE7C47O9omc3rRcaiowK/E0widWnfiu6Pf4XBVN6jMXkbrxNYhnS9Yzz77bETPL+om\ncTefxNx8EvPYFVKSorUeEe6GxITEJKyOStomdCTZmsIx7V/ySEkxiiae8SXFxfD553BRURFYrZCS\n4rcGyeoD73BWeTIXtrqxziQl2CnIvzrrV7z3/XtYOg7Dol+CnTvhvPNCeoljLxjLlA+n8MWB6ls7\nlztC60JqDJkiGB0Sd/NJzM0nMY9dsuJsYyQlYXFU4nAoOiSdQZHT/07ICQlGRcSTXKxbB19+CUU/\nFBnTj5XiRGX17JvVP77Fv4tuAqq7e3wFO4D2ZMVJftHqVn5KP83Y8P33jX5pHteccw0A1y+63rut\n3B75JEUIIYSoKahKilLqRgCt9eu+jxviOb7FSE7CZq/Abof2Cadx0mUsCZuQYMwuttmMLp6aCYcq\nrl7Irb7VXN98E8aO9d8WbJJSVHGCdJXGyVPaU5wIKd9/R2PWnfUdc9IuuR0/6/gzdh3f5d32wc4P\nuPH/bqRgQgFWS+gr2gohhBCNEWwlZRGQq5RK9Hnc0FeLuwGMSk7G4qjEbod2CadT5DSSlFtvheuv\nN5KU8vLa3TSqqMibpExeMbnWeT8rz+Wk86day+O7XMEt6Has/BitLGmU6KPsaA+fffRKo17XiYrq\n6k6rhFbc0OcGLKr6rfGH9//A14e+jugA2tmzZ0fs3KJ+EnfzSczNJzGPXcGOSekDoLWu8n0cd5KT\nsJUZlZQ0ayd2OYy5wp4ZPpb6Ur4TRd6F3OqSUzSSvonDuP7kB7X2ORyQmFjHk9yKK4s5ULKfYWnn\ncHbiZXzffg7pW7/B7rSTYE0I6mV5uqA6tu5Im8Q2tLK18t7Hx5dTR24qcllZaOu7iKaRuJtPYm4+\niXnsCipJ0VpvD/Q4XqhTUrEdL8FuhwSVgl37j9U4Wc99+VRREZzWFq01NouN0Z3nkLPvj37H/OT8\nnoo6ChUNJSnfHf0OgNNtfUi39eD7U2HAPlixupLTT00gM4gFYz2VlA9u+QCLstAqoVWdx0VyvZTH\nHnssLOcpKjJmRQXICYWPcMVdBE9ibj6JeeySgbONYGmbRmL5Cex2sOpk7C7/NUl69ar+t9Vn6IZn\nTEqVswqHy0EbS/ta59a4/JIUT1Wmvlk/noXiSqqMf6RYjDss72gPZ56EnVuP88UXtZ9bF884mbQk\n45M92Vb39GXfacnN1fLlsHhxtFshhBAiHEJdJwWlVDowGsgE0qid8Git9W+b0LZmR7VrS1J5EUer\njCSlSvuXPnwrHikpeFeUtZ40xqR4ulBaWWr/mW8jiXKfwkxSkjG+pa4k5cUXje233w5VTqMHzuYe\nLrT3FPfzT+yH5DNxOv0TJo9Lcy7l3kvuZUS/Ed7unlOSjCe3stVdSYmFJKW4di+VEEKIGBVSJUUp\n1RfYAvwV6A/8GjgLuAi4GugNtAtTG5uPtDSSyk/gcIDFlUSVq/4kxfeePtZid5JSaXyCJta4k0Ar\nlUaZPl69Qi3GjCGoXgfOlydxsdurkxQrxhOOu69rKzsCUGcXktaaT/d9yt3v3Q1Ud/d4k5T6unsi\nOCblyJEjYT1fsGvMxLtwx100TGJuPol57Aq1u2c2YAfOAwYBCrhTa90Bo7qSCtwdlhY2J23bklhe\nRFVV3d099SYpJcbAWU/XTDKp3n1PdNjFdalPUOo6RmVl9Ser51wrV8J+/+VYvPySFHclpcjdU5Pg\nTlKcTpi5bibqserlaz2VE0+3TnFVMcm2ZO9A22hUUsbWnH/dRJ98AgcOhPWULVK44y4aJjE3n8Q8\ndoWapFwG/K/WegfgmSRrAdBa/wdYDDzd9OY1M2lpJJQXU1zkxOJKwqmdfh/cvvfx8SQZFnsl1qoK\naNuW1btXG/t8Kinpth60saTjwklReXGt5wMcP153c+x2sLuMtfdtGE947Hpj+eeEMmM+s9MJj699\nHIAPd34IwBF3AuNJUiocFX6JSX1roURy4OyMGTPCch7Pz2DLFli2LCynbNHqirvDAVVVtY8V4RGu\n97oInsQ8doWapNiAH93/PoGRqPh273yF0fXTsrjXOtn77Uks2viAr3TUfUM/zziQxPLqmwt+85Mx\nZbmT7Wy/Y6+41KisFFdVTw/yTVLqWyvF6fStpBhVkHO6DzCe75OkeLpphr0yDIDDpYcB2HV8F+ox\nxecHPvcmLC4XbP22Okn5aPRH/O3qvwGRraRkBjMNKQhJSWE5TdyoK+5vvAFLl5rYiNtvhw8/NPGC\n0RWu97oInsQ8doWapPwAdAPQWjuBPcAVPvsvAuqZkBvD3PNaE8tPkKCqqxB18YyJSHVV31yw0lnJ\nZV0vw6L95xQn2owEw+6sTgJ8kxTf8RW+CYtfkuIek2KzpnAiCbYcexGXduJ0gt1px9fhssN+j1fv\nXu0dh3LyJKQc/KV335DuQ7iw84XG9SI0JsXhgOefhz17mn4uSVKarrgYjh416WIHD8KCBfCf/9Te\np3XjbwkuhGhRQk1SPgT+x+fxv4A7lFLLlFLvAuOAlrUkPngrKUnlRdgwPg0bSlLSfJMURyVJtqRa\nlZFE91iQKkfdXUe+y+z7zvZxOmHTN1VYsGJRRvXDShJFyaBKD/Bd1RqcTtA1bljtqaRUXz/Rb9qx\nUoqXzj3Jjnt2GG2xGI2JVCXFM6tp69bwnlcpGUDb7OXnG98//bT2vpUr4ayz4OuvzW1TvNEaXngB\nunSBb7+NdmuE8BNqkvIEMN5nmfynMGb6nIVRYXkSeKjpzWtm3JWUNs7qSkqls+7uHk8i0t5qdPe4\nUtOodFaSZK2dpCRYjSSgylFd8fBdvdZ3fEDNSspPR+1YqS67WHUSx5OhbQXMPXYlLhekJRo9cUlW\nI7GqWUkprSr1jknxJEGJpHJW+7OAyCcpdjvk5+cEdQuAhvieQ+v4HFvhcsHrr8NPPzV8bE5OTuQb\n5KE1XHklPPhgdfa4dq3xfccOOOz/vmTzZuO4f/7TvDaawNSYN0RruPNOGDfOGGm+Zk20WxQRzSrm\nolFCTVJOAhs8y+RrrV1a62la6z5a635a6we11pG70Uu0+FZSVN2VlHbtjFzGU/1o7TAqKWWJDVdS\nfLt7LBbv5bwftO+8A2+/Xf08lwucuso7HgXApoxKSjt3syrtDk5UHad7wkXebp+alRTP7B6oTlJ8\nqzdWd5UmUgNnT56EwsICv+pRqFwu42dw3nnG48q6c8gGrVxpdEHFom+/Naaub9rU8LEFBQWRb5DH\ntm2wejXMnm18gXGr8CFDjH9v2OB//A6jksd//lO9nPPBg5CTE9xNrZopU2PekAcfhH/9y+hy69fP\nSAxboGYVc9EojU5SlFLJQBkwNfzNaebclRR18oS3u6fmwNnf/x5uugkuuQTat4cOtiK0UlQkpHor\nKTXvkmzzVFJ8xo507QrXXWcsCuf5oD10yH+xMocDHFR5F3IDY1G4462MSgrA/33/stF0y+m4cOHS\nLo5VHOO8Ducx6ReTSLAYCU7NJMXzGbBrF2hn5Lt7Ro6c7zdtuz4HD0Kg23BoDd27Qx/33aXqWicm\nGLt2NXxMc+V5ze1rL2xcy/z58+vdV/N92mRLlxpz8x94AB56CF56Cb76yrhDZ6dOtbt8du6Eiy4y\nXtALL8CRI0YlZvz46gpMDAoUc1O9+SbMmQNz5xqVlBacpDSbmItGa3SS4q6QHAJKw9+cZi4pCZKT\nsZUWYVXGB3d9g0lPOQX+538gsew4Vcmn4NSWeispSe6Bs05tJAGjR0PnzsblTjut/i4Llwscuso7\n/RggyZJCeetE2rnHeTz46TgA0qynA8Yg2gpHBZ3adOKZq54hPSUdqF7AzfPB5HQa3TArV0LBRmvA\n19pUdVVv6lJZCXl5BFzu3+UyxqJ4BtDWqqS4uxm++67+ey3VcXhMCUdFCoyfv6+KCvjssxBi4nnD\nL1sGv/wlzJxp3DZ87FjjZIMHw4ABtZOUHTvg8suNzH/SJCNzP3LEGDtR10DbPXvghx8a2bg49tZb\ncOGFcN99xmNPkmLSm37NGuP/sxCBhNrd8wpws1IqTL8OY0haGkkVJ7AQXHXBtmcXxek9cDqN8SuJ\nlkTv7wDPOTxjPpwYnwqqet01kpLqT1KcTuM5VuU7FcjCdRfexZlVHfyOPcXiTlJcdiNZco9PSVSt\njetYa3f3eD6kSosjW0nxJCeBKvhLllTPUg0080Rro6vMk6T4VVK2boXTT4f33+fjj+H99xtuW6jd\nRcGKxOeB52fY1PE4NW/J8PLLRhdSo2b+fPutUdKZO9dYYe+aa4w3+AsvwJlnGj+Ps84ykpTPPque\n4mW3GwlHr17GfSDy8uD+++Gjj4y/+v/v//C7jwQYSc9ddzXlJccPrY0s4QqfSZn9+hmLMv34Y/3P\nC6Pt243KqBCBhJqkbABaA18ppSYrpYYrpX5T8yuM7Ww+2ralX5ciBg8K7oPbumMbRaf1xuEwuoYS\n3cnBP3pvYWZH4xeyp5LicldSfAfN7t9vjCes+VctGB8iTl3lnX4Mxge965R2tC737xNJs5wGGJWU\nSqdR0QGwOFKMdrpqV1K8XT+O+l/rX9b+hSfWPhEwBg3xXKeu+xR5tv/0U/UKsoEqBS6XET+bzfgs\n9MatqMjoPzt0CGf+pwGv9999/2XJyYeBuuMeTgsXGkMxwnnPoWCSvmAEWp8naK+/DidOwB//aJzw\nt+7bebVrBx98ALm5xg/qN78x9nfvbiQbe/YYF+rVC5KT4dpr4c9/hr594ZZbjDKY75/hlZWwfj3s\n3Wky1toAACAASURBVBvqy62Ty2XidGwz7dxp/HK5/PLqbf36Gd/r6fKJdMIuRF1CTVLeBPoCfTBm\n8rwBLHN/LfX53iwopc5QSn2klPpWKbVJKfU/DT+rHmlpJFeeoNuZwSUplu+2UdTJSFKqnFXeJKVn\nah/aWjsD1bN76qqklLo71eoaW2G3u7t7fCopLhc4U9uRWuH/SZJq7Wg8x2V093grKe7vVu1fSQHY\nt899Tmf9A2cf+egRpn00LWAMGuJwwPz5WfV+KNa87UZdN0z08HT3gHH/I+/rmTTJyPbOPRfn1u2A\n/1o0vq546QqWl/4Vl3YF/EB2OOq7S7Vm0AuDWL93ff1Pxvilb7cbM2xzcwMe2ig1xxUFkpWVVe++\nms9v516usVEVmrw8GDHCqIJkZxtdNR7nnlv9Idmvn5GFTp1qdOV4+vTOOqv2Oc8+26i8zJ1bnUVu\n3AgVFegw3wvh66+NoRuBxkE1VqCYN9qJE/Dxx41/3po1RjY/aFD1th49jDFDdSQpVVXGEKItW5rW\n1GhVTsIa82Zq2zaj6NjShJqk/DrA1298vjcXDuAPWuvzgKuAeUqpIIZp1qFtWzhxIrhpucePow4d\noui03j7dPUZS4LeEfo0xKb5Jyq9/bXyvK0mpqoIS12Faqeq7Krtc4EhtS4K9gp+p6r+SPAN97U67\nd2wMQILF0+1jhMO3+8GzhIVN1f1afZOWmgOIDx8OvivD4YArrphYb2WjZlXfEuBd6+nuAbBaNW/s\nXEBl0VFYvBimTEEPHkzFt8ZI/8REjBLGz35mDAK6/HJwuSh3GBcsdR31Jinj3hlHr7/38rvWK6/U\nPTSiqKKIT/Z+wsOrHw74umu+rnBpTJIyceLEWtteLhrHy0Xj6k3Qgk5SCgth0yY2X9oLnnwSXnst\n8PHt2sHdd3uyViPLPPPMuo998kkjMfnjH43H7oG06tgxftwdvomFx4yFm8NaURszpnbMQ1JUZAwk\nvuIKI9Y1rVxpzJ6qy8cfwwUXeCcDAMZ/nPPOg2+MlbEpLzf6WLX2vlebUqhavDh6Y1Dqep/HsnXr\njCKlr//+13ifxuI4ukCCTlKUUoOVUh0AtNYfBPMVuWY3jtb6oNb6a/e/DwFHgCDmPtQhLQ2KioJL\nUrYbf7GfOO1cI0lx1Jek+FdSfD+EWxtDRrwVFV92O+x1bOKMhAzvNpcLHG2MP3nTq6q7gTxdSnaX\n3TvLCCDBXYWxYlRS6nqDe+7l4xk4+5//GEMN3t5WPR+61F7dwGPHjKnSwS7O5nBA377D6v1Q9HzY\naq1xaaf/h++qVX7/W30rKdvtq/jL17fz8d8mGb9ws7P5tPVxbDu/Z19lgRHnl14yGnr11cYHnc+0\nnhOuH71temHTC+w8vtOvXVVVtT+8th/Zzs//9+cAtE5o3eDrjoTGJCnDhg3jSNkRv1WJPyl/gU/K\nX8Dp9H8zeN4bQScpS5fisCoGHviL907bDeraFX7+cyND7tmz/rLZwIHw97/Ds8/Cq6/C2rW4Uox4\nH9wUvj/XPa856K6zL78MWBbbuRMOHx7mTX5CVlVlvGd37zbe8KtX1z5m6lRjDZSatDaSFM/Ub1/9\n+hlVlj/9yahiDRsGK1Z4u3pideb3sGHDInbuN9+k6T/PRtq61chRfXl+Ri1tbajGVFI+An4VqYaY\nRSnVH7Boreu5t3AD3JUUz9ohx8uP89XBr/wOGfziYB796FGj/gaUdD7HGJPirPRWLhKq8wfvOikr\nSubgqjGDxtMlUVJSuyk//qg57NhJJ9u53m1GkmIssJJeYfx4b037N61bedZi8R84a69wJ01U37un\nJk93lCchKy+Hzz+H37/xe+8x5fbqsoCn+hzM7Bmo/lDV2ignv/uu/35PovDEkf7MPHJx9Yf71q2Q\nlWWMUfj+e+85PEneEaexpHqflZvg0kuhRw8+sO4mxQHJx3cbL/Yf/4Dhw42pmMCBj6vvSnjSddBY\nsbcRf5r0nt+bfSeNfrKUhJSAx0bql0ljkhSADk92YPzS8cZzdPWTSiqr+zi2bKn+pRh0VSEvj819\n0zmZDJ/uq2NF2fpce63xvVevWru2H9nOuj3u6sAddxhdSBMnQn4+5b80nnfgi/B1+Xh+9DUT6B9/\n9L7lqlVVGV1bt99ebwbqqYgGs9BeQPPmGV1iy5cbFZFVq/z3OxzGD+3bb2t333z/vdGXW1eScsUV\nxr6XXoJhw9BpafDZZ94PwEBVzHh07JgxZqlm+KMp0uPozNaYt5xq+JDwUEpdppTKU0rtV0q5lFK1\nOhSVUncrpXYrpcqVUhuUUg3e0FAp1R54Cbg95MbVqKRMWDaB8/91Pu/vqJ4qsq5wHX9e+2cqNn8F\n3bpBSop34GydSYq7krLDns9Pru1+l/MkKXUNrDxwpAQHlZxi6eTdZnT3GJWU1ArjN0uyOoVWicYF\nN+zbQKWzkmRbMgsXVncD2XTt7h4P7+wjl9P7u/ekw3+giKeLBKrHkAQaO+LL82Hqchl/QO/fX/f+\nvY4vKXQUUGV3GZnSjTcaf3l37gyTJ3uP/fe2vzLwhYGcdB3g1FI4Y8MWGDkSgB0djUalHdxGx00r\njLnI994L6enQtStvvvon73XLXEU4ndV3jQb/ZKwhiQSupIT6y8TlMpLE+taAaczAWU8Ctuw7Izk7\nUFz9AV/m81o9XX++5w/I4YD8fL4535hVdrSsEaNPPUlKHeNRes/vzeCFg40HShmVlORkKC6m9Lc3\nAZB0LHyzU+pLUpYuNSYa+Zk3z3g/lZZCPYuHef5PBPuzr6oyftZ+P8s9e+Cxx4z37cUXG10+q1b5\n/+fdsaP6T+vFi6tP5ml8crL/zB6PUaOM5+3fz9qxCzlwWn8oKPC+/kh0JYR9PR4Tef54bNMm8HFm\nilSFNlqaa17cGtgE3AXU+m+hlLoJeBp4FLgA467LHyil0n2OuUsp9aVSqkApleRewv9t4K9a6/+G\n3LIaY1JKqox36a9f/XWtv7g3rHwRevfGajV+ZzU0JgXAhf9vL6vV+H3iqUo4tYMSl/ELv9hl/DnW\nxmJMN7ZYjOLC9p+MJKWNb5KSZFxj1JJRFFUUkWRLoqoK7/Rlz8BZrWvfpM/T3bP0u6X8eML4wC51\nHvc7pq4P72B/oblcsGnTEr9fxPXdCgDgeNXh6g+EN95gy9inYOlS9Erjz5l53zzM+r3rOebcz30b\nQCtlrLUB7G+fQKUV2h/aQfelf4fMTKPKApCZSe/C6tdRrk/gdELu5ury/bHy4Ou6e/fUMzLXLdQk\n5fhxo1fhv/W8i30rU3V5911jNd0jR2DR/y0CqhPRncequ7R8Kym+gvpQ2bIFysrY0t347V1UUdTA\nE3xcdJGxfsfgwX6biyurM3VvN2v79sYUqV/8guJLr8JpS6RdeeQrKbUcOQKPP250r6SkVC82d9z/\n/4nTWfu9HsjWrcbP2pu4O50wYYLxe2jGDGPb0KFGaWe7zx84nvsdZWUZ3U8jRsAZZxh/+uflGevV\ntK4niXb3l27bBke69YeNG4OPw/9n773D7KrK9v/PLqef6ZOZSS9AgJAEQkIv0kRAiCCoNBV8VVRQ\nBMXyWrGi2BALomAEgVCUJKCQUBMSCGmkk57MZDKT6XN63Xv//lhn7TLnTBJA/fq7Xp7rmmtmTtl7\n7bX3Wute93M/z3OI5n42D3TMgQEn8fA7sXnz5r3zg1QwuVE41A3Zv9oqjfH/6yDlPyLJsSzrWcuy\nvm1Z1nwqMzi3AH+wLOsBy7K2AJ9BZMH9hOsYv7Msa4ZlWcdblpVDMCgvWJZ1EPXeQWwIk+I2WZFY\nWkt7DI46CssS6elNy7SZFDdIcR/LVMpXLk1zRJaPxm/mS10Ci/207zQAqlURuaOqYq5sTwl3T3M+\nxIntEM1D0O8Aoe5UN+lEqR02SBFMimmK853nFEJGMUX7Hlz/IN96WTANeUssYDefdDMAi19LM3++\nt92HOqGZJqxY8YgHILmjKYZO6LH+LUI4+elPw9SpLG2+nP5Rx2DNmQNAXUD0j7LjZb66DDZ9cjY0\niT7KUGBHPUzduIKmVc+I3agUsRx/PMd3Yj/lGVOAlGV7l9nn7k31C7aqwkQgAau0rFE53+G8eULz\nORSkHCqok/26dWvl9w/m7pEL3qpV8EhJPyGfwe0ukJLKOzehurr8/Ae0lStBVVnVIhoxHEjJ5ytc\nt6qK719+uefln7/2c/tvN7vF+94Hr71G0RciXTOS4MCBmZS9sb2HXOLhkDUpL70kttXf/KaIPFqy\nRIhOR4yw3b4g+m7FikcOeWzIZ8QW03/720IQO2eOc1POOENQs3fc4bg+N2zAam4h8bEbhRDmqadE\n+77xDUGLHWK0S8+4mbB3L0pvj93+d2qm6T3OgY75zDOV5TZv1R45SPhcPP72IrgOlj7h322Vzvt/\nHaT8VVEU4xB//i1dpSiKD5gJ2F5AS1AYzwOnDPOd04APAZe62JVj3lYDamshn0fPl19etpi1d3i6\nAYcNAEeJ8OOiJVgNn1Lu7nGDFINyoYKul7J9Zh5mcfp3ANzQqRA3uwAIq0IDLP3Fhi+EofuZEatl\n2X1w2qsLCfq9oKp1p2iHUnoEVNNhUhRlSC4S0/lH1gnKWWIBPmvCWQDs3puhq8u74BzqYLEs+PSn\nH7VznAz9rmlC3nIYjvFPf0+gtv8VgCkQVNg943KUp59CLeZFQUULvvX4drbXw9rrLrC/G8vG2NoA\n07ZtJlfbJHaY0o4/noYMjCtpPDNWjAe3/I7HNj3GhNoJAPzzhQTPP1+ZBRnIiF3zM9c8w9TAReSs\nCkIihB6hEkg51N31wT53qO6eTAbu+MMdgHgGDQP6Uo5bxg1SIhER+Vtf/xZAypQp7FdEHwxkB8o+\nYppOnpiDmWmZ/Py1n3PaWAHMh9afAtGuVM0oAn3lTMru3WIhSuVTjPvVOO5YeschXMRbYFJef91x\nPZ55pgi/+O53xRddfqFi0XnW38r5TRORB+ZHPxI/73XJAyMRIST+y1+Ea+dXv4ING9g/Yhpzu89x\ngMlNN8E994iDXXzxQc+tqiUmBfBvWG23I58fXm+WzCc55nfH8Mz2Z4Y97tDQ/QP1rXSnvFM306PS\n5TWMzZ0rovXeqsm2/7/SgRQKYhy7n6f/y5oUEEDgd4f48+8qXdoIaIjU/G7rAloqfcGyrGWWZekl\nVkWyKwetSX7RRRcxe/Zsz88pP/gB8wA97lqAdgAPC12GLDg4YRC+aMJ9O3aIfCYIkNKxfR+//e1s\nEglnJ6gqqpAlL/WClLa2NmbPnk1X1xa60h3cN3iNeON1YJFzer8SJp9Pc9dds9mxYykoCoVILWes\n2MDjFty39ClbkyJt5YOPsHHjPGpqxdOtmSEWLVrELbfMRlG8IOovc74AJRe7ZYotXVvrOngY9KwA\nMIUSiPj2t7/Ds8+K4nFyAMvr2OLaUQLcfffd3Hbbba7oHSgU0vz2t7N55ZWlvPCCYKdNExa//juY\nB9VZuOClxSKz6MiRfOQjH2HjxnnsnvFBlFiMUVtfIr8tR+QvcGo7fP082JlqpyvZxY033si+xfvY\nWnIKbj73KtZs2sTs2bPp7e0Vrh/gM7n3Ub1kBFteeJHvrrwRgNFVo2EQ/viTW1m+fItnYpXXIe99\nSA/hKwTZ+Zdl3PHwHWzrbmXpUtEfDz30CHPmXA+Iyf6O3pNZmPkhv7/3g/z9715KetGiRRXzO3zt\nazeydOl9dp+BKKAmr8PNpHznO9/hJ7KYX8n6+9v47W9ns3fvFpvhMIs611xzN3PvdCK20oU06XSa\n2bNns2HDUjRNLFyGIXam119/fVnbPvKRjwhqfcUKOOEEUvkU7IDHvvlY2Wc/+1lxHe5M9u7rcNsX\nv/pFki8muXqa0Bbti3WXPVfFIqRrR/HIjuXcdtttnu//4x9pzjprNn9b+DcAlu8TyOhg1yGfTcOo\nfD+KRbjxxhu5b/58UbAL4MwzWTM4yOxXX6U3EvEIen7/ezE+3IvKgcbHXXfdZp+HhQtJNzQwe9ky\nlrpFQsAjl1/O9ZdeKrRZc+fCG2/Q0zKNP9x3DfNmzRLi2ttug3CYRUceyewbbii75htvvNFTLdjn\ng42ZGJfoOomVS+x+eOEFuPrq8ueqra2NSy65hM2bN3PRwxexZ3CPfR1f/rJzP4pFiMfFON+xY6ln\nLA13Pz784Y+UuWyGGx9DrwOGf64qjY+DzVduSybFdaxfP+R+HGx8vMPrSCbFdRQKItryE59w5l05\n/t/KdchxXvZcHcJ1PPLII2JtPOUUWlpamD17NrfcckvZd96uKYcauaAoiglc+47dJW/RSue91LKs\nBaX/RwL7gFPc2hJFUX4CnGlZVkU25S2e83hg9erVqzm+tHDZ9sorcOaZmG9uRnt0iuet3TfvZtfA\nLs594FzO2QUvPADs2MG9LxxGzOjkK92jeOCCBWTWXWJr3UB4LZTbxeL/5eaF3PkZb7jc/Pkwf89f\n+PPgdRXbe3dLGr8SIhx2KMtrvn8UkXbhD+gZN5PXH3mUS55zoiVuqP0bJ0Q/yBPWNTy772G+Nn4+\nP75uNitWiCjcc891Ki77/XB9q2jfBw+7lvelH2RjYQF3936AzZ/bzJTfTeHk0Ee5vOpObrq+mQdE\nTUMmTfK6jYaz+fNF8UQpPksm4eSTxQ57wgRoaIDT/hElZ6X44+unce1zywi2dYiU6sA//wntey2u\n/s4EdhxxJld9aD0X/mM9P3wRGr4CGT/MGjWLFZ9cgf8Hfj61rYmfze3gnp89zq03X4FlCXnL5MnQ\nXauTrm1kUEnz88vO4qFakZPwQ1M+xOObH+fTtY8zM3QFV14p1gJ5/wA2dG1g+j3TWf4/y7n14T/Q\nWXyT3YXljA5P5Ns1uzjvPFGLSe7Ypk2DUxc53sy/XPIIHzvexewMY21tTkr/66/3AkoQ6euzWXGu\noXOfZcEf/yj+HjECzCP/xhWPX8HEyDF8rXojL+i3Mr/9XnJWii8dewc/u/SrgAizbGkR7sSaGmg6\nZjMXP3wxaz+zlupAtfck2SxUVcHdd9OS/C5dqS7OmnAWL33cqzSNxcTGPxIRP1OnVgzoAeDOZXfy\nrZe+xYbPbmDybyZzQ+0T/OJTlxN2BVC98Qb4bv08o7a9TMfCDUyZ4jBzsqL1yPc8xey5s7liyhU8\n/qHHD9rX//ynCHY5+2zBJEmTx/voRyGkF0SnfP/7AiRkMuL/Y44RETRPPmnXFFq+XMhFpk0TXiFp\nmzeLcTZhgmApZHFI+fmzzoLJX7hA+GKHhr+5bdMmO3Psyx//M9tOvc5+PgHha6yvL9P7VLKHHxZj\n8eMPnEM+XMuX3n8l0xtOxJ+eAIg6j0OjfZa0LuE9c0R+pmumXcNfPyge9kLBSTR25ZXiOZTkxhVX\nDF8MU/bz1VdXFqfmcuUaurdj8jyevjoEW71a/FRXe0nZf7fJ9p54bgeLFkFjYJQNTs49t3IOxP+k\nrVmzhpkzZwLMtCzrHZWg/m8Vzh7IegEDaB7yejPw789nWEp+pMYcvlMpyWayxSznPnAuAKdZY8Sb\nY8TvQsndo1fQpLjNVCq7eyxr+OAqXyl82C3eMqqELiVZN5a6zs0ENe+tHuebIVLfW+J8uiVEdJXc\nPe5dn6TtCwg0JAsULs88yG8GLmZ7Tys5M4WivDVNivwtJz03nWyawr0ULMCHlm5kzrEQr3NmLJ8P\nUBTumdRGyxt/JWBpXLQdXpogAArAqo5V/GTZTyiaRZTzbmXcLdBTLfpt506RGmL3bnh6ikY4W2Ri\nb5bLljjK1FFVIjvw/qJI/lLp2iST4leDBJSo7e4ZyAvXhGU5QjtdL3eHvdZ+4Ay1Q/sLhi+XAMKt\nNGeON2mc+7uaBuu7hMByd2oTvcU9ZMwEVaVosZ+v+5otFDYM8XlNE3/f+eqd7B7c7RHa2rZ2rWjE\nCSeQKqSoDdayoWvDsO3M5URbV64c/po3dG9g1qhZ1CuTAEhbA2X9ZxiCSQnHOnj1VScnmfte7RoQ\neXAqaVKee668DW4mpZIVi4gQ30yGfUePYda9s0ioRfjFL4Rr5fTTobWV7I52zzUPdfcsXSq0FytW\niLJE7vIUAKZhiTclWzOcHXOMzQj2jZle/v6llx4SQAEH/GanzCS4YSV/HPgId+x1cnRWclm5heUh\n3cmXOdS9c6jungN95p1mwc3nBVB2j6G3mgfmX1WC4q2Y+1wz/jqar3aP9rz272rL2v1r2dzzDlIO\nv037/x1IsSyrAKwGzpWvKYqilP4/tFn+nVitWPyJOcmpfKU8J3KRAjg8FaAr6pTjlW4czaoMUmY0\nnFb6XPmqo+uVX5emlFR17ky1uYiI8Fn3vq+gFzLUdgk/fb02ji/WP0+jPhGAnuw+FBPeu/B56Oys\nCFLcE0RXTLgHpCalPlTPiLCILuoqbmP6/RP4UlcjgcChgxTLgjlzrveAFJlHQtfFoKtTx3DXhvOp\nSib42akQzzqiVDko/zodmlLwyed7OLMVnjnCe56vv/B1AGr1CfRFIF0U0SJywszmTD51QYEf/ORH\nPHHO0Vy4vpua0gJfFxT9uSD5baByjpOcUQKiSpCAEiFuCszsU50oHy9I8bKY2iHW63RPQsMJ5xTF\n0Q+4Q7rlPZH9+tD3H7Lfezn9G5L5BFUlITbAF575Ai/segHDEPdGghRJ5VuVtPQrVoDfjzV1Kql8\niqMbj6Yv08eCpwtCz5ATCFQuDodSBTuWixGw6vjbExohpYaUOVA2GRsGpGpHEUz1oxZy9mIo2UXL\nsvjyc18GYEd/ecjI7t2CjRl6TPD2uZt8LhQQehRN49eFpazuXM0b+98Q+o+TThJaEWDVXUJ8/Xjr\nb7j7/ouGvVb53MvnS/aRvnu7CHU5GEgBUfsoEmGw5Wjg7Qsp5aYnfvxZ+Pa3c0QfZEwnwqrSNbhD\nzYO62ATEYl4gcSBNSm+vyM049NiVziXv66HUQ3S7LCzL4sk3n+TJ+UUefdQLUt6qnmMomPxPmBMO\n7jyI/wmQMuMPMzjmd29PyvlO7L8SpCiKElEU5VhFUY4rvTSp9L/Mkf0L4FOKonxMUZSjgHuAMDDn\n3964xkaxAsjCNoBfE4uQOwx3XEyhrdqyHyQpnFWMyiDl3jOF0KyScFbTIGUePPTVzT4U60aQHHMk\nu44XZYr8m8WO95jABRwdsPEd6WKCy7bAmY/eAXPnegr0SbMseOPTazkschzJnFhl82aaoB5EUzVb\nPJu1SosPWfz+t8akTJlyvg2Q3Kaq4v1z3uzjE08/z8r3XMzOBkhkHVGnPM+6kXDPLLjp6Xb8Jjwz\njOugmtEoKKQNb/KZZCGOhUVYqWPpGcfiM+HaUiRn1O8wN6ZleoqtyUlBglSdAAElStIUfmMZdm5Z\n0NVTxLCK+HyQzHvDCaQo+WBWtki6TPaFuy6R2w0v2xoMioWiYVoD5x92PjPrz2bAaKcvkaBabaZJ\nEwjvoQ0Pcd6D52EY4pkYClLShfKQiBXP/IktLTrqHUEsLEZWCbfcjvY433vhTmruqKEj0VG2eB4o\nUdhgdhC/KVjMsFpH2ixnUopF6IyKCwzHOikWRfStDOx4M/8cRbPIxVtF7hx3lt3hrBKAcvd5LocA\nKdOnE9PEG55w/JYWYiMOo3aTACm/3vF5No55hkfbfnnA8w4FKYF1K8QfJ5540DbvuepCrvrxTLI+\nxXMst732Wll0dJnJ56z76Pdg6j7O3yn0b9IqgpSMA1JkQIB0iUgrFr3fdd/HdetE0sCheaGGY1KG\ne2+ouTPOvrj7RT742Af5x76/lJ3/rRZRPBQmZc4cT4DXOzZ5zr3FNw74fqXXly///19G2kMGKZZl\nqf9BPcos4A0EY2IhcqKsAW4vteUx4MvA90qfmw68z7Kscsn/O7BbbrmF2bNne8PXwmHh8Nvg0Nc+\ntZxJaRnI01bjpIuX7h6M8ugegIAuVhXpfnGbz4e94B2KTZ4M4Z99nze+PY9MdTPZaANju1u5se4p\nPlL9a65wlVf8xclz+d1rYhFh3TobKAxt37SmYzk6fAZGqX2J4oCtRVCV8iQBB2NSVq4UflXTFD8n\nnniV/Td4d9m+nZt46LEMa6Yex0sfF4KvRDbNS7uFxsE9QfzvudBdpbGtHnY2VD53Lu0noERtkCKB\n0WBOzNoBaimOGMWCI+GG0hN4TJOzg0iavZ7JbOnyLF9c8E1bhKoTxK86OSj0EvgoFOAzr1zIHb0n\noSiQyHtDJPR/AUiRE677/smUGeD0VSAgvtt4UiMhPURjYDSDxj4yZpKgWsX/1DoMCzAskzIUpOSK\nOQqbNrCy1nl9ZFQ8X1kzzt+2i2JHrYOtbxmkhDXBYkbUOtJWZSblj8oPAAjHOsjn4ai//ZAZ//g+\nAHf1v4/qLDw+P8C8vxrEn3uag1klkOJudz6PmPlPOske6/O2zGPy3ZPpTIgtftdhp9G800XyToPX\n+4aPfgHnvsrfofWvi4EtqzwewH75+l3M7V/C3sLasvaCYPM2bBDBQgcyec0ZLUr31KmcvxN8HBik\n9Gf6GV01mvpQPYm8GF9dQ0IcDuTucQOPgzGG8v1DYYquuuoq+++hLgv3GDrQsfr7y4udHswdmM+L\nn2Fy+70tk+fcnXfc0e7ox+EAU0eHmAuGKXLtHCtf3g+HWmdMimj/lcLZ/0omxbKsxSVQpA35GZoH\nZYJlWSHLsk6xLGvVv7odv/zlL1mwYIHnAQdEbREXSAn5hO/VDVLqe5LsrS7tAP1OdA/FykyKTOhW\nCaToukjRfnjgVL7esOKg7T7rLKiZPp7M+KNAUYiNnUr13o1MD16MTwmgWUWmL7yTD9xxMmd+73aa\n93bSO26GB6RomhCzTSlpgw1DuC1k+3qLe5hQMwFwNDluCwQs8aBbFlx4IeZdv/a8v1tkrKerSwwq\n6X6QA0xOVsUiNCz4A7EA/PljXyQUrALggQ1/5pwHzuHF3S96JohYCN5/pcEnPlC5bybWTqTZxTCZ\n/QAAIABJREFUnEFQqSJjeN09MoQ4aNUR9Vfz5+NgWjd80bie8w87n0XXipCqmNnhyfb64Mr53PXG\nD/n9KhHQ5leDBBWHedFL7p5kLsOKvudpK65hMNdPMu/dLhYPkXo6UJh3JSal0vsSpGQKGcK+MA3+\nkcTMTgpWBh8hfEPqb25OLbY1KW431VCQ0h7by5Qe2DzCeU2ClIwVs0Peu1PdZZPpgUBKLBsjrAiQ\nElJqSZsDZQtDayu0R8WKE+p5E2VvGzOf+i4nLPg2E9f8jYjSwPc3TkDPmawcDbVXf8IWtHL//YzZ\n+KzneIbheHWHAymFzl6xTT7tNLqSYjW+Z/U9bO/fzjM7BBDpHXc8dR0byabyKCiElVr2ZZyc+u5+\nkIB5KEgJrnudbfUnce+9Ts3AVD5VMZOvTL5oIho6dOfsdjkeyNwAbfPUiZy9B6pNpyDhcO6ekVUj\nOWHUCbZ+TT6vnYU3iRmdB3T3SPBfKBw8l8rBAMJw1pUS98nCtM8l7UDunieegL//vXIbhgMG8nhv\nxQXT3X1g5kVer8xVBTBoOD7d4c41dAM4nM2ZA4+7NOWWBXMeODSf4VVXXcWCBQv45S8PzBS+Ffuv\nBCn/9TZ9uuAlS4PvkcsF02KDFAvqe9O01QiQcuWVMGOWGH3WMCBF11RUNApmZSYlZnRQp4/EpwQP\nuZm2CHXcNLRNJVBlmkQvPosTn/wa2Wgj4RUvkzjvUrae+gmsTZuwcnlUVUyWV1xh634F3a/4bXdU\nr7Gb8TZIKX+M9EABw4DM3Pnw7LP03PUw27Y578tkl8Wik0DOssoH0gudDzHihUd4dCrovgYiAbGT\n29Yhdqmtg62Yptc/u2o0LBsPz3ygfPvy9VO+i6IohLQq278uzyUnr2qtiXGRw1l0GMSDGteuGY2q\nqIyrFtebMWOenUVQEcBpa6+IpvIpQfyKw6TIIo49KWcb9tOOC3i13xuKKHfifek+Zt47E+V2hTc6\nyyndt8qkuF8f6u5JF9KE9BB+NUTRylG0cuiK366KLe3+/k/YICVVcMLvh4KUW/5yFXXZISCl5O7J\nmHH7Pq3uWPuWQMpgdpAgLnePNehZnCxL7PYGQirLxsKJT3+fcXNuJx+qZs+xH+DMB/6Ha9YV+eji\nXt44+RIuvhqMUABuvpn0io1Yn/40Jz35Vc85MxmIGftZmZlLoeg01t3n/lUlOuL00212SVpDSFB5\nfWOmoxdzxNevx8JilD6VvnyH3RduECELicrXatYt4bIfziT65ko6J58FOIU7Z/xhBo13NgphuYvZ\nk3XFsqa4T4ei76hkA/lucmYKw4DnD1epysOxbQ77VykwtD/bT0OogRGREXSnvAWKftp3Kl/pHkWh\nYB2YmcKZFyp9RtrbBSlS3Js2Bz3nhEPTpPS4+Hq3u6dSfxyKO6gr2cVf1ztJWubNcxIWVzJ7I+cC\nKQOGkB90F3cwWCEnEThsyKFkx3W721IpURH+/5W9C1Lejk2bBr29NJfm6uNHCjV9tphlfM14vn/s\nLeiZLHtLICUYhEDkwEyKqoKGH8OqLJyNmZ3U6SPRlUOPt5OTfmL8VNi2Da2QpaZ7O9ryZbz4yUdY\neNPTtK3oovv3f6d37HEohQKh1i0eXYhspwQpvcZu1mT+TtLspTEsokAquXssPY1RMDG+8S2KvhAj\n9qykdb0jNpaTgaR1t2x/kYHCfntQJ/JxduZfZfXL1xLt7+WhaWJxivoFSEnERMPiuTiGIXQwbjtl\nzCnMHD2jrF11fiEKrQs0kig6uQYAOpP7UFCoVkdy/ugPc/+0JJ0zrmXSmifAsvBrAiAWyJJKwY78\nMm7oVOg32gDYlxC7GQ3hTpKmldyBkqkB2J1fyV86voqmaLRExCIuM9Y+sO4B1nQKgPXsm0sOmPRN\nvveL137BbYtus/tvKEgZKvALBMTE37ull5AvhI6fInmKVh5dCXi0B7qq02vs4sk996FpkCg49zGV\ndwTMlmWRWicIzTftAhUwrmYcAE8nbyeeFwvD91/5btmkPhSkSGbNtEziubjt7gkrdWVMit1HClx3\nKURi3YxddD8bz/4CL183h76xM/jD32PUJFOsOO+zDIZg+/9+BhYswLxkNobqp6F9PTVdDpIuFmFJ\n+g/8afAq5rX9yfO6tNCapTB6NLnRLWUgRQK4/tHTANi79J8A+PeGKVh5+333IimTl8nrOeqx76EX\nMrx66xNsPeU6+3MdiQ629ws2ZskSEeUi+1MyKWlrwO5Ht+ULRsV5xm0DmQFuam3m4fhnSedzvDoi\nTV8Ijty+wf5upUW5L91HQ7iBlkiLzSzJ+5q2xL1vi7d52uS+j5LlMYzhmZRsVrhd3oq7x53/Q+pm\nMiWQ4gZ47mPt21cZAK1z1ZN1X8f+/eI+VGKJDgRSPjL343z0yY8ecl0wh0nJUKOJ+WzQFHPPt3qO\n4OOvnFqxKKoUGrvbctmjl/H8rufF65bJFX+6lWXpP3u+l8k4JVjk5/6T9i5IeTs2XYT2TSvdN7lz\nMSyDVCFFy6AYxG01wkcPjnDWLAwPUnTFP6y7J20OEtXrOXaqA1KmBS5mgm/4uopycohNngWGwQdG\nr+LiJuHHbJ8ihGSqpqDpCv2jxTWFt6/zgBSJug3DEXb+JXYdRStLUC3ttiuFR/vSjF7xJNHdG3nl\n2j+gWiZLX7icRC6BZTm+XcMQk93vn7yar3SPtAfgV7tH8dO+07hmA+ysV3h9DNSqo6gKhkt9LWaC\nWC6GaQq9g9sm1k0k6CKdrpoqXHYSpDQGWxg0RPSNXBD2p/fRHG1GtXwoCoT1CLtmfoi6/VtgzRrC\nqzahmCJxXToNO/NiF70663Cjfs0PlkrAxaSYpXDXSllXR4UmseOzHUwJvI9YbpBCAVK9TtKIlWsz\nPFNBvlCwciStLntC/NKiL/Gz137mid7JmSlu6gyzLbe4bLKUuSU6F+4npIdQ8WFYBYrk0AngVx0m\n5X9P+yYA31z5yTKQ4mZSckaOo3sgp8Eul3TisDqRtKGtsJrujGDAFJSyiVv+H48LN+DDD4ufeE4I\nmrMDJZCi1rGnsIK/vnmv/V25SwwEYEcDzL/iZhKNE9h0zufJh2tZcOvzjL0Fbv/iN8k0i2d957kz\n4OyziXbv5sX/eYh8IMqk1Y9jmo67IVNaWFf3vWyfy70IhdcshdNPZ/X+NRRM78IvQWcu2kiqZiTP\nPPUdFBOyz4o8kve/cX/Z8aQtXQr3/sGibs8adpxwNa2zLveguNG/GG3/LRlKWUFchlenzVK6gAI8\n/bQjlL3y6Qv53P4D15WSgHt55kEueT1IR66dbQ0wZrBIR1FoOtz3b+1acd8GsgPUBetojjbbzKRs\ntoqY9Nb1rPQAHHd5B7cmZTiQ8thjwu3idrVINmm4tF8/LVU53xvby2ObRGJBCeLcIEWeP5sVfbmi\ngnd9hIsldLdr3TpxrFSq/P2h7ertda57V5dYSGT19EpWKXonb6Wp0kag47fvNcCe5BbU76ks2rnI\ncww5Ruwisbk487bM4zNPfwaATd2b+Nu+X/Jw7DOe7xkGJEyHPlq++m3UD3gH9i5IOYBVFM6CyFIW\nDjO9JAhzVwlOF9KM6BVPw95qp56PoZTcPYUAilK+azwQSAkGhX9Zw8dJxzsr76frHuPrjc4oGjNG\nZOV2HxMgMek4qK6mcdNiIhtexzrqKPLhWue8OhRC1ZgTJhLdsdbTNjdIkXV+fJpO3srgV0VbFFRq\nMjDTFepaIE3L+kWkJk5l+0nXsqcGate+wPO7nmfz5vIJJnap6MxCQUREyRDnS7bCo1MsUGBkdQtT\nJguQIicYyaRkLLFwfqn+ZZZfu4V7L77XA1JkmHZYFdfdGGohNgSk9Oe6aY402xFOlgX7jn4vuVAN\nnHgize+7kP95AwpWVuQrKxV27DGccNagHsQw8DApO5MbuaFTIZYrBylVvlpUFaJKI/3ZPrZvh607\nnWcgYfaUCQ9NE/40eCVf2t8yLMvi88Hm/CIKZFife6pMACo1K9VXVxP2hW1XnnD3BAj7HZBy2eQP\nA3DayPOEu6dYGaRkChmm9MDOERqGi1xrCDdw4aiPkrFiFK0C50VuRcOHaXpnbtnGuXNFgr90Wvz0\nJMT5Qqrj7gH45utO1lR5XRJgLznpDB75wS5yEQH4slaC9hrYP346QVW45+L5BDzwAEtveJA9My6j\nbfrFTFr9GEuWiMRjzz3nLPSpogOCZZ9r+QyhTavg9NNZ1raMsM9hnyK+iO2+A2gfOYZp3fChzbBh\nzz6O6IUvPPsFz3W7LZ+HaH8bgdQAveNmHJAtkLvmZcvE7l8KVmVEYDwuRJNywV2y77nhD1YydzFH\ngO3x9eSqRtKUgnxpbLpT9q9YIe5bKp8i6o8yrmYc8VycPYN77A1ZSBFC+/V9KzyLtnucussQDOcS\nyqWKnH3fNdT++KvUdr7J4CD86U+CxRhODDy3lHmxNdZqv/ZmTjAIbibr5ZfFb8k6yIV9ONeTaTrP\nnIyuPBQm5dVXRW6mfN6J0mmLtXk+I7/z6qtCDzP0/HkrjU8J4Vci5KwUhuqtFfZ6u7cCqbwW2SaZ\nu6glKhK1r+4UIVhF8rarUH7ezaS8ujI9LDP0f0Y4+99iwwpnVRWmTmV6F7x30ntterVoFskVc9T1\nJrF0na6oA1IKZolJyQcq+t6Fu8c3bHSPYRVR0O3kaQBXXBrg9NOdz110kbckhxw8ml8TSaUWLy6l\nK3fCGDXNYXWKxxxLdOe6iu6eYhEUUzApAZ+PgpWld3+IVavErvimFfDKnyEoBX+kqd27nsRhx2EB\nL06Ec3YLl5g79NFW8ZcWzayZpFjSvdRkoCkN65uhWm/gumsDjBvlR0UjXgIYg5kSSDGdhezIhiOJ\n+CP4fPC5+id54KxX+Ml5P+GmE26iyT8egKZwM3FDAiNx7lQhQXWgGstyQp9N3c9rH/4VfOMbpE89\nh1tfg6KRJpdzfMLS3QMib4xp4mFSpMUL5SAl4ouiaRBVGxjM9bF0qcPQgDjH0OfFsmBtdp6n7e7+\nlPettyjUyVG1sWyylCAlp+YI+RwmpWDl8CkBgrqz0x4RbuGU0MfJGmkSCehPib5WUDwgJVvMMqUH\n1GOmMSbsxH9H/VFPf4zRp1MkT7rgddENpy2QuXlCJeFsVG0s+4yttSnl5tiQe5oCzhbZeT6q8RHE\nr/qFNmHMGDaffCkdhU3smvlhGtrX0/WKoCbicQekJAsOSJGLxMiOVajFApmTZvK9Jd/jpNEnsfrT\nq3nk8keI+CMk80n7sxuaFaZ3wSc2hAgjymZIG04H0bBXLF59Yx2QMnkyoHg7amiNqFhWtFUCeTtk\nda/3+EPD/b3HSJS9lqtqKoEUcc8loPBoagopwr4w7z/i/eiqzjPbn3EtqmKV3Bpbb4+xsWO9GzY3\nSPGwB6mMnaK7ZecyjljxMFUP38OlPz4RPecsztsdPbLHwqXUxJJlOv+w8+kz9gCCNZHznOwTNzMH\nXuAxFKRI16p01f1NVF6gt9fr1nab/OyadifUpn2wyx2PYffFxo0iLHtoscsCGXyECShR8laKlOLd\nzTSGG3n5ZScEX7ZBtv+hDSKCb2TVSIpFUWpC2s1dVfbfhoFdJw7E/c96h65t7wpn/5ts+nSuVY7l\n6aufRlVUFBRyRg7DMqjtjmONHoWpYlPAttsn7xsWpKiKbrsx3KbrgklR0dFUjdFVozms7jBamlU7\n+qaSyfNoGvCe94it1rp1KCef5PmM3MnkjjqO6M61KK4EXW4mRYoHdVWnSJb4QJA1a+CTx97E0b0Q\nKsJppfU6ZySo37eB2IRjSVl9vDgRju2C7t0bPdc/dDL68+DHGDDEbDqxNJHvqoOoXldqj8JY3wz2\nFkV45cauzbyZfoVMKUdLSKnxTL4n117KMVWnM6Z6DHdfdDeWKS6oOlBlT+5yIskYSaL+qL07ku3a\ndup1ZL7+PZ4/9bsc3QtHv7mabNapV+S23nRvCaSU5/BOFRJlwuewHkFVIaI2MJDtw7CKvJaZA8Dx\nzbMoWNmKIEVaea4Q8WbaGuCJxJcAeDLxdZI50dahmpVsMUNID6FJTUrJ3ePzOZ0YUWuJlEDU9u0O\nazWqapRHV9HVL5gU/9RjefQ0Z7VQFZVAKSQ7otbaIKM/PShYMzNF3OgmkTTYubOs2xjIiAdBsmDV\nrmRz0myGqJRYcXH69/w97ghhY6ZwM9WoI1EUhUm1k3mz501e2/saX2k/gtt7p9Jx5NkANLQ5YuVU\naaFPGeUgZdzmZymEqvjanoUk80mOH3k8x488niunXknUHyWVT9mf3dikMXEQztshVvSxcRhbNZ4l\nS4YHKY1ta0hXN5OuGekAo5HQU2j1fE7eD2mDmRJIKQEsCSKGghI511z00EV87MmPVTyG26zqwxmR\ndkDK4rYXWNa2zANS0oU0EV+EV16MUKOMYdO+NopFwfYUyFCtjWBXfJMnaWSlLLym6V3Y6x/4laix\nsXYt49fOJ1U7ivY5L+DPJenYeSdzY1+o3IlDLFMU4+D0sadjUMC0DPJ50Y5Jk2B0yYsmXUBynAwF\nJnPnirBi04SUIjZM7nFpmsIl5RbAut+X8+ozmxytzPo3U7z2mvMZw8BT10q2SfZLgXQJpETIWkn6\nMsKHPqVGzO/xXJxt2xwRrDy/vBbpXnpi8xMsf91k+SrvfCaBiGliAzo4MEj5d9i7IOXt2owZaJs2\n4x8Qg1lXdXvCruqOYY0Veeckk5I38mj4yGXV4UEKOoZVvp18YuccMlYM1RJwf+tNW9nw2fI045WO\nCS6Qkk6LGdGVEEpVseufpMdMxh/vw5dzdmZuJqVQGh26WnL3IFwC0xpncHZM1As5pxRanN75Or5c\nioFxxxJq6mDRYZDygfGTH5M0nMR0Q1Xxa3Pz+FaPSCQ2qUQ87KqDiO7Uhzks7AhiX+9cxo86z6TP\nECcOqtWe/h2afl4uCNFAmLyVxjAsCgWh3+jO7aUqUGW7e9yT5KZNsP/w01k5WuXCxc9RLHrV9dKe\nuuopAVLUciYlU0wTHPK6qirC3aM2MpDrpbvoLO5V/hoKVqZscXH3VyZX8LAZz7cJcebGxCue7yzY\nLrZ3ZUyKkSOgB9BKVFbeSqMpfo/w9sUXFSJKPYO5fpom9PBs8scoKDRFmuxzz5kDL/y1laY0FI48\nAsOA90e/jY8gpokd7RRR6wmVwMZgZpDHE7fyha4ot3U3szr7uF3Pym2rNonnJawIoBpVR5R9xt4l\nugSh3YbTlzL6oVYTq9Dk2mPY0reFU+8/1Xb7dfpj5IPVVA04zFi2xMC4XVzZvMH23j9w5MJf8+Zp\nn2LxTgGYbzvVKdom3T3y2VvTJBYAxYJ8sIpziieiFWvYskXskkGQnePGibJHAI1tb9A7dgYoiid/\nUVfRJeLAiVKRJjP65jXRb24Q4S4H0No1iGXBMzue4cH1D9qvt7XBmq1D67dCoH4aTSmRbdq0TD78\nj/M4/c+n24unZVmk8imCWoS2NqjTxrJnYK9H2D4pNIPuXDudqXY71UGlrK0ed49pUv/kfeLv3/yG\nCevmsefYD5A9fCqGqtG+43ZeSt9N3soMq0npTHSye2C3LU6tDwk3oHTd6rro26Gh35WqshuGYNlW\nrrR4sfchbtozkjeyT3rmHfn5pIvkcoNRCVLWbuulWm0moERI5LzummXL4LK5l3FX/wWkzP4ykFIk\njY8QfjVC3krZ+qnvTX2M8TXjGcx6wevQaCiZ1wlg0br1FKwMKo6f9qsLv0HRLGIY0FvcRaNfaAny\nVrpsczRvy7xDFv6+VXsXpLxd+/CHxZNWqvSkq7od6RDtGkAZAlJyRs7OPDo8SNHs3AbSCkaBWxeL\nlM5qKW16xB+xc7McyDwg5fjjRdxvIGALf+Vn5IK1bUBM/gFXhWYvkyKe7vZEGxamzQpkMxYN3Xsw\ngXMlSNkgVpu+0dNJmN30ROHFSy7l88stujfMo77eSfhmmsCicu554gDE/dAX9oKUKq2c7n8w9ikU\nFEJKdZm7aqhvW9ch7ItgYZHOZykU4PbeqexIryLqi3rcPdLyeUBRuO/EMNO2bycY7yZmlpeKOn7k\n8R5NSlh3QEmyEPOEJgPkTcGU1GljKVpFthcEuLj1uB/iV4MV2Rr3RJzIpe16NAC/XisSl7WnvQtZ\nR0kI6WZSTMvEXGjiU31oCFQi7msAXYfp1WcRVuro7QWfEiJnZHmo/2baiyKUNuqPki4K/3Tdvo18\n/K6PMxCEwiknYBgwu+p2fjMyU+oPcd1hpcHOdzKQHeSF1K/sNg4Y7WVRSQDbOjvQ8NFU1UBVFWXh\n0eCiwF0h/O7Q+AGzXRThLAGdKl+dvZhL+9+eCSTrx1ETa7PHjgSi8UJpTHz2s/g/fwYzH/4MBdXi\njYu+QcYa5P3jrqY56pQTG+rueb2hG0NVaD/6vdziD9Mcy5EthcDLhezoo+GCCxzGo3HvGvrGOQVO\npX5MskLS5OIkTV6XdPe4F8dYzlm4epIDHveCtGefhTs33+R57bi6MwiMmESkANW+AZIuIWU6Ix6q\nInkMy7ATvtWoI+nN7KdYhKOnied4bPgoAN77zNgyJmUoSJH3dOT2xQT37YQLL4Q5c6ju3U3rsR+g\noAXZ2zyCGaVh6BaPDrVRF4xi0q8n2UxKXUg8BwWyNpPiBin5vNCsnPAPhY89+THSOee5knqVVdlH\nuWvvtQBsyD5NoeDd1A0192vy+SpYGVtXki56Qcr27WLTtjm3kFu7GljbsdHTT2lrgJBaTUCJkDNT\n+KrEczDQWUNUr2EgUxmkyN+D2UGOahT3I6PtJ2+lqdfG25//9dof8UrrKxgG9Bi7mBgRG1EJUnYN\n7EK5XWH+lvlc9uhlfPPFb1bs+3dq74KUt2uNjaIE6m9+A/m8ACkloVyodxBl9BgUFAekFHP4tQOD\nFE3RMYcwKe6wRsmkHKpJ8KGqiBF4xhkwc6Yn05c7Zr7bFIu/P+ZMQHLQtbVRVpRNJvwqtu8nkE3y\nzyNgVgeMKzaSX7+UdHUziXCz7VbZNft/2V0LZ/7uV+iqWUoMVjpPrc7xwcs9x580UIoSUSCiOz7S\nqFY5lexofRqa4nWnuc8BLpCii4k0lU9TKDh0pptJqSQOW3yEAEt1Gx9mcfp3Ze9X+cX3dcS9DutR\nPn+4SGQXK/bhV8LU+Rx3RW9OLDjNflE4r70g4hsvqr+ZYiZEwcqWtWMg5+QsSOczHpAyOjIBgBf3\n/83zncGcmMDksfald/F4/FaoEbWnJJMCou26Dt8//AV+3iyeBV3xCwCgOAgp7AvbLo2z77+WoqYx\n69PgGzO+TGjoQ4CUarXFZlK6Uh2eNlY3Jsuu1bIs5sZvwqBAOKSi6zBaFyC7MTDS/pxNgZuVfSeD\nRjt12hhbQB1UIxVT+ifrxxHqabMTuxasLONCR5MxkyxpXQKPPspRryznyk1w/3mHc128gW35xUQD\nYc9xov4oqUKKQkGEqnerHSx874d546Jv0FTTTNNA2k4mmE5j5yYC8Xco3kVksEMwKTiv67oQxEZ9\nVfy4STA+Q5kUmck4VRxEUbxMijsMPm0O0DNwaLtfBZVCnRBYHtXQ7dHBPL5VaBskoJNA3K+EyRlZ\nDAMUv3hvbHiyc0zFu4kYylbI16Mv3URXc7NQM2sa+WA1HUeejWHAjlEjOM4FUiqKkI08pRQ79m6/\nNiiewYKVIZerzKQ8lfgOAA+uf5Crn5pN0crj9zsJ/qSbRxynVFy0NIwqpZ53t03Oq3ky+JWQyIBd\n9GqLGoZMcwu3vGL3jWmZtOc3MVKfQkARTIqvKoaCQjFdRS5RY7tJQWxs3HofECBl9mRRJj2JSOQY\nVRs5PfRJ+3uifIVFr7GLSVERSr84/TuKRdjSKzLOPfGmUPV2JL3j+V9l74KUA9iw0T3Sbr5ZVLd6\n7DE0VbMnvWDvIEpLC37N72VSKoAU+bCqKihoZZoUD0jhrYEUCUBsIHLvvfDgg57PuNuSiQ7PpGzZ\nAiZDQYpgUqytQmj4+xNAs+BjPccwaW+c7lFHkcuJyAqAgD6KGy6BsZs3ctSTP/ZUSjZPNBnnm+k5\n/qQB2F1aLCI+N5MiRm+12uL5/GH+08quaai7xzAkk1LKt5JLed6P+gUYcmtS5P8AA9URdoxqomaj\ntx+dzyklTYtCnTqWLx/7Mxp9glWLF/rwKxEeO2MHv2juK/WJoA6a/RMA6ChuREFh++YQmUSIPOWL\nSFvCiSaK5+LESrTulVOvpD25h/bCetb3C2X/w9NijNAOsxM8yf6+5eVP8GL6LjhJlHVQcSgMA1Fb\nKJdV7Rw4miKe5Zqgcx/CvjDpQppiwaK2ayuvnnoBu+rh2adD7N7tpcp9pSrbddoYm0nZmdjkua6B\nVKJMYCijvMAReWuKzjnhL1Dtc0K17YXOBVLaC+vtyJcBo516fYz9no+wJ8eLtGT9WCL9e6muBsMq\nEE6nWHhfmnGD0NG6EQYGeOpzn+KKD8EtxzoJM4ayOxGfw6RIYfWOS/9E1+Gnc/Ux76FhMEm6pHPJ\nZr2bBVWFpl3LAUQm6JLJqMC0OUDArLejyzqLm+1IFXBASlFNoWmOtkBRvFWKU2a/1/Xqyn8RVRs5\nIegOGFDI1whwrfb1kHWBlN09AmjLqB/JpPiUINliRoR0l57jSVWi6GF9oMl29wzHpBgGVPXs5JT1\nm/n5tC5oboabb+bNM2/A1P0YBmwfVcf0LtAMwSyMKPcEijwgJRneijcy+DW/Pf4L1vBMiltX9tLe\nhWzLv4zf74QYR11z0orsw+zMv+q4USvUAKq06ZEZngNKhMwQJmWo6ypZcgcZBvQbrWTNJKP16XZ0\nT8oYJKRWoyoq9epYT7SQm5lyMylNkSYaQg3E6SRvpfErIQ4f51xXa6yV7nQ3eSvN+LAoDbIq+ygd\n8U67ntm+uGBpTct8N7rnP23DRvdIO+YYUeV03jxbkxLJgZ7KwFCQUix397z//XDZZdiJ43WUAAAg\nAElEQVSvaehlQCDhSp2uHACkVFWVvyYBkD0Bjh0r1GEuk21paoJstMSkxB0mxe06GQ6kqDu2YSoq\nz0+CXbXw0cV7mdkJu0c1k8uJ6ANN0fCZNbw8EV669FKOfPjbNG1fJoqNWUVMDKrVFm6se8o+/uGD\nGoURosZklRuk6KKdMveMtEm+U8raPNTdI0WxEb9YNFP5NFvSy+z3o74qu1/kYHbvcv1qiBWTx3DM\nVpEzxW1rPi0El4YhPn/nqDbeP+ZaO+dIvNBHgAj1kSoiaj031T3No5eJKJ2QHiGiV9Nd3I5fCaMq\nKjrC3VM2WbkiTb6y5QyyxSwKCrNGzmLr4HqPi6g6UE2DNoHYECbF3fihTErGHETXvfU6dPxYWFQF\nHHeVBClGRxd6IUt3vUCUxUwIw3B2lcUijLFOBWCkPkVcHzqdmT2e69Ii5WJNGfr4waqf2hlvAXQl\nYEfMua8rb+S57dTbuOf99zBg7rVFpYPGPhr9DkjRrXBlJqVuHNH+NkI1ST63388pOwc4alsr5+3R\nUXcKxmr3mCb+dgwUXcDCnfwO8Ahn5eItP1NoHkP9QIyClSdviUgxN0hRFDhq6R/pHTuDRKMzXmXt\npLQ1QLWvDp8SRCfA3xNf5Vf97wUE85TIx6nWmsiaKVTVyY+iaWJnLC1tDdgbCIB4yunPnJmiRnOY\nKgWVfI1AAHpvn4dJ2d0h7psElH4kkxIiawiEVKCk1wtUcV7dJxkVmlDGpAwdp6ZhceqDn6CzCn57\nYinc/ac/5fXLRc6TYhG2jKwmVITJfZAaxt0jF1EVnd6YEIqHdDEmC5Zw9/p84kcyIPm8cAVprkSV\nOgGqq125VIZEVe0rbLCjgebPL2/HUBCWUDpYkr6HXmMXfiXCwoHfkjCcubesnIHpgJQOUxTkGu2b\nZkf3ZMwYQUVQRiP0w9k16GxmCgUvk1I0iyTyCWqDtdQEa8hZCaEzVMJcf4pTU2QgM8D+lNBzjQ0e\nbb/el+m31zYZ1m2YxrvRPf+VduyxsHWr7e5plmC4pQVd1e18A3csu4PubGmwlHp99GinXphYCMuZ\nFPdEqg0DUi67zAE7brOTKB3gLtsJ32Jg+gIUQlUE4t4qWhMnit+y1oU06e7Rd24l0TiRvA6fnA3j\n2roYG4fNLWKVypoJwnoVFAWoefLic2mtMtFeuV0MmBLY8CkBO6OuasLEQZVsqbBfMe3sasaWeNC8\n5d15SFHkgdw9UoAod1J96X5+2OnEcdcFxe7czaRYllOkL6BEWD19BA3JHMe6tIU6fqaVAJVM869p\npRBFxHUnioJJkRPZtOD7mVg/zm5zvU/Uz5E7OL8i3D0HAimxYjeZYoagHuTE0SeSNTLsM5wdvs8n\nomJk8cRcscBj8ZvZNeiE0fhUH6rlMCkZK4au41Hwyxw5Mnrmlml32CDF3LUHgPgoMUHK58IdvllT\nnMyvxrdxZvgzKIpCRK2jM7Pbc10Zc3iQMjVwgSdcXoAUZxZ3u3tGRkfaqfjPPi/POeeapP17qPM5\nzFtQjdiZR6+uFm67em0cyfpxBFN9tPaKMIuzxKVxXI8P/24RddZWX16awkfIk8pdCmeFK7GVoBZC\nVcSDaYwaSyiTJpoTxSqHMilV3TsZt/GfbDr7JlAUzziWFdGjmnhOZcSTtAFzLwWzQLN+BFlDgBS5\nqy8W4eU9L9MSHIeGj7Q5YLucANo6cliWxTbzWQpkaNaP5Pymj3Fk4Ey+OPluCjVic6D0Ou6exnAj\nnaXkbtLdo5dYs7A/SM4QSFcC52ggjGYFyZlZuovbWdG3aNgijuEnHmD81iV85mJI+x2wIc0wYFOL\nuBfH7Ye02V+RrZBJFE2KpM1+fEoIvbTBKlgZ8nnoyL9JURXuxmJR9FnOTHJRi+P6MH0Jj2YqY8Q9\nkWZ+JWw/85VsKEjpQmxq0tYgPiWAhcn9sY/anxka9bWvO8WCBeIZ6zV3EFKj1KqjqA5F0EJJgtUJ\ngqVcNCP1o+lO7+efiR/a/elmUqRuqS5UR1APUiRLnjRBLcSZ48+kVhMi2cHsIF1pwZSN8I9nUkAk\nD+3P9NllYGT9qH9XJtp3Qco7tcmTYft2dEsllU/ZqfJpaWEgO8B3F3/XLgkvJ9VKCnSbSRmiSXFT\n0sO5e0aM8CZEknYoNRrkZ848U/zOVTV6NCngPNynhq7zvC4X3+LmbcSaha/5pUnw+E9+zbMza1l4\nuKAfMkaSsB7FKOhoisaA0c/eGkgPbhLFxshBT0kLUdJyjI6DZhQYbBCL+Ig6h04/9xQBUlRF40fT\n59k7VDlAhzIpQ0GKqjogZW/cm0CpJuAkuXODFGnVwSr2nRgh6Vf4fNvp/HDELg73n8Hn6518EFLT\nIiOEJOPUltvIzvyrnonMzXaFDLGwSpDiU0IVmZREwbuY/2bFbwjqQQ6vF7lJOopCYNeoTUTXRW6Q\nVEEsRq3xnbyQ+jWd0n/cI7LkupmUsyOfLwO28v1kPkmjNoHrj/iqDVKsPWInNTjGh4pm3w95nXKy\nnTZuLFpJ/B1W6unIeZNapIoxT44YcPIzVKlNXiaFAHkXkyLZq7yRx6f5ROZfoKEpzxeWX0xHqt2j\nZfKrrrT/SpALo99AQSFZAo3BXgFIzi7hqGO6IdLaAU1N9OgOOJLRED4lzP33w4IF4nUpnM3lDf6Z\n/AFZw6GldutijI+JQ9LsK2NSJi78PblwHTtOEAyu2yWsaaLYaEQRi+MI7TD7ewUrx9e7hfCxRTua\nTDGFUtIQySiavfG9jI8cRVitI2X2M5hJuL6fZe7Gufy860JAhPN/a9pf+ErTYsaHp4DfTzyio/b2\nkCsl++pN97I2N4/J02L2pkEzwuJ51oXYGhwAE/GH0AiRMzLc33MDn3vtffTk2u17CIKBM4sm9b//\nIa9Om8KiUsoddyI2EOO6L1RkX22QWft1UpZTKmHBApG1F0pRLKUpbdDcB4UQu7c7JS4APrl2Cre9\ndq3djkJBMEMhrYrvnCHE6PFc3HOfMmaCgFLF6PBE+xqHK+wJIlFcZ6fT9qqQm30Tk1bMcJiufF4A\nn4jSQI3WQqqQYv/+EoiyEoS1ahRFIaxFMLQUmWLaHnuz6s8FYH7ymyTNPp56yvIwVjKypzZYS1AP\nUjCzFKwMY0eK7/9k7Bam1M5iIDtAV6YDBZVqrYkvtZSKZqZ7bX1PMp/EX4TbfrEcfv7z4dP+vk17\nF6S8UzvySMjlGBuHf2z/By0SpDQ7Sv+htHIltC+jewyGZ1LeriblQCZ3BnJByURH4It5mRT5cI/Q\nD/O8LnfM0Y6txJqPtF9PTDyCe778HnaHBMLOmgnbjRLUgyQLcfpCUJ3OOUzKc2Ix95WYFBl+3N8o\nEP3oZgeFtVSLHZ2m6JxU8wGaNXFumZH0QJoU6e6R/tSh9VYCWsg+hjvSQtq4liiDVorVozQmd8Zp\n1CdyW8MSjgqc49kRygXFNJ1+ApgSeJ/nvsi/VRWadQH0ZKSNTwmRs5I8Evs8P3rlR/Z3UsW4J9/K\n9v7thHwhWqItBNQg+woiZOMrDcvQNOGiypaiGuJ5r+Kf56B2RzvvfehusOCCyNeIqg2e1N7gMCnx\nfByfGsAwBFuQLqRRWveQC9XQE0oSVRttxkACZ0lbh11zcot+JH2FfZ4K2ov3LeSnfaexO+9kUe4o\nbiKoVBNVRxzU3aMoFgWjgF/z2yAlb+TtasRhHA1LMu0ADV3xE1SiZM0EqXqhHwp276MpCVN6ob+5\niSO6MnSuXUrXyCr+uvd79ndHaGIF1S1xj7tL+bCkuyeWdsbvhRfCpZfCz5+ZA0iQ0mszbwBYFuNe\nnMPWU6/HKGX9dSca0zTBzNRY42lqgrOnTrWPn3O5blq0KZiYmKroI1nQM55LENKqqPbVYwUH2LXP\n+c5Le5+12SWAoBLF73f0DIoCyZowvr4Bm0kZXyNAUVLZbwMRzRRsoV8L0l/opLu4A0MpMSnBEJoZ\nJGdk2ZsXjN9gQSAIOX78fqhe/RL+1u08dvoUJlRNpk4dw8Nr5nnWP8MQQGLl4S18arXB1OQme27d\nv1+UCQD48dIfQynJblvhDRq0CRQz0t2TsaOstg5usI/77a0X0WVsJahGOcP6BgoqWTPhmVtShQRB\npYoHzhbJUDJW7IBMyuCgk9HWMMDvYmXkOHC7ak3TomBluLT6RzT7J9kgsFCAAilCmpjDgppg7dLF\nNC31pcgqnyPOyVtprtmu8qf2W+xzDwUpeTOH4k8xokZ8v8pfxajQYQxkB+jO7KNGbUGxNAJmLQoq\nfZleOroFwDMsg++/CCeu2g9f/rJ40G++efiOeIv2Lkh5pzZZLCxjOsUD1JIES9eh3pkQ3emx4QAg\npUJ0jxukKNYhoA6XHQiknHGG0LHICVDuADJhF5Mydy6kUsNmAvUpQdRCjuqeXQy6QIpP02mKNJG0\nxIydNVNUBcUsGdCCJAqD9IWhJp0X+VesLFwkFh7dBVIsRWF/nQA3VUFnoZfKfBWdfB4MxFY9VPLH\nDq095G6/ZFLqQuIYbr8tQO9+Rzc0bhycdZb3mmuCVQxmB9lTU6Rp0Htf3Vld3UxKjea4GS6Ofqes\nbpNs52S/ONl+Q6jmm/TDiZtdvJT6Dd948Rv2Z5OFOGG1hklRZ4EK6SEURWFK7Sy25F4GBLOkaRBQ\ng/ZOPjEUpFwEo59ezMmLnuC8XY5uYmiUhGRSEjmRkK5YdDQpStseEg0TSJg9nhwmEqRIbUt1SVZ0\n9tnQoh9tt/EHI3ZyWvWVzvWZDkhuza9igu8EVEUd4u7xlzMpmoGFhU/1eUCKNMUVHbdLogkEKAwo\nUSE+rB2NpSiEuzpsV8/aC97P2LiIXFtoebPNSXeLZnmpzKg/SjKfZOnrzvgdO1Zov778td8DMDGh\n83Tiu7yRfdIZq62tBBJ9dBx1LgUry4rMIx4mBcVgwGinQRtPQwO0RB13g0xlXuNrtENJC4p4RuW9\niGfjhNRqqrR6+nP7Pc/DbcuuJ6A5q2yRPD6fU51cVSFbV0VwIEHWSuJT/cz70D8AyNBna1KUonB7\nyHIh9w1eTbEEUqoCYVQrSN7M2hlos6bIGu0GKSPn30P2sCmsmhihPtDIaN903tixz1PnxzDEAvzr\n2aewrSnIF3/6V4JdreyN7bUBk6yfxEXiV0dxI2N9M+z7lbcydnG+Kr94QPtSMVbHBbANKBGiUZHa\nIGPFvYEGZoKgWs2E+jE0aBNIm4MVQ+grvWYYoAbconil1B7neSmSx8LCT4iQHrHfy+UgR5KgJubU\nkB4hlU+RLqSprw4zc6Z3/EoQtiQhcs2YZiWQkiVVjNlzq65DVKtjIDPA0+1zaNAmiJxWpkZErWcg\n28e6zQKkvGc3fPlVuP9Dk8k/+Kigi4ZbNN6GvQtS3qmNHw9+PyM7xYPQkgSjqdGznZcum+umfA44\nMJNiHkCTcuz0t8ekVGLfjj4a3HpgG6RUlZiU9nbxgb//3Z4kL7rIe4zDtq3hIz+cjmIZdE862V7g\n/JqP5kgzCbOLAaOdv7f+Eb1UOsCnBkgUB+gPQU0qj2HA4/EvQa0AKXIxnDgIhabRNIVEdMOp405x\n+kpRqQvWcXXzD0mnsauyhlSRI+Vg7h7BpERQ0dg5IEDKadWiM3q3ikRy7nBQt0X9UVpjreythhGD\n3tDP1aL0hSjG6NKkVGtNNgsWVRsqgkdVhVnBj3BZ1Y+5vkZEDo0fEu0kLVVMEFKrsVw+YJkOflbj\n2fZrPiUkQIrm0O4y8sO2WqjaJPrg5uUQVgW4PsxLmuFTSyAln8CnBjwgRdu7h2TDBOLFHqpcPnqb\nnSvNxdEofPKTMGGCs7j7lTAj9EkcET654rXGzE7qNeGCGeruKQwBKajiOXAzKdJvDpAvuV39fpgV\n/LD9uq740fCJDKS6n3TNSMLd+zlrj6jmvG+G8MNP6YUdpb3Hj6c9zcza8zkzLIqx7U97i8NV+auI\n5+Ke6CRpLeOOIFPTzIn5w9lZeJV7Bj5Ilhitg622+Klv9HSeSnyX+wavpqeUlE5VIW0kMCkSVRsJ\nBBwBODh5UW6onk9QFbvsT20XrKMNUnIJgmoVUb2OZYmHmRP7uKdtimvw/H/snXecHVXd/98zc/u9\n23vNpvdCSChJCCR0pBmQqhQRVERFEeTx+SGKPnYFfeQRG/r4iIAUaSJNQERRAqGEkIQU0ttme7t9\nfn+ce6bduXfvbnYTxP28Xnnt5u7cmTNnzpzzOZ9vG+890pZxVVEgUVlKUVc/Mb2XsCdCkUeY0HrT\nbSSQ5h6hpEi1RcNnOM5GAkF8BImmBkySovcIs2/mPS0d2E3V3x/i3TMv4G/9/0t/qtcgCQMDsCX+\nCgPpbsMpOR0o5erL54KeZtIzd9B8WzM/6Vhhe/6TJ0w2n41aZSzwcb2fzoyJRZKUf+78h3Hs/thO\nkV5KKWLyLLu5J5ruJqAUEQyKDdKA3uVq7rGSFCP/Thy8IXNsmkqK+ZlUVbxKkKAWNsZSPC7uO5i5\nh5BHhLv3xUVJAll3TKI9Jcxk4cy7bVNS+nW+9YPVLHv5HXqTHZQExEbP4xHZoV/b8xp7o9voSO8w\nFLWwUsHG7jX0xLpR0/Djx+FvzfD92dX8uv88UWnxxz/O7ohhYoyk5MGgIcggZs1Jk5jYKhaMmj7Q\nLaYegCk/FmrLxTNFUrbcJCU7usdKUpobRo6kOCGJSDRSiber1SytumMHS5fCggWigKHR3jSc8evP\n4K0t58EvvUpb0zzDVBPwBKgOV9Od2sdjPV8FYFu3MO771QB9yU7aglDanySZhNdjfwAE2bCae+KN\nE2j0zuH22hhHN5mp/AHav9jOadWfJBaDck1I9F4lkGWecYvuEaYYhaBSyputqwgqJXx+3F3cXhuj\nwiN2oLmcjot8Rezp3cP2Eijp6kBJmQxIprCW8r1UUnTdVChCapmrkiKUNJVTIjeyKCJs403JBqZ7\nlmUd25foJqQVo1vKF8jkflICDnpChvrg14K5lRQdIms20F0zidM3wOk9x4m+dOz+JHnsjnXjU0UR\nxZA3RF+iD8/OrfSUN/NG20vUekxFTfaBdMD1ek0zmOwPaQor9VjfGfO+etL7DOJjJSlBtZg0aWOn\nnE6DniEpVp8UmRekPFjOAr8goqEQ1Hmnszz0WePeNMWLTpq0nqK3rJmyd7eyYi08PQEGxk0mnRlY\nkqTMKjuC7855kiODH2Z56LOcUH6Frb9KAiWk9JRNFZJQFOgvb6TRwhc/t34eLT9sgTffJB4po7+0\n3ghd9ngU43vxtLlweb3C5CYhk5n5lLDhnyXh9af4Qdty1ratIaAU2cK3rejOOPqXqvWUavW2CC1F\nAb2qitKeJH3pdkKeCBFVkJSeZBtJpV9Ut44H8PshmilV4VfCxNMDaIpGyO/FqwQYSJmRMVFdhJ4/\n+6z4f+mDd5JWvdw1Q4zldR1vElCLieqCJHyzbSE/7ThXKCn04VNCJCJVPH9YLc0v3QM6rI2JCsCS\npHz+6M8b1/MpIfxqUJQy0XstJEWotuva1hrHTg4fIca6Vpxl7onqPSZJUUv4S///8PlXP0Bn1eNM\nXbDL2DxZiYuimE6s0gT27AffMUhK3GLukSTFpwQJqkVGpXeDpHgy77oWFnlTuncQ8oYMp/87Fwnn\n72f6RKRNSCkXm7ZUmnN+L3JSFT31PAvfaOUrd73KA3e207xX9JdwuDdLmV9R+jvDbBtRK3lqz13c\n3/MFLn4TZrXCdSdBPz28/PJYCPJBxaAhyBJTpjA1Y8qt7QWlRsj7p0461XaYNHnkNvfkj+6R8mmh\nCGYsJPkqqFqvDxCNVOHt3G9W6topdhLz59uPP3YLRLr20vnlW2nL5HKQIYBNkQnCGUuPGRFBcqfv\n0wL0pYS5pziaIhVPcUTgYgAm+5bazD2JRuGQJv0h3Noci8HHyu7h2zOfst2HRK7oHlEvp4zeRA/1\nnpl4vYrtOvmUFIBtJaDqaUJdZvZP6R4gfVLa20U6/XQapvtPMO7FTf617tCCQVBSSc778lSue702\n69i+pJDsrdFWMqTSmLgyyeoESTGjLHoSXQQs+R/qesC7v503z7mFRGklc/7yMyIRsoiUJCmv7HqF\njQP/NJSUdDqNd+cW2ioq2dW/jen+E43veDzi+s46KKpq5hWRP51RKhK5SEqRKkjNvj5htunthb6o\nmEWt5h7pY/Gbs3+DjyKjfwE0xWv8lPeXIkFveRNT/voakTh8ZzGEwtXsrxZOzZKkVIUqhY+I4uGC\n0tso0pts7S72C5Igs8O+/DHTz0ZVob+skdpO0xS1L75F/LJ6Nb3j54CiGPK+qqWN78Utu2trvh8w\nqx77lJARnQHCoTapdbE+/py4x7ROyEJumjxmLpY39gg/kRsz1dWtSoqqglZTS3UftKXeJeSJoKeE\nqaw32UGcPgJaiERCwecDPbPh8ipBoql+gt4gPp9iRNZISCUFQEmnmPrCz9h4xIXsU8yXL6AUE013\n09svFv7Nib8bSopPCeNVAjxxeCVF+7dw1A6RcwrMmmkyDb7sHwVVJJvT+3hx4OeAGb3YG+ulRKvm\nJ7VJlpZdSDotSHF3rDuLpPiVInw+s1TBczse5ytvn8txjzbwhPYpcT3L9KWqZp4VSUImlk00siMn\nLHmRjJBuJUyRp5w+XYxlkdbBVFI8mnhZ1+5fS8gTMpz+KzxCgVwbf9q4b68XOhKWsMTHnubtpnI+\netlUxrfrXHjRN+Hii2l5+ucUpcQ8cWHTF5mZmM7sO67GO9CFknkuviTc8hw8MB1WNiLqqc2G2257\nhEsvHQtBfm9h6lSmWEiKVicmiPNmnmc7rDgPSVEUoaToTiUlOXySUlwsUrk4CYYbTJJSiaenA9Zm\ndhM7drgef+Fb0F05nvRCU+EYHxCTXdgXMipDS0dgn8cn8iLgpz8tlBRVh3s2fZw0SZrfmIimeIzo\nngkdkGyaQD7s2ZMpaa9WcFT1ibb7kHD6pEgHQEXByHxa55mRtSi7KSnnnQcVIbFz3J7ZqEY6RBRI\nSYm9SqmmmZOTrsNHS/+PW6rEROZWfVYuvkuWiO9Vb3mZcNduFnTWMtW3nJaSFuPYvmS2khLwBGhr\nA38mR0XA4gAc0ILE0xlzT6KLkFZifK84s3vdO+Fo2lZcxcyXf8Upi7qz+sNK4BJ6zFBSqvpAjQ7Q\nVin6JayYi4GTpFjPGdDE4ionZ+mwDOaYSehRYnqvUZBQng/MIoN7e8WE+847EE+a5h6vKlZXmbys\nyG8mEpILr3RQVtEMwpLSE0aEz7eWwM4SqA40sKdefCZJis+nGM87ELDnlAHoaxcDRFbrti6S//u/\n36avrIHKzuw8Lbz5Jr3jRWZPo9qwKsiMqprKgC9DUtzMPX4lTIlqkpSe9D50r2l2qvA0U+ozI50m\n+cwQ/IfXP0S1NomyTDi/rQq6Ar66Jqr74NWB+wh5IsRi4noxvY8k/QS1sJEw8cPNNwFQ55vKQHKA\nkFcskvLcFZ5Gwt4If+69lePvFubcxjVPUtS+jbVLP8GWzLv1X8fcioLC3tQ73LdZlCGJ6X10xzuJ\n6/34lTAexc/KCSF6Sqq5cLUZdRVLicH31G+eMu7Rp4Qy6maYgXQnG+LC8VVmfO2L9+FXw6iKZhQ6\nDGnFtuieUEj4egTVYnw+83qAQZBf2CcUYqe5py2zVqSUKH7Nj8+r2iaFP/Z8nW2JVYa5zK9EiGgV\n9KZNkhJN9xL0hGlc8ySnbjP9iKS5J50Gr24vcqoqmshYnBSmHi0F6tNP8tLUZh6ZGGPW1bDh6gtg\n40Zm/PiTXHvLt5myH6r9zcx87sdMeeYnTH/hZ6iZph6/GVq64KvHmtf4ZedF/OHJVjZvZsQwRlJG\nAlOmMK4TAomMklIndl3WREAAxcHcJAUyPil5zD1DJSmqKnLNRbIL8roeCxAtyjg+/lNkLHUjKb4k\nnPM2bFp4IR5LtdxPl/yZr1dtFDK/zAlhKfjm9YKHAP3pDtoyG8A3dv1SFLXLZMbyKH7CMWE2Szbn\nJym2NmXWUCcBkH4hEtJxVuzmRSPqvDMLIimlpVAdFovj9sw6H2nfZlzHGuKnqsKnWubB8SkhI3on\n3/PweMQ9NK55EoBQ9z6m+pYZ42DTJhFhE1SLbD4pIU+YBx6Avdsz4cuZ2bS2FgKeIEk9QTKVoife\nRVAzTQFVnZAqLaGnfBxt51+NOtBP+SO/zvKbsYYon1b9CUNJacm45eyvEOcMWhQRj8dUu8A+WVtJ\nCWCLVpJ2ebmTlCHZ0lQEppKyt8/cFSb1jJJiMfesbRVkW0r5YMnEnCEmaZIGYUmRYM+kJWyf2sD3\nRP45Sv3l7GqezJ4wdITMe5EkxevFFg2l6/DWq3YlxUomotF+BoprKO6xkxR/AnjnHXonzmFv8h1j\nkbKSFBn26aakSHOPXwkbod4AvelWUpo411Wlv2d5yZV8sOkTxt8n+Ex/oK5YF34lu68kSakbPwd/\nCopi4FMiPPmkWOyjaVHgLuQpMsZ/ia+MOs8MdCVBT6yHsDeM1wsTvKJjl5V9lIpAJXtS63hlj/AD\nmfHCHbQ2z6d13AL6051MCh7BNQuuZXdSPMfbNpg1hbb1ryOWUVI8+IkR583Dl3Pxali0PdPXGVKn\nJMyJQZIUvxJhf8rM1dObCdPvzZAUyCSVS0NQFT5G92y5ldvbz2R14iGiejd+pShr7pD1kXokGXAk\n6jNUWv8AQa94jtYcWI/03sQTvd8ynv85Z0Qo0iroS7ej63rG3NNPSA1y7P9ezqIvX8RZA8JMHfQG\nDZKipe11wjbEX+CidxRWRUUa+6N2gNbTycppE+hIbWfABx3XXQ3//Cer7liJNxbjxTuhoT3BjL/8\nhJTmZebzP+aTVUJ5WrgL9gdhtd27gajeTWfSXl/qQDBGUkYCU6eiApPaEXlSaoVEL9UECRmhkouk\naIonKwTZmidlqCRlKLAqKYCoQw6uJOWkTVAehcSHLrK9gGG1jCrPxEwdIvEHa/ODgs0AACAASURB\nVMHEeByS0QD9eiftmTWqYkC8cLPOFSqMTwkxPrPwpcYVTlKsobzO+3JTUlTVlFsrtfFZJhgnSTEy\n82ZISncA9OJiwu3bjetb86TISBRpf5YTmTSlTJ1qBIZlXVfXoWnNEwAEe/bhVyNGhNif/wyd/UJJ\nSVvMPRGvYE3+jMOkR9O46iqoqxPmHoAHHx1gx/4uwqqppFwbhPjsGaR1hXRdA5x7Lvz3f+NR7YNU\nLuiH1x3OZyf+RIQg+8IGSdlbKRZLGWEl+8TN3AOm0iMjkH0WkpLMkBSrTd56PjDDzWVSqkgElAbh\nudxQ1GCQlNv+eRvNJc1MqZhiJFs2SIpcGLSEcX9dqd3cMWktt/7gw0QN85TCC8ev4OhMXq/barps\nJMU55np7zZw90t/BSiY+9amvMlBUQ6S7F2s3z2gF0mlWVvby5dapbE4InwIbSUmafeJcHE1zj1ic\n/nzxXzP9GDXaUe2ZjJ5WqQs30hIR/kMyjFoioJokRd6bJB6+OqGC1PWCL7NTDygR+hO9tKW2U+1v\nsiUz9OAjRZzdvbupjdTi9UJAjfA/E3Zzfs3NTC43fZhSiU6a1zxO4KpLQVEY0LsIqSV4PPCBSHbx\nujt2XklST+BTQkbenKePO5ON5fDcL2NMf+Gnhrnn2huvNb6noBpFL/cnxZZ/QcUy+hIZJSXRZxTE\nlCRFUz08vflpbl37ed6MPcp3tn0wk3gxbPSxE9J07yQpRlbbtMh+6/XCxSV3MN13gnGcjDYDqC4N\nU+SpIE2SqN5NLCbSNszd3E64azfJkgpu/dk2KvrgqKrDaPr0WSx48EukkxoX1H8pq12P9H4ZgNM2\nQKKsik1NzcbmuKFYPN/eyYfxyHX/wBcI8+mvfItQ9x7+csmdFLVv49h33+K65ntYuBNWNgAKjC81\n5+qo3sNd3VdlXXe4GCMpI4Hp00kqcPo7EEhhkhSHkhL2u2Rcs0DFg54nBNlJekYScjKS9XuIx0W1\n5H37svIzz9kLrSE44nK7AjFjhnku2VaZTVbWUNEQmRXbJEnpFztmaSPXFI+RIyU1bgJTppA39wDY\ndyduSgoYxaptPilykQ8qxVle+U6fFHmeqpAZYqs3NhnmHqeSomn2WiDOOkrHHmsPb5YLnscD3u42\nqrauJBYqJdizl4BSRF+ij4GYmPQG9E6CWhGHlR9jfF9WiZaTpmqx5/szafl37IkykO4i7CmhIVJP\nKA7zd0Ny7mzDoZjPfhY2bsT/9+ds/eHJKClnTzvb2KmFvCHGdUEiGKIjJPoy5KKkSMdZ61iRPjPy\ncVmVlYRBUsTP6ZPNfCHyecj2xJLC9BSPw+b+12koamBm9UyDpAB8Y/k3CPvCHH+8iC5yKimoSU4/\nVRz/eO/XeajnS9y68ttMLJtI141dwtGVMFsyqlhQLRaOupln5iSbd99tZj+W/g5WkqIo0F9Ug6qn\nKbeYieZkRKE3quypRtNK3PieoaQgSIoRYgu0pjZRrNYYKkpDidjiJvU4H35KLIB+JWI86/6kUA4q\ntBbb9aw1a6z1lxQFmDaNpN/HM7+BpWtMR92+RB/tye1UB8bZQvA1xWuQlLqiOtPUNlCLpqrMqjIr\nsnv2voKSSlG0VGxYouluAmoxXi+M9x3pqCUE2+NvZdobNvLmvFs0wKIr4MlJMOmfdxnmHr/HT51P\nRPjo6CQS4j5bUyKkvDE8kb5MX/ZbSEprq3if1/a/gBukr9mlpb92/Ttk12WKxcTcMJDsN5SUMq2R\nayue5qf1Keb6z6Qnvc9ImBfxRYx6VwN6F7oOcT3K4pWb6C1r5O2fvEA4pvO3O+HEL/+aomcfYe5T\n38G7eT2Xjhcmt4vfgKVb7O06bQN0H30KAYv5ty4irACpFFDcQsk9fyDQvocd009k41EfZvekJYx/\n9EdM9R/Lwl3wSsaq+OnDv8Cny0TY9kC6m950OyOFMZIyEigv50+T4VoZuZaJ7nGSCi1ffnoGT+bm\nrEI80lBVi7kHxCqq62aaxAxqemFPZh5zvoDyp1wk34iJFJzSf0JmqZXmnooB6EvvJ+QNcfzPzmPS\nP+9iQgf0e4CaGo47Di61R0kaOCzj7ydJh7UNzjaBqDYrzT2KgkEIpfOb2/ecP6VDJACNTXnNPVJJ\n0fVskuKEXPA0DWrefBpF19l8+HkEu/caheT+srKN/ckttKW2Mj44j9tfm8T//VnsYEIZE45UMqz5\nQSQhiKX7mL1pC89++Tm237CHvm+IUO/U4fONfuGoo6CuDt8LwtlOEosKzzj+/tG/86VjvmSoPdIn\npb+0jKgiJO6AxVRg9UmRi5ZEIOPoK8mgzdyDXUkJebOVFEVR8Co+YqkYv/2tGfEgfU+sJOWMqWcY\n17K2Qzr5TgjPNgo9Bi0qU0mghGJ/sZEN2oriYvOZWZPUScgoNRCOvFYVVFFgoFjMETUmx2D2Pki2\nNNPldZIUsdBalRSvEkRR4PjxxxvH7UmupVIT46GyEgJe6QxsjoWAYlb5DmcSLBZr1fxn85OcWf9J\n4xgJp7mHujr++MA32VIK194rlKtxdULp60zuptxbZ0tmqEklpWc3dZE6m5qmKPCROZcY/w/tyqi3\nU4W6YlVSILssh8T4xjCTJ/jZ3LOO33RdQVqFF5uhbNdbRDOkLuAJsKz0cuSZBEkJG+a4xvAEo05a\nX6LPUKNAEPKrGm7Puq5H9fDxs+YCMDdwBouDV2QdY+1Dec/xuNh49cZ7KfIV2frE61EpUqvpTu81\nlJSwL8y8GeKZyFw46cQAC1euZ/Ph55FsnsDRHxMFyv0PPcbeb95Jf0k9U37zn4T9AU59B377B/jD\nPVCbyd1X3w3z9kLH0adRknGy9qo+vJmyF3V1med94oms+vJD/PXiOwBYu/QTlL75As1r11LTBysz\nJGV/XxuNXkE4u9K7DDV0JDBGUkYIdx6GrW4PZCspEk1Nrh+jKh4CwdxKiltBtJGEpkE8WCKS0YG5\n1XeYfGr6YPZsMTlad8dW1cF577quM3++uRjFPdDrhfIB2JfaiLcrxfhVDzDtxV8wvgM2l4HmcfEw\ntaDOrH+WMxrH+v9Vq+zmHiPySI1kSee5lBTr4qc3NxO2KCl6Xz8kEnnNPW5Os1ZoGlSue5GO2mns\nb55PoKeVYoQJ7tQ/17I9Kep9TA8fTe39P+WUN0V0izctSIpMkmZFsTeTy0Lfz+J1u0irGv23fpeL\nVsDyc0BfscJQmFAUWL4czwvCo9brhSOPhNNPh6ObjkZVVCOM8qH7QpRFYSAcJk6fiCywFmTzmOYr\np4nTICmZKUgWYQRTQZGRLL6MucrqOAsi504sGTPMSVG9zwjJtT4nG7HEPMc47+H8tE6nKTLBcLQN\nWMxVsu6WopjRIhJ1dSZJCdrdawxcUPzfop2a3ZbY1bWf/kwStm/6TYfOeXtgYOZU+i3huQAb+lbx\n8d0K26JrDCXFpwTRdaiJ1PDWJ4WisC+5kYpMErcVK8w+GEibYed+JWKQiJ8f9yeuKr0PgIXlJ6H1\nZEoC5CAphulnQgv3zoSynijoOsXBTAmAtHBitZl7FEFSumPdlAZKURTh2yX7dWqlmb+kdO86esJe\n1ivtHHOMaLfMfQRmgdMvVLxAi/cI43tHHx6mstQut66uhkB/B/ou4bjc29HLjNBxANR7ZpFIgGZx\nBi/zVQqTqpK2KSkg1NCjy87m8nmX267hUT00Npov9CWlv+BHJ96BE7IPN8b/xg/e/QjRWBqfT5CU\niC+StdELq8L/JKb34lGEf9XEJrErlFmFF27upbirj00LzkfTYHM5HHkl8MIL9J13OSvP+hpN/3yA\n6bdcwP/9AZ6cCHENfvYooMOpGyClQMfCkwxn94smmFlifT4xvpNJ2HvUWfRUCfK7Zd7ZpIJhDv+/\nLwBCSVlYdRznTr7UMHHuSLyZ1QcHgjGSkgcF5UnJ4LEpsFeO6xw+KQAXXwwnnJD1MQDTp2qEi0wl\nZWP7RhFWlpGKlcFWuAOEyGipkCytFG/WMRlzwk6zsJeCInZ/Ui3KoaQ4711HJxDACDFuLh5HqqSK\nigzv+tsPH0fV09RufJF5ezIkZRDrlrmrNq+dy9wDUFFhN/dIdcevRHKSG8O8kCEZfo9lMmwS5h4t\nPsDku7/Ksguq4aabbDtJwIh2cLYn1z0FuvbSV9ZE80JhEqgeMBe5XYk1ADRu345v7w4qu3qp6hUm\nCBDmsv9oeoInP/yk8Z0yn1gQe1KtzNzZw/qJDfCJj3P3HHhuNUQyEUtGHyxbhvraq/j6OykqEjU0\n681gEREKm6krUjYA/aEQSQYM4mG9l1z32xCcxBGBi/h43S8Au0+Kkf47Q1KkuUrmXZHwKn5Dzu9I\n7eDejT818sU4iYGzXVb4fObx1oyf8tzCbGGyWKnGDEZSZOi005fspps+Sl9EvD+hnkweFV2Y3npm\nTbblEAF4tU0Qxg19Kw1HUK8SJFNn0xiTSWKEVUt9ogxJ2ZQw6yGF/UGDRIwrHs/hwXONPpFRVH7N\nlIZkfxskFhHu3hoGXypNJN1NJFOnKK4P4FWCxvgX/SZISiwVM7LZWp3cwwHzfSrZs463yhLc+Ox/\nUFEhHDDluF6+HMJhaZ4t4cjgh43vhbwhW6ZcgLcyOQUDa0Wiws9/6vNMDh7NT+t0qj2TSCTsVd2l\nqpRU++hPCiVFboL6+kT/WHPSgD1RoERtkd2L9NWB+4zfv9u2hOc7fktnf29ekhJQiojpPcR0UfMM\nzNQHCVWMjaPfjdJXHKa1ZSGaBvVF9XQGgWOOQVVh45EfZuXZ/0Vw92Z2FcGF58BVZ8AZ78AFbwlT\nz0uNEAuXG8kSj6w2VTn5jOJx+wYjHQzTc+I5VG5dRXtJCZeMf4mfL36OmkCTIHar4YmffxOeyOqa\nYWOMpORBwXlSgFkN8/jNXOjyI/LN4+7oGg67p0mWx6csPilfef4rAJwx5Qx+ddavWNaSndhrJGHY\nn0srRVrQigrRYIuS8s6n32Gh1mSQFDclxWrusSIQMJWUqnAVamkDtQNicrlkoVAA1HSSpdvg3SGS\nlEKUlJ4ebBOooaQoRVnkxklS5LWsk6HS1ESwp5UT71jB+Lu/Qby4Ep5/3vBJsYZv5nLslbCae3w9\n+4lGKgm1iJm2utcMNd6dfBsVjZZVz6JnTjp3r90XZF7kZKZWmg6JZT5hLupN7mHOrgG2jKs38tZw\nHOzepdrbtnw5SjrNkbEXbH4zxn0rYnfpU4SS0hsKkGTAIAjW4+R9V1TYzxH0BLmi7C6mRBYA4FVN\nktKfFt640uwjyY+m2UmoVFIAHukRzoBr9gkSpyoqIW+IBfULstrvHFcDA+aC/ny/mSkzmU4a92EN\nMZX9JCfvXD5TkpA7VcVrrvkKyUCEpD9EsHsvS0OfoKUTyqLQPWMi0VSv4YMAoqwEgEf1srVrK+XB\ncj75cY9hZqoIWsKJG0tZvlz8Lu9pXezPFPuLeeOcJOmUavMZkUilTJISp5dx40Qybesxsu8DngCt\nmWtX0UrQEySajBJPD+AlmOU4m9TjxJIxg0xZVUVFgWurRGr9mn07WVcpqulqmlDUpNP0pEmgZgoz\nBpUS43MQ5MG2eQC2lELcFyS0XkTv3HzzzbaklsIMm8k/k4bxG3eCDkmll4EMSZHPVZorZcr4fJhX\nN9f2/992fTwrmWZ3fy+zf/MFPvXDv3PFQ1ttPn+KIhyXo+ke4avnEcRIRqdJP6J5OxPsnjzOeChv\nfuJN3v2suFdVBV3VeO3UL7Hpdy/z1B++x4yKK3h0Gtw/Hb73FJy4GR6fLC493ncE36jawjF1Jxvt\nkGuUk6SoKvR+UFRq7m45lgm+o4xswYqiwGzgIqg6q3nQvioUYyRlhPDax19j0S+fYvXvf2wMnFzm\nnlzQFM2YGBOpBHetvgsQCakum3fZwVFSgFRFjfAGVBSRZtZCUiaVTyLc3mOQFGuTrGqGm7lHVUUI\nMkBZoJRkSQUNUbFTWpJS6C+uoaNOkJXNZbkXdAk3FSefkvL667BrV7a5x6eEjQJszvM5fUlsvgXj\nxIvY9PaTrPn6Q2w8/XPw+uvosbhNRSiEpHR3m9fx97SJKKtMH5f09XJ0pgL17uTbhNQyalc+Rtey\ns+n3eZi7BxoCpvemsw/8WgC/EiHcup7imM6OCU2m0lUPL73kaNv48dDSwvTdzxrSvLNvBEkJUjYA\n7ZqfBANGQjmJYNA8p/M8zn6V5hYwo1ScSorz3qxKinTQltE+ALuv283fP2qvqmy7zwwSCfv1Jawk\nRZp7VDSbumC9Bydkzh/nZmXWLJG4KFZSTbBnL2GlnPkZt6+O6eMZSPUZIdYA0VQm3TxeXtn1Shbx\nKg2UGmUXJjWWMSkTrCNJys7kahY1LcLvFXk/pOnG2g8dHTDJdwwRtZK54ZM5+WQ4+WT3dyzoFUoK\nQLC3FZ/mYyAxQFJPGCRFnt/joqQ4TZ/zKhaBDnX79rK+QvjeaZpQhqy+PTIaKaAW24p2hn3hLCVF\nV2Fv7XiKN25FQeGIBUfYyMLkP9/BP255jt8+AO/eUc55136BC1dDVGkXSgph2yZDVeHqhVfTXNRC\nPkwos/+9X+8gnbazFM/jjzD+ge9TubeXDz78Dtx4o/E3RRE+cgmiDKQ7s5SUaLqXVDrB4btg39SJ\n4l51kb+ppVRc2/pcPR64btF1XFomFMvrTobKqI+iOEw+8kmDH1V4xtm+J5WURMJOUjQNkscso7Nm\nCjunCeUllRLzKsBxIREiXuW3pCc/QIyRlBHE4mknsuTMTxn/H2o0TlpPs2r3KhKphOHEBXDhrMGV\nnJGAnJB2X/8D+N73xH8cJIVYTJTzdKT+B/sO06mk6OjU1ppKSkmghGRxBVUDYiao3t5KR90Mts0S\nBYKWtPyiYCUFClNSrJ8pChwbErWUNMXD+PFwxhnZ55O7VZnvxEoUlcliNVh55tfpXnwqbRMWQixG\nZOsam7knkTAn5lwkRebZMJSUcIXRx8HuvVxacice/OxIvkFLbxGlG1bSs+wsttRVM2+PSJjlbLv1\nfv1KmHHbRSTDroktANRoU4xCdVltW7bMzFPu0n/xOKiKRmkUuvxeEvqA4eBaUgLnny+cSw0S4kyz\nn8eRuF8XJEX6pAS0HCTFoqRIR0erElnsL3Y1+zj9j5Yts5uHpI1eOqpbzT0aXqOfZH0jt3otYCbA\nc84D8h5iJTUEe/ZxQuTznNUzjl0R6KsoYiDdS7Fmvl9rO18X7VBUdvXssiX2E+dTjLosVREzlbn1\nnubVzDPu201JkUkRv1/TyjHlZhJKt3dMQWFfhqQEegRJkblBPBaSYnWcdVNS5PV9mp+aXohEY4aS\noqoiaaDHQlKumfk1bqz4B2G1zHDAB6GkGMqgBXvqx1O6UaaKF8n3fD7wqwkO+9M32FoRZNY+iFbP\npnf6PK55VWV19HFDSZHtTCREW+uK6njjo+/y0zqdYq2KxU2LjWtJ1UXTFC6efbGtHa+0P2MUjlXS\nsPSxH7Bh6mzmfKSHP1x+NNx6K+Nef8joY+kT1JXeY5CUgCeApmgk1B78HZup6YPOGWJj4swo7qxd\nJj+r1iZx2OwreO3M/6K9bgaJcSfaAjfdSEo8bi+rommg+TTu+8rbrFn2aeMYWa9Tjnmp3o4ExkjK\nKGKoSsqvXv8VAI9veNyWH+Wjh310RNuVC3KQJqbPMeOJGxttPinGaByEpDgn5kvmXEIoZErgDUUN\nJEsqKBkQkQwVW/bQWTudLfPORldVko3HDUpSrC9gIdE9ElJmXh65hvtm6gZpsTriWs+3YoXIBJuF\npiZ+940tvH7qf6Bp0NY0D1SV0o2v2PwnrNE9uUiKjFTyaDr+jLlHLQqjB4MEu/eiKApJxIK8aKNY\nxHuXnII+/nTOjU/Cq5tSkJOkKIoghy07trO1VCFZJha0m6ve4mtVG9z76rjjYPVqkdvfpf8kyqIK\n3UEvcfoNc09zsyAqYHE4dizkhjJlIQxvX/02CwMXZCkp0nHWeW2rkmKNYBkMTsLk89mVFEmkFzcv\nNq4pzT2q4jH6aepUEdLsVosJzLHuVFLkPURLagj37iWiVnBufCKr6kRUVjTVZ0R0WZFIx2gfaLdl\nr5WQyejKAiZJsW4UJpRNyDLN5npn3Ey41nYn00kjhYBUUrqigqRout1/yKP4iKX7SempLCVFwq/5\njbIi6yVJ0dKkSdpIit/jY7xPZLi2KikhbyjL3AOwp66Fii27CWdIrq6L0hXL9/+eSMd2bjxnCvM+\nCQ9ffTcdV36JRVvTlG5dR3+yDy9hAv2ZcZjIHsc/HLeD5y41w/SXLTNcETljimW3k+mvP/fdBsD5\na6B21wYuWbIagFfOXQTLlzPn6R8Yx8s8Naui9xmpGRRFoTRQSqi8nYqtomRB96xp4vx5SIrVJP61\n6g18ed4vePOkL3D/zW+J0gs5SIp8R5xKiiSfumraXnfuFJ9VVJjjsMRbyUhhjKSMIoaqpCTSYsEu\nC5YZybsOJlxNJg0NdiVlbyaZgwtJaWwUO4qqKjtB++vlf+WWZbcAZn6LieUTSZZUUNwfRUvB/Vt2\n0VE3g72TlvDUnTvpqZpYMEmxOvXluie3z6QikKvgn0RlZW4/ot4KYRfWNIhpIfQZMynbuDLLyXMw\nkjJ3Lhx/PIRSPWiphCApmkK6Suy2rVi0NUFPyyzSFVV0NM4nsPFdMxkJ7kqKBz+Tduzk1TrT0VJT\nvLz0t98YodO2tkmn6b9nm0uM86fTlMR0uoIaCV34pFx6qYhitl4bcpMU67OYXjWdMq2ZnnQrAAN6\nNyFPiJoqcXBRkWOXqPh5bc9rvBt/2cg2+8JlL2S11wnnuPJ47NFAmuJj46c3cvc5dxv3K80pGh5X\nE4gbcpl77rvvlwBEi6spje3jyo/pBN5cbZCUlJ60ZX2ViKYH6Ih2uJIUWZRxYvlE17a0lLbYxrl1\nfKqqGHtusPoByeNbSltIadAXKiLQvQ+f5jPMbGraLMegaYLw9aWEj1EuJcWjqczYr5JUYFOZICmy\nwrXV3GPtd6tPStAbzDI1AuyubcEXTTCzy8cvf/lLMU+gU/Xr77Jt5im8UyO+41eK6D3+bFqLNU5+\n8RWRXHLLHhZ+oIo5T37XFtlkOP2mfTalqrkZzjxT/H7+rPNJ3mQyh1gqyv09Ihrm8tdgw+R5/CMT\n4VkRroSFCwl3mnOsXOh1dENJAZForS2xk9Itf2VPGNSmelubJJzmHjCfodGHmQ8KJSnW8zjny0BA\nECWv11RSwp7s8TtcjJGUUcRQlZTrF10PCJl5Z7dQL764+Isj3q5cyElSdu0y6XQeklJeLnKaFBXZ\nd3EBT8Awk0ysbAFgft18ksXllPRFmdgBr6fTdGb8UaKlmeioQbpPvkhGjg9n23PAmvjNaopxO6ZQ\naJow2awvWUj55lcMdUZCmo3GjXP/vs+XMR9kCnvEIhVix1JdQ7Bnr+3YRVtidMxaiqLA/sZ5kEoR\n3LwmZ9sUBXy6n0k79/JKnW6z32/btspIOGdbcFtaREjPiy9mnc+YsAe6UHXoCCjEdeGT4ve77+Ry\nTaROs0+FNo6O1HbSeor+dDulgXKam+GSS4TJzeo75FX9/GPHP/hW25GUVMQ4eeLJHDPuGAaDm9pm\nNeN58DGxfKIRVSccwbOVlMEgF1hrIjeAt98W+UAGimvwduxF2bMbtbXVICnJdAINLz+t07m67GFO\naj4LgFiqj85opytJGV82HoCpFVOz/gZQG6nNqaQoihh70pdlv6Vws6KYpgzZRQ3FDdw1JUWiqB5/\nt1BSZKSclrLntPErReyIippVuXxSVBVm71PZVA4JDzy9+Wk+9KBY8a1Kim3xtUSDqYpq+GwAfHaC\nyNy4o6GFtKrw1Ld3seqrX0VP6wS2riew/g3WLLuGUyP/SbFaIwoO+rzcv6SCE15eQ1MnnPHsM+he\nH0c9eAOT/nlXlvI3WNFWTdV45FyRa6g7kSFpCThmG7w0vcU4rjRQCg0NhDp3ga6TSsFEn2lGko6z\nAI3FjTy1+276N/2cV+qhsizAiSeKDY4VBZGUDHotgWTOTZWcH6UjtPVvViiK8Evx+TDKZ3i1bGVr\nuBgjKaOIoSopn1oo/FkS6QQn/J+IU3bG5o8mXP06qqvFG9mVybUgSUp1te27WTtly71bd6nfvvhi\nOr7Qx5LmJSRLKwgkkszfDbcDHXXCxJQr3bgT8mWxkhQncpl75N9iMZHkrZDv5YOcDFrHLaB0x2q0\nRNR2jkBAmAamuq8hJjIrRDRSKc5ZVU2wW/T5teVPU90Lk9tSdM4+BkWBtgZRjC68eXXW/VnvZdG2\nOJFonL+02COULrrodneSoijCxpWHpPj7RcbRzmAmxbc3ezcr4RwfGzfa22okjdNaSJGgK72bvnQ7\n5QGxIAcya1JlpfB3GT/eHhacIuYq+bvB+Wyd4aWao+q21dxj9UnJhw98wFxgnSTlq18VicEGimvw\ndew1PJdfrRckJU3SuLe5gTP51ckPEVJKaU8I79qyYBlOPHbp77n33Htz9kFVuCqLpFjJOpjO24Ml\nNgTQVJWBomp8Xa2295ukPTuwNXutW3QPQNnGlVzxapLHLJl7/7zlGQBb/hPr9UtUe3VwSVKm+pYx\nr0gk6esuLuHm73+Iu4+v4fbt2wl07iG0UeTw2Df+SGYFTuG7NXuM3D93n1RHb8DD7x6AuWtWsfOG\nH7Fpwfkcdf91Wc88ZU9n5YrTZ5xAUItQ0SBIyqLtEEzC9+ofMo7xqB6or8eTjOHvaxeFARW/4S8X\n9lqUlKIGemOdLMhkem0sbhTvgQtpkLCae6z/d4PzHmXGbF0nS4WTKC/HiO7xejGyOPvVHI5aw8AY\nSRlFDFVJkS97ImVmnLQWJhttOCctQKwKYG6v9u4VI9Ni/7jwQrjgAvu5rEqKdRLzeKA0LCbtVInw\n2Lz8NYgXR4wsnHICyGVicUImaHNDWVnunUa+xWY4SgrA/ubDUVNJwu++jJPOoQAAIABJREFUlTOs\nOS8sJEXTQK8xzT1Tfcs4Zqs4TJKUlC+EHg6LytU52q4o8MHXu9hZFuLvTeBzLGRS8s265yVLYOVK\nmynJepwkKe0BnVg6O7oHcj9L6XQaCAhb9uGHi/+XaSIq4LGeW3i+/3bDJm9FSYlow+7oJvMe0rGs\nCI9cCDh8LDVNLPzfO1E4izuzy9rMPYo7SZk+3XSuBtPEBtkkRX6/P1KNGo/BPfegT5/OjpJMmn/d\nLHgo2+dVgnQlxNiwFkuUqC+qz6q6bkVFsCJrN+xUH2WSycmT7d91e18URWSndpIULW2SFJHU0bwP\n6dxqIykdHcz/5rm8XgtfcjE5HTbHnaRUesYz038KF43/HGCSlDQp0ilxo2k9xYa6AI+eIG6sdMvr\nBDe+SbK6jljE7jOhKJCIBPn2ByawZDv0RUroOfsjbJ5/LqHuvfi6Wo1jm5uFD8pgUBSoCJfSn/Gx\nOmGzyKX1lmV/V+QrEmo1EO7cabwv1Zl6SmUBM7z82qOu5TtPi1IiT04SkZa5risxmJJiRS6SkktJ\n8flE060kRZodvdoYSfmXgNUW/dIVLw16vJFQypLS3JlA6GBgUJLiMPUUFblM/Iq7kmJFsly8rYu3\nQ+/11xkXnjcvOydGLowbJ+zpuY5VFJEx1fkZWGzsLe7fGwrki9tfInZ43u42V6fDQZEx90TDmUWl\nuoZg916OvP96zvzBCaxYC7sqyohXNZjnDIdRBkw5KIscxaOc+tZ+Hp5XBQpZi7lUUlxJSjwOr7xi\nP5809/SLHWJ7IE006U5SJAFykhSrk/I55wg/JsBwGJU1b17enfu9ubhe+DkFlCKS6XjOceZEfT2c\ncooZFm1koC0VtjhrQjdwmHscPikSxxwDp51m/86ZHxD97NYvIOr3APDwwyinn45H9RjmHlXxsHSp\nGLuqKnw7upNibDhJTyHwat685h4Q790pp8CiRfbvOnfj8vsDRVV4HSRFOrRKkmKtReZq7vnd7/C3\n7+ZDHxJZqJ2IBNx9UgA+U/4nrpslHE4NkqKnIC1OpJOiP9FPV10ZFBdTtu11ghtXk5w2O+s6qiqc\np387K8EvDoOnz7oOLRygo36mOP9W05x6yinZRC4XqgOV7P7RfzFzL1y2bwJvT56BLn1xVA8rpq8w\nMiWGOncZVvVZgVMBOKHpTONcU3//LF94CT57Klxz2V9cc1HJe5EYKZLipqSEw+J3Gdbu9WJEMRW6\nYSgEYyQlD4aScdYN0uQR8oY4qvGoQY42IwykAy0cGiXFNljl6tGa2Um4kBQ35DL3WNE/YwFPffwB\n7v/Wfoq++BXj84kT4YorCmvzySeL4wdTKawkynmfuUJIhwIj3Dgg8r54+rttk3rBJGX/fhK+EClf\nMENSqol0bGfu09+jasvLXPQWdE89y64ehUJoUdPR2nmtin/8keJokvvnynBGMYFIX4OcJGX2bMFA\nHSYfp5LSFkgSTbmbe7ZvFz8dNSoNzvv22/bPZUIxiRuPvinrnLINx1dewjeP+wE6+pCUFBC74eZM\nvin57GRxtd50W9a1TMfZ3OYe5yJeVSkOrCuqcz3OICmJBHzgA/g0n+E4q+KhsVGogIKkBOlJih35\ncEgKZEftuOUUam52Nxc6709RIBqpwtuxz/Z++xS7ucea1TWdSZ5mWzgfeYSOOcexPUeeNOszzecE\nL0mKThpdKimkGEj0E/KFYd48yre+RmDDapIz5mSdR1HEPNWe2sWVZ8GmRR/D44Gu6kmkNC/hd99y\nb2A+9Pby4zt384uHdVb+HOo2vMvbU2YYf75x8Y1insww9nDnToOk1Hqm8T+1cY6ozYQV6jra93/A\ng/OL2H3Cd1lQtTTnZZ3j0PqZlaRUOQLICjX3OKMq02nTJ2X7a6/D7+CPd/5v3q4ZCsZISh4MJeOs\nG6SaUOikYjX3yGq7he4ORwKu5p7yjJNeHiXFDbnMPbbrqQpb5q8gFqlgxYozXY8pFIORlPPOM0ME\nnfJ1rhDSfJALmoRJUjLZhvu7h6ek7N9vSNGKAkptDYqus3PqMu6/+S12TDuejUd+xO6HEw6jRnMr\nKRXP3sfahlLWVGQqwmYm/nPPhZ///MzcJMXjgaOPziIpkdUvUdS6yTT3+BPEUu5KiqwQ7KxXJc07\nlY5IRat5AODsqSuyzinbqusQ0iLE9T5i6WjBPikSRxxhL14pyUSf7kZSMj4peRxnrZ+rKlSHq/nF\nGb/gh6f80Hbcxz4mxrqs30NpKSxaZJCUpJ5AU7w2p2IvgWGRlF+e+Uv+evlfgWySUqizuVtkmqLA\nQFE1no5WfLbwbZOkFBXZSYrcuBiLXE83PPccbYvMkN0V0+3P2/pM3VQAJ0mxmXtIEU1GCXgCnLl9\nO9Ub/oZ/57ukZmQrKYoi1CapokVU4ROma166aqYSeje3Y3pOXHop89a0c+6H4G/HjkcPBFk/faHx\nZ8PZ1+slWVHNjNJdLFhgllnQFK/5bNatg3ff5bjP38NJkS/YTIu5+sT6uxtJcc57QzX3eL0mSZFK\nSv1hs+Ai+NDHP5G7gUPEMKbnMRQK+VIWarKxmnsaihs4d8a5o9Y2N7gqKV6vmEStJGVO9k7EiULM\nPdbJ8eqrr2HOnMEjenJhsIk2EBB8a8+ebEfBoV6z/Yb2rCRhRji0qpHwh9H6e4anpLS1kSixrNyH\nHUZb4xxe+Mgv6Kkcz+OfEw6FzZYES3owv5IS3PI2a5qqiKtdkIKJLWLiD4fh4ouvyd/GJUvgBz8w\nM4Cl04y79mzSU06ho34mPQEfSe8AadVdSVm4UDi5Os09wSAsXmyqGblQHMgdyqjrEPJE0NHpjnUO\nmdCrqj2lvSSes/ynZR0nzT1DUVIArpifLQledtk19PXBQKCMtOZBPeUU8IpCcjc8c4O4js9jIwde\nJUhHUjjO5nNQdsKaY8kZOl0oScnnk6LGY4RiZiINSVICAfFOLAicx596/4tHLnyEJc1CFZDvSvnK\nJyGRoH3xGZy8YydTK6ewS/mT7dqFKimSuLV4jzCVFD1JMp3Eo3q4ZsUKwt//PgCpmXNgo/08imLP\nVqwpZv931M+kJk/0nCueeQYefJBH//NsHvA+xIRF59LS8HViTz8MwkpqVOwG8DQ3UBXfSdV8mD8f\nfvYzxz0/9hgEg5SvWMaHdffK29Z7kchHUpzzXiHmHpnzaf58YfJ6912z0rvHY2bx1rSR0z/GlJRR\nhFyoSwIlgxxpHq+gkEgnSKVTrrV/RhOuSgoIXXAUzD3W65x88kk0NJhqx1BRiFNqrkrEQyUpZcEy\nW8ij8/oJfxHaASgpkZYKowilOm0KD9z0hlGFVEJyBkAoKbEcFbJ1Hf/OzeypKKUzIRxwy8Pm5Lh4\n8Un527hkicgwLO0yr7yCt30fpXvWEUl0MBAJECzuF+YeFyUlFMpNRGbONMpcGfjgB+HIWlPKLslB\nUqSSEtTEc+iItR2wHTzoDXJz5RquLnvI9rnV3JMvBNltcXDDsceKPtcVlX0XfQ4+8xnAviBrmD4k\n0iflQM09zqiPQiPY3HxSFAUGIkLtLeoyHas9+CgpMd+1Bu9s7p+d5vQpp5vHZP5W9Y9HYc4c4vUt\nrCj+FidXf9TI8nvLccLfqCJkOo7mIymKorD6Yxv5UPH3bUqKJCknfVgUJNQ1jfTU7ErhigI7ukWu\nkmuq7zfuW9OgvX4mgc1ryCrCkwvJJFx7LSxezDsnihIIPs2HGvARUk0JxDaHyFQPLu0CBEk54QQI\nBvMSFMivpLj5q7h9T/5dVnF3zpELFggHdlU1w7GtJCWXv8xwMEZSRhFyoS6kMBWIF82reUmkEsbL\ndSiQNRlUVgolJZEQjp1DNPfkchwcltKQA4V8P1dUj1sIcqFYvBjOPtv+EseDxWi9w/dJ8dZWMmGC\ne1slrAns9GAIbSCHkrJ3L9pAH3sqTHXGOvFbJyrXNh5xhDhImnweewyAkr3rKUp2kCwJ0BfvYyCZ\nPwS5UFRVwV+ueAolMzUV+3OTFBC5UgD6Er1DNve4od47I8vkZDX3qBRu7skFaz/vuvY7wqSGncyX\nFHls49WrBElk6hMNl6Q423ogSgoIJQUg3GU1NSq2d+GCC+D00+3fk2Ou8tUn4PTTbe2QZQ0+Nv9j\nDPznALURc9eSz9wDMKliIh7FRzppkpSUnhKbxRkzSGleYi1TUYPZ40RVMUosNHvnGddTVeiom4mn\nq91MvzAYXngB1qyB737XiKTzaT40DYKWwpG2KK36eltm7zlPfY8FD9+Egi6yPv/tb/a6HXmQT0nJ\nFUrs/Jv8ezqd7Tjrdozxe+b5aeoBTugWjJGUUYShpPgLU1JAOM8K57nUkEOYDxQ5lRRJUvbsEf8v\nQO6wtj1XYcSRJCmFTLjO+5MRJoXkPMiFmTNFyhjrZBkPFKP1DV9JyXLUyOD0083QR1uW3VA4t7ln\nkwjT7awys8hZk4ENSlLCYaHtSpLyxz+SLCnHP9BFeOd6YsUh+hP9RJPRnGR0qPB7/JSqmYgHX3ZN\nFgldxxaqO5IRBVbYzD0WXxG349x+dyKX4mIlH3NmeWzHeC3Jy9zq1AwVbnlScsFNSdF14ZMCEO6w\nZ8e2HldcnO2Y7vGAd6Abf1crzJple3dnVArH0pA3lHWf+ZQU6+962gxBNjZ7Ph/tjXOITj8s5yIu\nSyzItPRWcw8giEch2LxZnHjBAmNM+jQfqmqvVm4j9Q0NJkl5/nmOeuB65j/+dYq+dzN86lOCCZxm\nN0PmQqFKinMcu5l/0ulsx9lc19I0QMkoKYXKdAVgjKSMIuSgL9TcA2Iw9yf6D4mSkpektLYaC56x\nzc+DQuQ+63Ueeuih3AcWgKEoKfJYWY/H+t1CQwtznRtEhI/aN0wlpa0No+KfA1VVZk0ca3SPHgqj\nxXI4zmae2UClkLj9mt82+T//fAH9LpO67doFq1bRcYFINFXyzkpiRWHDlj8SSorEaZH/hwcfao4d\nmTT3yAyXMHpO5opikqHloc8csLnniSfMPrd+xyr/W32eAgGorcj4engCIyKlW5WUQo61/gTR99GI\nGKcVfWnX43PB44FIRybsq6nJdv5blt3CqqtWuc6Zbud163OZ5yZNyjCbP/TQQzxz1X20fvH7ru+l\nophKig/xHKRK0F01kbTPD28VGOGzZYsgHV6voe6ZJMU099g2ofX1oi5aeztcdhm7Ji/l1Q/cRNFt\nX4M//hF+8xsjn8pgKNQnxUk83Mw/0t+kUGJ+Yvh6jg1dzfSyue5fGAbGSMooojosdhqXzr10kCNN\ndEQ7+H/P/T82d2wecsbakULWZFBVJXb4mzaJ0T5+/KDnKKTtVnY/3DBvCWMXlcds7Jxs3V68446D\nK68c/vVBRPiofT1DV1J0Pa+SIkP+wElS8jjObt5MoqKGQKgFsFcJBvjTn+52/54VS5bA1q0iHEhV\n6bjoU6RVDU+0j0RJ2KgzNVJKCsDxpR/n9rpYzr8bY8eqpIyAuccNUkn5aZ3O4cFzD5ikPPywe5/b\nHCkdG5SGGkEsK4LuBHaoGIpPiptKmU6LyJdUSRmNcbviMdhY93gg3J4hKc3NtvN7NS+H1R2Ws81O\nuC7ImWVNJ0VST6IpGnfffTc9leNJVdbkVBrkplKGUauqeCV1zUN83pHw+98X5peyZYuRfEkSZ03R\nUFUIKib5sinMDQ3i3NdcA62t/OWyX/PqGV+l679/A2++CRmfmkJQaHRPIUqKVJnzmXusv0fUCi4q\nuR3fcEImc2CMpIwiSgOl6DfrnDDhhGF9/1ApKTl9UjZtErGk/sEXg0JMVdZ39N577x1CS7NhLUM/\n2PUGMwkNx/Tk9ElRh+OT0tMj/H5ykBRVtZcCyKWk2LBpE/GmiUQySdKSaXvRkR/9yOz3nG089liR\nzz8Ugl/9inR1LT2Vgqgmis2FdSSVlEIWUF0HPTn65h5nvxTyLPMd87Ofufe5VeGyVmUGkwA2FBe2\nmx4MQzH3uM0Lcq1OTpiC9tzzXL/oelZMujjrODdIJUVXVaivL7gdg6175nkUVFTSuqmk3HPPvUbb\n3DYPimLmcZHkwepv0fPJG0Sxzeefz98IsJEUmzO0Zi/lYKuzlEnoxt13w/XXi/dLUYid9xH3bJN5\nkK8f8znOuvmkWJ1iBzvfsMzbBWCMpLyH8Z4y93R1iVh9mc98EEhJWiH3aC10cioEMsQ1H0nJ5URW\nUrg1btBzgzD3KD3dBe9UDWSyzQ6VpBAMocVy+6QkmibiUwZ3tsz5HMrLxbN/5hm45BIUBTprxASb\nKjFJinNhPRDksHgZkOaeRHT0zT3O51jIcx2OT4qV2DvffUlg6ovqB794ARiK46zbeyq/23fltfDU\nU3yn/Hx+cuJvCzqfxwOR9m0MlNaBx+4gPNj3crVN/m4QKkUT0T16Ek3VbAJIPnMImATMuvFInHia\n8M362tfMg556yjUih61bjUqi8jnq6Ma1ft6go9+s01RiSR4kTTl1dXD99W63XzDy9f9QQpBV1Uz0\nOFQ/rBF0SRkjKe9lvGccZ2Vqwn/+0yyVOgis2XaHfL1hoJA6P87JMBQSkTmHuavLQ4LTcVbt6c65\nGOWEzEWTZ4W2mrXk+dOD+KTEmyfaHC+tGI7fjKJAlwtJGUnz5EkniXDkfG0AWLtm9M09zn45UJKS\n69zW/nOSFKlSVYfshT2Hi6GYeyTxt5IE6QybOOtDYk745jddHWzd4PFAuGM7/ZUiNr1QslRIqgDj\nHc+QlFTGt08SD6vJ1HpNRYFfn/VrPjjtg64kRfMocNNN8Nxzon7A/Pki3fWSJbBjh3lgPC4cYF3U\nj7z9XVEhvPC//31bqe/hLPb5vuPMlZMPbkqK09qVyxF3TEn5N8F7ytwDYtdQoJIiCVa+tP6jQVIK\n8UmxXs8ZmTNc2EhKsBilt2fo9yVJSg4lBdwnAj0YwuPmk9LTA/v2kWyeaNjZnRguSZFKSrrUlKFG\nklT7fNlpu50YjeieU08VBMmKkSYpufrcGpLqTBYolZSRCD92tm+w5y5D9K0kRWYh0Hwa3HgjPPAA\nJcvnM/vp7xfUPxW92whNabK1ZTgkJdezUdHQlSQpPcXatzVWrzaPd7uOqsKl8y7lwfMfdL2epiF2\nNI8+Kkyf9fVw//3CaePoo0XBo89/XtSB0PUskqLrum2D4Xojb70lqrUeINzuz62y/GD9LaN7YOhK\nynCTcrpeY+RONYaRxqFynHU190gM0dyTL9uulaRcfvnlQ2li9vUyI7mQ6JyRZPnO64NI5kZPt8hx\nMJRrDkFJsU62ejCMmkqgZKpnG9favFm0p3kiHtyVlBtuMPt9SCSldpq4dqkZUnkwx6s090RCpoln\nJJSUpqbsTfBwzD358JnPuPf5DYtvoCwgoj/6E3YfI+mTMhLhx87rTpuW/1iZecB63wsXwgc+kJka\nLrtMpEj1+5n959sKGkeVA9sJTW2ytWUkNguGg6iioagpkqkkybiHq6++3Pi7m29NLuIiYSy6p58O\n994rIm7OOQeefRaWLhUM7sc/xmBDmUEk/Vus5p6h3Mdw5qpCfVLkvR9zDFx0Uf5jcynVuZSZMSXl\nIOFACwwOB9ceea3x+3tGSbFuaQtVUoZo7jnJuX0dBq66ShRkK+R6Iw0bSQkWoyQSEDOjUwq6Zlub\nsEEFczug5lJSADxxh/NsJvw4kUdJWbp06P2uKNDasoCdJ19O37yZZtsOonnS6M+UOXuOZgiyFQe6\nmC5bZva59VzTKqdx51l3ArC7Z7ftO5KcHChJcd7LYO9MvvMYEbGaBldeSfqijxDs3ovfm8cxDMTq\nuH27kYq4UCUlVzusMHxlUt3c23E9rbFdqGjMmHGScbzbddx8UqzIqQxMnAh33QW/+pVw4LjzTnGy\nTLEq6ZOn67rNn6xQjNRclY+Ier0QiWR/ns/J1u0YVYWXX76b228/k6985XPDa6jbNUbsTO9DHGiB\nweHgOyd+x/j9PeM4W1xsjtKhKikFmnsORh+PBjmRsL7E8UwlZKM4kLz2K6/Al79sKiZO5Ak/lnDb\nrcQzapUkKcZ9vvMOlJSQKq8yaqo4cdZZZr8PRUlJ+UKs/cKdqJb2HmzlT9dBT703o3vy4dxzc/f5\nzCpB+hqLG22fS5+UA+3j4RAsucDV1eU/ztdci5ZKsGh6R/4DW1sFgW+yKymF9OuyZXCUpaB8IQRS\nUzwcccSFxvGD5Vtxw6Dmi4kThW/OH/8oTEEupdWH0/cjNWfNnSsIqdu5c7XLes+5lBSn2nTEERfy\nqU89wte/fuvwG+u8xoidaQwjAuskdLAdZyWyXgxFEYtnVZUgLAWgkArQo0ka8mEoO5lC4fRJAbJJ\nyl13ieiASZNE6mwnhkBSamrM/nvxNUFSvJkIH6Nf162DqVPRUWyhj1YM1ydFwhZieZCVlI4OUNMj\na+4pBAeqpOTr88kVk9n0mU1cMvcS2+cjZeYZrKCjG5YuFbmDBh0fmUzUgc49+Y/bbiZyg6GZeyZP\nzl/f1JWAYM1+PbiSUl6e/feCfCxOOUU4cVjshTJBX5G/6JCSFDdIwTacYx9ZiJIyZu75N4SqqIZE\neLCVFGuisCxUVhasooCZNKwQn5T3C847T/xMSCWlp8e+S+zpgenTRczz/fdnnyBPtlkrLrhA+OrJ\ncyd9dnOPjaRMm5aXlB0oSbGaWA62Twpgq7MzWuYeJ0aSpLida0LZBJylJEaKpCxfPqS8YAYKGhuy\nXMaeAklKhjGNpP+Fq5KCOY/mUlKsn512GqxYYf97QbnJTjlF/LSQlFMmncKdZ97JRw/76HuOpEye\nLO41Vyk2N58Up6KSy1l2LAT5fQ5pKjnY8rnMdu+aq23qVFH6skCUBco4bfJpfPuEb+c8xvoCvihr\nw4wiDB+OUVBSAKQPaUJW7u3utk/A3d3CkD9pknuxshxKyrRp0GhR/4uL7cm4kv6MuceqpOg6rF8v\nlJQ897typdnvw1JSPIdGSTGueRBq9+QLuxwO/vGPoff5SPWtpjFoFd1hQ652g5GUbdvEJJPxdRtJ\nnxQ3qIrGxo0v5j3eNqb92a9hQc/8uOPEly0ZuRVF4fLDLsej5i5MmQ+jSVIUxT6vOOGMbpo7V9yi\nFWPJ3P7NcbCVlJYWYbcMuG3a7r4bbi3czqipGn+86I/MrJ6Z8xjrQP7Od76T87h/NeT0SenpgaIi\nMZkPgaQsXepeW8yppHjjFpKybx90dmYpKYubFtvO8ZOfmP0+HJIio1Hg0CgpVsVhWuUgYSrDhCTt\nkoQe6AT8ox8Np89NB8z3LMJhMb4LUVIaG42bH8kFra8v+zMVD08++Z281xqsDQURjHAYHn9cFAQc\n7jmG2K7RhJOAHHmkSNmQ65gxJeXfCHJCOtgkJS+83gI1z+HhnnvuGbVzSxyMFz4QsPukZCkpxcVC\nFnebyAvwSbFCnjvhc3GcXb9e/NGipPzl7K088eEnbOe44w6z34dDUsaVmtWVD0V0j22SHCWSFAjA\npZcKS53zmsPBr3419D7/l0FdHezenf+YdetsSSEPpA+c33XjcCoaV155T95rjdhzWL48Z5X4oVxj\nNCMRh9oGyK0+y3chHB7cjDlcjJGU9yCkT8qhcpw9WLAO5NCoadDZGM3N6Nlnw6kfDIptRY8joVs+\nJUXXC/ZJkcjySbGkxmfdOtHBlsWgIdJsq7QLEAwOvd9zTaCHIq+PHENLxy0d1ev4/UNz8MyHcNjs\n8/cdSclFwK1YtUpkbM1gJEmKmz+dqnjwZd6RUScp7yNY50lXdR1TPXHGU4yRlPc53pNKyhgKQnEx\n1Dco4heLkqKqmEpKTY0wxVjyqAxWXNANeR1n168XtnG/P2/CqpHc/RzM8Wrt19c+/hqPXfjYQb3m\nSKHQcx3ZcCQl/hLOn3X+yF18NGAlKdJB1oo9e0Tm6sMPNz4ayf6UePmStVQEhM+L03HWDbk+H8Lr\nOCoYjb4pFJLwHXWUa0Q1YHFgH6QO0IFgbBV8DyKeigMj59H/XsXBfgEP6m6pqEiQFKsN16qkgFBT\nZExoASnxnTDuR1VJegP4EhaflExkD+TPqjmcPsmZV+EQmXvm1c47KNd0Sy0+HAzHwbAiVEHnjZ0H\nduGDgdpaWLMGXnxROFNt2mRzJGXVKvFzlJQUielV02gsqact2poVgmxFvncD4KyzYGBg+O07VJgz\nR2TsPxDIcZpLRQHhAeD3ZycEHHOc/TfB9Krph7oJBw3XH2Dlz6HgoPgeZpSU2Y9/m/p1z4oU+Vaf\nFLCbfGQF5GGYewCSvjDehENJmSrq68jLuZ36llvMfj/QieVQmHv+FWX6L31p5Pr8PQeppDz7rHjR\n3n3X/vdVq6CszBamOxp9oGnmeNQUD/fff33ea+VLaOaWjfVgYbh9c9RRsHjx4MflQ3OzOE++zBOq\nKvy1jOzDls9HCmNKynsY40rGDX7QvzCsA7l5OFmmhoiDuiAUF8P27Rz26E8ILLkSJb5IlBR1KikS\nw1BS7PWCwkZ0jxYfEItDRkmprs7ONinR2Gj2+3AcZ604FErKwXymvb3i54G6T1nH+qGU80cFtbWC\ncD/3nPi/04n21VeFimJ5cKMRguzxmOZHFY3y8ua8x79XyeKhju7JlzhvsO+OWDtG7lRjGGk4Ezq9\nn/HpT3961K9xIJPhkFFcDE88gZpKEuhtQ+vrNj+XtZCsDoYFFBfMh6QvhDcplJTQhjeEQdkiqefC\nlVcOvd/fC46zh4KkTJ8u/g2WHn4wXH212efvu1dcynYyo7KTpDicZmF4vj6D9ZumWUiK4mH58k/n\n/d7Beg6LFgnn+kLxvhsfw8CYkjKGQ4aDvYtsaRH56DJWkNFFcTFEowD4+9pQ+3rE50VFIpy7sjLb\n3BMO5y0u6IS1/5K+sJEWv2j9K+Ias2YNeo6R3L2+35WUkhJRMfZAYW3z+24RkiQlnRY3ZyUp+/eL\nRG4Wp1kQ/uIweFCQG/KZb+R4VPM4zg7mkzLSKOCVtOF9Nz6GgTGFsnU8AAASYElEQVSSMoZ/G6hq\nQeLCyKBIZJ1NKyqBvjaUXouSAtlhyPv3D1tFAaGkeDI+KZF1rwid1jV1sB0jQVJURSWtpw+JT8q/\nIv4tSIrXC4cdZicpDz4ofjoyV8dFnECWX0MhyNd/aV2Ep/gVszTHoVZShor3arsOJsbMPe9BfOaI\nz3Dbybcd6maMOqwv4Lp16w5dQ0YDGTKye87JBHodSgoIkuI09wwx3tHpkyLzpITXvVpwCYMNG4be\n724kBd7/SkohWLZM5PPKh/XrzT5/r7X/gFFdLQbm4YeLqB5JUnbtghtugEsuyfLETCbFzwJrlxaM\nRFpINH4lzJ49os9lfy9YAMcfbx77XvUNet+Nj2HgPfpo/r3xw1N/yGeP+uyhbsaow/oC3nDDDYeu\nIaOBCRNgyhT2zjrh/7d390F21fUdx9+fzfNzIk+BCoGQEB9AINogw4ANoZRhIFhrRxZHGRFbBXzA\nUuSxJmXaIag82ISOTmEEa+3YWqNOLQ/GSokICy6hFvGBmhAhEAnBULJkgeTbP8652Zub3c29u/fc\nc869n9dMZndPzj3ne3/33HO+5/d0kuae2pqU2bP3rklpMEmpLr+d4ycztn87Y/u3M2nDz+pOUpYv\nb7zca0+cuycfbPM+KfWYP3+P+fMGddVVA2Ve1IvjiI0Zk1SJnHLKnrPPfuITyVjWQR6tURniOtTT\neAdTz+f/+q4k+5mgqXzzm5fvsf7ChUmuVNTjqKKocbVSu31FrESqv4ArV67ML5AsXHwx/PSn9E/b\nn3Gv9tH1wvPJ8uqalM2bk0ndfvnLhmebhT3L77UJUxjT38d+Tz+Gdu3aq91/KDfc0Hi575WkqDNm\nSG6WL35xoMzb8iJ0zz1w9dUDSUp/P6xeDVddBW94w16rz5sHZ501sg7J9SYp3d0rB13fSUrxuU/K\nMC699FJmzJhBd3c33d3deYfTdqq/gK0YgtxSEowfz6vTksRjzNNPJcsqt4uV5p73vjcZrjlxInz4\nww3vouL1tCZl/6ceYdf4CXS9degHO1Y79NDD+PGPG9rt0M09OdSklNGcOY0P+y6VdOg7Bx+czA3U\n25vMLDZM7d4hhzQ/jN1JStdUpr4h+R4OlaQUVdHjq9XT83UefvjrzJy5rWnbdJIyjJtuuomFLetp\n2XnK9gUcid1Jysb1SS1K5U3Png3btsGaNfCBDyRPma6embMOtZO5jenv48D1PfTNextTh5rHepht\njGS/MJCkVH62QtHvgOvVds091SpVI/fem/xsdGhLHWo//3POGXj4eCVJWXrGFH5wz+DrH3poUpFZ\nNJUp6ct2fC9a1M2iRd284x29vL3O2tx9cZJiuSnbF3AkKklK12827NkzsDKh2znnwJ13wuc/DzNn\nNrTt2pqUcX3bOPTx/2Dr+z5GvZNkjiZJ2T2/RQuTk6FiKZORTItfSpUk5Z57YM6c5veMZe/yO+ig\nga9WJUmZPH7CkOufdFILR/wVxLx5u2dHKIV2zuOt4KpPGCtWrMgvkAy9Nr1Sk7JhoD8KwHHHJZ0L\nKx0JDzxw6Kd4DaG2T8rk3z3LxO1b+d2p76l7GzffPPpyr3ScbaUyX9yrj/Uyv499qiQpDz4IxxzT\n8t3fcsYtnPjGE+nqEnfdlZR5bXmPG5dJ7jRqtQ/sa6ZTT4Uzz8xu+83mJMVyU33C6Ovryy+QDL06\neSYhoU1P73k2nD0b7ruv4SaearXNPQAv7Xc4ryw4vu5tvPJK4+U+VHNPK5W5uaf6WC9j/HWbNSuZ\nq2fnzsySlOHK76yjzuKBDz+ABK9WPyG8BEb76IW8dHfDe+q/R6qLkxTLTfUJY/ny5fkFkqWuLvon\nz0pG3FTXpDRBbXMPwIbj34O66j8TX3PNyMu9CM09ZVR9rLd1nxRpYHK3JicpjSQbEixdurzh1+Wp\nwdkICmPatObH3s5fESu4spwwRkOCHVPSocUZ1iu/PiGpSVm/8E8aPoE3qvYp0jMmzmh8I6NU5pqU\namWPf58qTT451KQMtk5Zynvx4mTgn7njrOWoLCeM0eqfsh/wq6bXpFSMHQuHX3gam2d+js1HvJNW\nD+Ze88E1rN24tqX7bJdjp13ex5AOPjjp+NGSB2YNroxJytixg04p05GcpFhuqk8YW7ZsYf+y1nEO\nQ4IdU7OtSZHgyBP25xczL4P7Gnvt1q1bgNGV+9xZc5k7a+6otjFSZbnoVNuyZaDMyxh/QxYsSOYD\nGjcuk83XW34vv7yFqVP3b//ybkNu7rHcVJ8wLrjggvwCyVh/pbkno5qU2qaP2uaY4Xz0o+Us9zI3\n91Qf623dJwVg2TK4++6mb7aRz1+CO+4o53FuTlIsR9UnmGXLluUWR5Za0SdlNBfsa65ZNuL9tv0F\nNiPVx3oZk6yGTJiQWXJeLwnOPnsZ4GO2jPyRWSG088y+/VNbW5PSiJGU+8yZyQznixY1vj/bs8zb\nPknJWL01KYcd1r7nl3bnPimWmzJX2dcrqUlJe8BlUJMyfz4cddTAvho10rLPO6dsl2PGd/aj0+jo\nHpd3+ThJsdy0y4VmOOPHZ9snZfHigd9bmaSYlYWP8XJzXmmFcNttt+UdQiaWLIEjF2U/TwqMrOPs\n7beXs9zLfOFp12M9D/XWpKxdm5S5a1LKxx+Z5ab6BNPb25tfIBmaNAmOWDI3GYJ5WLYzmIzkwr1u\nXbnLvYzJSrse63mo9/PfuNFlXlZOUiw31SeYVatW5RdI1o44Al58EY48MtPdjKQmZeXKNi73gmrr\nY72AurrgvPNW7f7dysUfmeWmjHfBIzZlSua76KQ+KWWN25qjEzrdW8JJilmb6KQkxaxePsbLzUmK\n5cYnj+bqpPLspPdqQ2t0CLKVj5MUy031yWPp0qX5BdLByl7uZbwAlb3My0aCVatc5mXlJMVyU32B\nueSSS/ILpM000nHW5d56LvPRa/TZPYsXu8zLykmKFcLpp5+edwilN5JahbKWexlrUCrKWuZFVG+S\n8pa3uMzLykmK5abMFxozKwefZ8qtI5IUSTMkPSypV9J/S7ow75jMJ49mG8k8KWXlY8fAx0En6Igk\nBXgJODkiFgInAFdJmpVzTFZl9erVeYfQkVzurecyby0J1q1zmZdVRyQpkdiR/jkp/ekcPGfVd0Er\nVqzIL5A2UalB2bmz/te43FvPZd489fZJuesul3lZdUSSArubfNYBG4HPRcTWvGPqdNUnmAMOOCC/\nQNrE5MnJzwkT6n9NWcu9zDOOlrXMi6jez3/aNJd5WRUySZF0sqTvSHpG0i5Jew1yl3SxpPWSXpH0\noKTfH26bEbEtIo4DjgDeL8lHbc7KeIEpslmz4Nxz4fjj847ErDU8mVv7K2SSAkwB1gEXAXt1A5T0\nPuALwGeB44HHgLsl7V+1zkWSHk07y+6+t4yI59P1T872Ldi++OTRfNOnd0a5dsJ7tH1zktL+Cpmk\nRMRdEfFXEfFtBu87cinwpYi4MyJ+DnwU6AMuqNrGrRFxfNpZdoakqZA0+wCnAL/I/I2YmVlmnIC0\nv7F5B9AoSeOAtwN/W1kWESHp+8CJQ7xsDvBlJUe0gFsi4vFhdjMR4IknnmhKzDa0jRuTPhQ9PT30\n9vbmHU7H6enp4W1vS8q9TMW/aVNy7Lz8crnihvKWeZFs2ACvv56UX9c+brUjYP36HjZu7HV5t0jV\ntXPiaLelKPikCpJ2Ae+OiO+kfx8MPAOcGBEPVa23AjglIoZKVBrZ53nA10a7HTMzsw72/oj4p9Fs\noHQ1KS1yN/B+YAOwY/hVzczMrMpE4HCSa+molDFJ2QLsBA6qWX4Q8FwzdhARLwCjyv7MzMw62APN\n2EghO84OJyJeA34CLKksU9LZZAlNKhQzMzPLXyFrUiRNAeYxMLJnrqRjga0R8RvgRuArkn4C9JCM\n9pkMfCWHcM3MzCwDhew4K+ldwH+y9xwpd0TEBek6FwGXkzTzrAM+HhGPtDRQMzMzy0whm3si4r6I\n6IqIMTX/audBOTwiJkXEiU5QykfSlZJ6JL0kabOkb0k6Ku+4OomkK9JZnW/MO5Z2JukQSV+VtEVS\nn6THJC3MO652JalL0nWSfp2W95OSrsk7rnZS58zwfy1pU/oZ3CtpXqP7KWSSYh3jZODvSJ5MfRow\nDrhH0qRhX2VNkT5K4s9IZmC2jEiaCfwI6Af+CHgz8BfAi3nG1eauAP6cZNbyN5HUul8u6ZJco2ov\n+5oZ/jPAJSTnmEXAdpKZ4cc3spNCNvdYZ0ofa/Bbkvlu1uYdTztLZ2D+CfAx4Frg0Yj4dL5RtSdJ\n15PM6/SuvGPpFJK+CzwXER+pWvavQF9EfDC/yNpT7Xxm6bJNJA/zvSn9ezqwGTg/Ir5R77Zdk2JF\nMpMkI/cTqrO3CvhuRPwg70A6wNnAI5K+kTZr9kq6MO+g2twDwBJJ8wHSgRcnAd/LNaoOIekIYDaw\nprIsIl4CHmLomeEHVcjRPdZ50mHkNwNrI+JnecfTziSdCxwHvCPvWDrEXJIaqy8Af0NS9f1FSf0R\n8dVcI2tf1wPTgZ9L2klyQ351RPxzvmF1jNkkN5yba5ZvTv+vbk5SrChuBd5CcrdjGZH0RpJk8LR0\nziHLXhfQExHXpn8/JulokgejOknJxvuA84BzgZ+RJOW3SNrkxLBc3NxjuZO0EjgT+IOIeDbveNrc\n24EDgF5Jr0l6DXgX8ElJr6Y1WtZczwK1Tyt9Ajgsh1g6xQ3A9RHxLxHxeER8DbgJuDLnuDrFcyTz\nnI16ZngnKZarNEE5B1gcERvzjqcDfB84huTO8tj03yPAPwLHhnvSZ+FHwIKaZQuAp3KIpVNMJnl8\nSrVd+JrXEhGxniQZqZ4ZfjrJSM6GZoZ3c4/lRtKtQDewFNguqZJ1b4sIP9gxAxGxnaT6ezdJ24EX\nIqL2bt+a4ybgR5KuBL5BcqK+EPjIsK+y0fgucI2kp4HHgYUkM5P/Q65RtZE6Zoa/meQzeJLkYb3X\nAU8D325oP75xsrykw9YGOwA/FBF3tjqeTiXpB8A6D0HOjqQzSTpzzgPWA1+IiNvzjap9pRfQ64A/\nBg4ENpE8NPa6iHg9z9jaRZ0zwy8jmSdlJnA/cHFEPNnQfpykmJmZWRG5fc7MzMwKyUmKmZmZFZKT\nFDMzMyskJylmZmZWSE5SzMzMrJCcpJiZmVkhOUkxMzOzQnKSYmZmZoXkJMXMzMwKyUmKmXUcSedL\n2iVpYd6xmNnQnKSYWSaqEoHB/u2UtCjnEP1MELOC81OQzSxLAVxL8hTUWg09aMzMOo+TFDPL2l0R\n0Zt3EGZWPm7uMbPcSJqTNv98WtKnJG2Q1Cfph5LeOsj6p0q6X9LLkl6UtFrSmwZZ7xBJt0l6RtIO\nSb+WdKuk2huzCZJulPTbdJv/Jmm/zN6wmTXENSlmlrUZg1z4IyK2Vv19PjAVWAlMBD4JrJF0TEQ8\nDyDpNOB7wP8CnwUmAZ8A1kpaGBEb0/UOBh4GpgNfAn4B/B7wXmAy8FK6T6X72wosAw4HLk2XdTfp\nvZvZKDhJMbMsCVgzyPIdJAlDxZHAvIh4DkDS3cBDwGeAy9J1Pge8ALwzIral630beBRYDnwoXe96\n4EBgUUQ8WrWPZYPE8XxEnLE7WGkM8HFJ0yLi/xp4n2aWAScpZpalAC4CflWzfGfN39+qJCgAEfGw\npIeAM4HLJM0GjgWuryQo6Xo/lXRvuh6SBJwDfKcmQRkqti/XLLsf+BQwB/ifOt6fmWXISYqZZe3h\nOjrODjbS55fAn6a/z6laVusJ4HRJk4BpJM08j9cZ229q/n4x/TmrztebWYbccdbMOlltjU6FWhqF\nmQ3KNSlmVgTzB1l2FAPzqzyV/lwwyHpvArZExCuSdpB0jD266RGaWcu5JsXMiuDdkg6p/JHORnsC\nyWge0v4q64DzJU2vWu9o4HTg39P1AlgNnO0p783KzzUpZpYlAWdKevMg//cAsCv9/UmSocR/z8AQ\n5OdJRvRU/CVJ0vKgpNtIRgddQtKPZHnVelcBfwj8l6Qvk/RZOYRkCPJJEVE9BHmomM2sAJykmFmW\ngj0TiGofAu5Lf7+TJGH5FMnw4YeAj0fE5t0bilgj6Yx0e8uB14AfAldExFNV622SdAJwHXAeSUfa\nZ0gSnL6a2IaK2cwKQEntqJlZ60maA6wHLouIG/OOx8yKxX1SzMzMrJCcpJiZmVkhOUkxs7wF7gdi\nZoNwnxQzMzMrJNekmJmZWSE5STEzM7NCcpJiZmZmheQkxczMzArJSYqZmZkVkpMUMzMzKyQnKWZm\nZlZITlLMzMyskJykmJmZWSH9PxgCqSrdq9WpAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "for (i,loss_arr) in enumerate(loss_arrays):\n", + " alpha = 1.0\n", + " to_plot = effective_epochs_and_loss[i][1]\n", + " if node_counts[i] == 4:\n", + " to_plot = get_rolling_mean(to_plot,15) #very noisy\n", + " alpha = 0.4\n", + "# plt.semilogy(effective_epochs[i],np.array(range(len(loss_arr)))*node_counts[i],loss_arr,alpha=alpha,label=r\"$N_{{GPU}} = {{{}}}$\".format(node_counts[i]))\n", + " plt.semilogy(effective_epochs_and_loss[i][0]/(epoch_lengths[i]),to_plot,alpha=alpha,label=r\"$N_{{GPU}} = {{{}}}$\".format(node_counts[i]))\n", + "plt.legend(loc=\"best\")#(1,0))\n", + "plt.xlim([1e-2,10])\n", + "plt.ylim([1e-3,1])\n", + "plt.grid()\n", + "plt.xlabel(\"Epoch\",size=12)\n", + "plt.ylabel(\"Training Loss\",size=12)" + ] + }, + { + "cell_type": "code", + "execution_count": 152, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "raw_path = '../../plots/'#'./newtest/'\n", + "data_size = 'out' #full' #'Titan' #full, large, medium\n", "from os import listdir\n", "from os.path import isfile, join\n", "files = [join(raw_path,f) for f in listdir(raw_path) if (isfile(join(raw_path, f)) and data_size in f)]" @@ -94,7 +275,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 213, "metadata": { "collapsed": false }, @@ -103,7 +284,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "[24]\n" + "[4, 20, 100]\n" ] } ], @@ -113,16 +294,19 @@ "print node_counts\n", "files = list(zip(*sorted(zip(node_counts,files)))[1])\n", "files\n", + "effective_epochs_and_loss =[get_effective_epoch_and_loss(open(f).read()) for f in files]\n", + "epoch_lengths = np.array([get_epoch_size(open(f).read()) for f in files])\n", "node_counts = np.array([get_num_gpus(open(f).read()) for f in files])\n", "sync_percentages =np.array([get_sync_percentages(open(f).read()) for f in files] )\n", "execution_times =np.array([get_execution_time(open(f).read()) for f in files])\n", + "# execution_times = [1.0]\n", "losses = np.array([get_loss(open(f).read()) for f in files])\n", "loss_arrays = [get_losses(open(f).read()) for f in files]" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 75, "metadata": { "collapsed": false }, @@ -140,7 +324,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 135, "metadata": { "collapsed": false }, @@ -148,145 +332,21 @@ { "data": { "text/plain": [ - "array([ 1.04816, 0.50673, 0.23602, 0.17164, 0.044 , 0.01846,\n", - " 0.02425, 0.07086, 0.09969, 0.08962, 0.07535, 0.06142,\n", - " 0.05591, 0.04328, 0.04155, 0.03081, 0.02032, 0.02561,\n", - " 0.02532, 0.0284 , 0.02367, 0.02229, 0.03027, 0.03846,\n", - " 0.03265, 0.02961, 0.02874, 0.02642, 0.02517, 0.025 ,\n", - " 0.00921, 0.0066 , 0.01082, 0.01557, 0.01925, 0.02301,\n", - " 0.02304, 0.02865, 0.02527, 0.02474, 0.0197 , 0.02035,\n", - " 0.01216, 0.0123 , 0.01153, 0.01047, 0.01168, 0.01081,\n", - " 0.00954, 0.01401, 0.0133 , 0.01342, 0.01963, 0.02554,\n", - " 0.02259, 0.0245 , 0.02127, 0.0151 , 0.01591, 0.01684,\n", - " 0.00955, 0.00737, 0.00789, 0.00833, 0.0086 , 0.00934,\n", - " 0.008 , 0.01141, 0.01709, 0.02115, 0.01687, 0.01529,\n", - " 0.01554, 0.0189 , 0.0201 , 0.01489, 0.01229, 0.01499,\n", - " 0.0163 , 0.01706, 0.01684, 0.01251, 0.01564, 0.01953,\n", - " 0.01224, 0.01 , 0.01241, 0.02032, 0.02303, 0.02526,\n", - " 0.0211 , 0.02159, 0.02427, 0.02567, 0.0227 , 0.01473,\n", - " 0.01185, 0.01649, 0.01486, 0.01425, 0.01366, 0.01472,\n", - " 0.01539, 0.01652, 0.01775, 0.00824, 0.009 , 0.01205,\n", - " 0.01131, 0.01181, 0.01002, 0.01029, 0.01673, 0.02227,\n", - " 0.01955, 0.01977, 0.02003, 0.02144, 0.02421, 0.02628,\n", - " 0.00879, 0.00671, 0.00788, 0.00856, 0.01104, 0.01433,\n", - " 0.00918, 0.01297, 0.01917, 0.0205 , 0.01939, 0.02012,\n", - " 0.02601, 0.02989, 0.03163, 0.01821, 0.01749, 0.01772,\n", - " 0.01844, 0.02178, 0.0138 , 0.00715, 0.01114, 0.01614,\n", - " 0.01746, 0.01843, 0.02218, 0.02118, 0.02202, 0.02389,\n", - " 0.01489, 0.01337, 0.01188, 0.01162, 0.00661, 0.00702,\n", - " 0.00745, 0.01256, 0.01604, 0.01596, 0.01663, 0.02029,\n", - " 0.02053, 0.02274, 0.02519, 0.01717, 0.01711, 0.01866,\n", - " 0.01277, 0.01254, 0.00807, 0.00318, 0.00821, 0.01471,\n", - " 0.01518, 0.01532, 0.01388, 0.01509, 0.0159 , 0.01706,\n", - " 0.00685, 0.00792, 0.01146, 0.01458, 0.01866, 0.0194 ,\n", - " 0.02132, 0.02143, 0.02523, 0.02708, 0.02283, 0.02433,\n", - " 0.0211 , 0.0224 , 0.02065, 0.01788, 0.00929, 0.01031,\n", - " 0.00955, 0.01057, 0.01111, 0.00907, 0.01197, 0.01531,\n", - " 0.01402, 0.01537, 0.01722, 0.01651, 0.01623, 0.01745,\n", - " 0.01407, 0.01603, 0.01633, 0.01667, 0.01362, 0.01197,\n", - " 0.01303, 0.01609, 0.01403, 0.01173, 0.01244, 0.01301,\n", - " 0.01147, 0.01222, 0.01194, 0.00741, 0.00676, 0.00858,\n", - " 0.00679, 0.00742, 0.00772, 0.00949, 0.01163, 0.01442,\n", - " 0.0118 , 0.01185, 0.0116 , 0.00998, 0.00949, 0.00898,\n", - " 0.00692, 0.00736, 0.01123, 0.01449, 0.0176 , 0.02064,\n", - " 0.02227, 0.02488, 0.02731, 0.02573, 0.02009, 0.01921,\n", - " 0.01612, 0.01428, 0.01823, 0.01897, 0.01688, 0.01897,\n", - " 0.01593, 0.01784, 0.01968, 0.01895, 0.01713, 0.01414,\n", - " 0.01252, 0.01399, 0.01416, 0.0139 , 0.01545, 0.01264,\n", - " 0.00813, 0.01237, 0.01377, 0.01352, 0.01546, 0.01804,\n", - " 0.01897, 0.01859, 0.01627, 0.01238, 0.01222, 0.01269,\n", - " 0.01065, 0.0082 , 0.01024, 0.01234, 0.0092 , 0.00996,\n", - " 0.00789, 0.00928, 0.01124, 0.01253, 0.0117 , 0.0155 ,\n", - " 0.01731, 0.01777, 0.01871, 0.01825, 0.01719, 0.01816,\n", - " 0.01359, 0.01022, 0.01235, 0.01616, 0.01783, 0.02237,\n", - " 0.02038, 0.01956, 0.01787, 0.01633, 0.01175, 0.01032,\n", - " 0.00903, 0.00886, 0.01097, 0.00908, 0.01057, 0.0129 ,\n", - " 0.01519, 0.01734, 0.01195, 0.01263, 0.01511, 0.01728,\n", - " 0.0125 , 0.01178, 0.01326, 0.01412, 0.0153 , 0.01432,\n", - " 0.00872, 0.01072, 0.01293, 0.01349, 0.01042, 0.00954,\n", - " 0.01316, 0.01913, 0.02055, 0.01444, 0.01506, 0.01873,\n", - " 0.01856, 0.02205, 0.01669, 0.01574, 0.01831, 0.01909,\n", - " 0.01225, 0.01186, 0.0135 , 0.01187, 0.01645, 0.01636,\n", - " 0.01764, 0.02119, 0.02382, 0.02477, 0.02088, 0.02233,\n", - " 0.01276, 0.01427, 0.00863, 0.00889, 0.01219, 0.01422,\n", - " 0.01419, 0.01377, 0.0116 , 0.01341, 0.01562, 0.01946,\n", - " 0.01797, 0.02002, 0.02222, 0.02313, 0.02175, 0.01983,\n", - " 0.01343, 0.01282, 0.01085, 0.01083, 0.00957, 0.01189,\n", - " 0.01154, 0.01372, 0.01439, 0.01486, 0.01312, 0.01532,\n", - " 0.01685, 0.01685, 0.01815, 0.01651, 0.01426, 0.01444,\n", - " 0.01078, 0.01316, 0.00942, 0.00868, 0.00963, 0.01259,\n", - " 0.01156, 0.01316, 0.01156, 0.00928, 0.01273, 0.01548,\n", - " 0.01416, 0.01672, 0.01641, 0.01711, 0.01914, 0.02005,\n", - " 0.01421, 0.01478, 0.01284, 0.00926, 0.01021, 0.01135,\n", - " 0.01046, 0.00885, 0.01046, 0.01085, 0.01136, 0.01341,\n", - " 0.01335, 0.01794, 0.01694, 0.01519, 0.01548, 0.01655,\n", - " 0.01565, 0.01876, 0.01672, 0.01414, 0.0149 , 0.01593,\n", - " 0.01432, 0.01459, 0.0142 , 0.01369, 0.01116, 0.00959,\n", - " 0.01168, 0.01628, 0.01822, 0.01698, 0.01843, 0.02064,\n", - " 0.02327, 0.02276, 0.01851, 0.0158 , 0.01724, 0.01927,\n", - " 0.01469, 0.01532, 0.0099 , 0.01145, 0.01081, 0.01134,\n", - " 0.01419, 0.0195 , 0.02169, 0.02199, 0.02111, 0.02175,\n", - " 0.02103, 0.02286, 0.01475, 0.01652, 0.02084, 0.02156,\n", - " 0.01918, 0.02 , 0.02155, 0.0117 , 0.01052, 0.0114 ,\n", - " 0.00717, 0.00829, 0.00948, 0.0118 , 0.01753, 0.0201 ,\n", - " 0.01797, 0.01866, 0.01845, 0.01971, 0.02038, 0.01982,\n", - " 0.01534, 0.01584, 0.01613, 0.01668, 0.01073, 0.01189,\n", - " 0.01272, 0.01643, 0.01759, 0.01775, 0.01806, 0.01535,\n", - " 0.01642, 0.01686, 0.01551, 0.0142 , 0.01528, 0.01712,\n", - " 0.01977, 0.02154, 0.02471, 0.02318, 0.02011, 0.02283,\n", - " 0.01978, 0.02004, 0.01602, 0.01344, 0.01226, 0.01305,\n", - " 0.00992, 0.0104 , 0.00942, 0.01002, 0.0106 , 0.01396,\n", - " 0.01307, 0.01566, 0.02064, 0.02236, 0.02486, 0.02383,\n", - " 0.02147, 0.02158, 0.01913, 0.0123 , 0.01026, 0.01087,\n", - " 0.01193, 0.01014, 0.00819, 0.01018, 0.0116 , 0.01271,\n", - " 0.01186, 0.01232, 0.00937, 0.01078, 0.00854, 0.00815,\n", - " 0.00942, 0.01446, 0.01882, 0.0209 , 0.01972, 0.0214 ,\n", - " 0.01971, 0.02035, 0.02008, 0.01186, 0.00761, 0.00768,\n", - " 0.0067 , 0.00521, 0.0052 , 0.00456, 0.00606, 0.00605,\n", - " 0.00411, 0.0055 , 0.00907, 0.01308, 0.0161 , 0.01757,\n", - " 0.0149 , 0.01452, 0.01329, 0.01399, 0.01114, 0.00947,\n", - " 0.0115 , 0.01317, 0.014 , 0.01635, 0.01656, 0.01632,\n", - " 0.01956, 0.02112, 0.01707, 0.01909, 0.02521, 0.02495,\n", - " 0.02113, 0.01792, 0.01446, 0.01603, 0.0168 , 0.014 ,\n", - " 0.00572, 0.00666, 0.00767, 0.00917, 0.01245, 0.01354,\n", - " 0.01577, 0.01918, 0.02355, 0.02418, 0.02145, 0.02312,\n", - " 0.01922, 0.02056, 0.01905, 0.01542, 0.01349, 0.01301,\n", - " 0.01185, 0.01212, 0.00662, 0.00624, 0.00935, 0.01149,\n", - " 0.00933, 0.01044, 0.01115, 0.01209, 0.01365, 0.01331,\n", - " 0.01156, 0.01139, 0.01131, 0.013 , 0.01427, 0.01273,\n", - " 0.00812, 0.00803, 0.00904, 0.01081, 0.00836, 0.0089 ,\n", - " 0.01264, 0.01588, 0.01718, 0.0194 , 0.01791, 0.01612,\n", - " 0.01467, 0.01102, 0.00499, 0.00308, 0.00577, 0.00721,\n", - " 0.00873, 0.01101, 0.01232, 0.01494, 0.01712, 0.01915,\n", - " 0.0153 , 0.01596, 0.01585, 0.01247, 0.01412, 0.01264,\n", - " 0.01007, 0.00426, 0.00514, 0.00735, 0.00884, 0.01022,\n", - " 0.0107 , 0.01343, 0.01477, 0.01704, 0.0185 , 0.0171 ,\n", - " 0.01746, 0.0181 , 0.01054, 0.01263, 0.01483, 0.01582,\n", - " 0.01439, 0.01615, 0.01264, 0.01458, 0.01775, 0.01775,\n", - " 0.01582, 0.01464, 0.01613, 0.01627, 0.01873, 0.01709,\n", - " 0.01686, 0.01549, 0.01654, 0.01899, 0.0153 , 0.01168,\n", - " 0.01093, 0.01371, 0.01289, 0.01248, 0.01158, 0.00956,\n", - " 0.00743, 0.00948, 0.00978, 0.0119 , 0.01958, 0.02394,\n", - " 0.02435, 0.02716, 0.02882, 0.02844, 0.02613, 0.01909,\n", - " 0.01031, 0.00965, 0.00814, 0.00791, 0.00856, 0.00924,\n", - " 0.0102 , 0.01234, 0.01362, 0.01243, 0.01478, 0.0115 ,\n", - " 0.0095 , 0.00912, 0.01022, 0.0101 , 0.01257, 0.01402,\n", - " 0.01289, 0.01205, 0.01094, 0.01123, 0.01206, 0.01423,\n", - " 0.01087, 0.01244, 0.01347, 0.01233, 0.0133 , 0.01501])" + "array([ 2899., 2899., 2899.])" ] }, - "execution_count": 19, + "execution_count": 135, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "loss_arrays[-1]" + "epoch_lengths" ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 78, "metadata": { "collapsed": false }, @@ -294,52 +354,18 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 22, + "execution_count": 78, "metadata": {}, "output_type": "execute_result" }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAo8AAAFnCAYAAADHb6MJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3Xl8XGW5B/DfmzRd0iXplm5Q2lIoFLElKUsFitCrLELx\nCloDeJGCvSgoRq4oF2QToSpQVMSLUi0I5opewCJS9n2noQh0QUq60YVuSZs2bZPmvX88fXreOTln\n5kzmJDOT/L6fTz/TTCYzZ2bO8jvPuxxjrQURERERURQF2V4AIiIiIsofDI9EREREFBnDIxERERFF\nxvBIRERERJExPBIRERFRZAyPRERERBQZwyMRERERRcbwSERERESRMTwSERERUWQMj0REREQUGcMj\nEREREUWWM+HRGPOgMWazMeaBbC8LEREREQXLmfAI4HYAX8v2QhARERFRuJwJj9baFwA0ZHs5iIiI\niChczoRHIiIiIsp9GYdHY8zxxph5xpiPjTEtxphpAY+5xBhTa4xpNMa8Zow5MtPXJSIiIqKOF0fl\nsTeAhQC+BcD6f2mMmQ7gVgDXAjgCwDsAHjfGDIrhtYmIiIioA3XL9AmstfMBzAcAY4wJeEgVgLus\ntffufczFAL4AYAaAn/kea/b+C2WMGQjgZADLAezMZNmJiIi6mJ4ARgF43Fq7KcvLQnkq4/CYjDGm\nCEAFgJv0PmutNcY8BWCy77FPAvg0gN7GmJUAvmytfT3gaU8GcH/7LTUREVGndy6AP2V7ISg/tWt4\nBDAIQCGA9b771wMY595hrf1cxOdcDgD33XcfDj300EyXLydUVVVh9uzZ2V6MWHSm9wLw/eSyzvRe\nAL6fXNaZ3svixYtx3nnnAXuPpURt0d7hsT3sBIBDDz0U5eXl2V6WWJSUlPC95Ci+n9zVmd4LwPeT\nyzrTe3Gw2xe1WXtP1bMRwB4AQ3z3DwGwrp1fm4iIiIhi1q6VR2ttkzFmAYCpAOYB+wbVTAXwy0ye\nu6qqCiUlJaisrERlZWXmC0tERNRJVVdXo7q6GvX19dleFOoEMg6PxpjeAMbCGyU9xhgzAcBma+0q\nALcBmLs3RL4BGX1dDGBuJq87e/bsztiMQEREFDsttNTU1KCioiLbi0N5Lo7K4yQAz0LmeLSQOR0B\n4B4AM6y1D+yd0/EGSHP1QgAnW2s3xPDanUJnqpx2pvcC8P3kss70XgC+n1zWmd4LURyMta3m9c5p\nxphyAAsWLFjAyiMREVEanMpjhbW2JtvLQ/kpH0dbA2CfRyIioqjY55HixMojERFRF8HKI8Whvafq\nISIiIqJOhOGRiIiIiCJjn0ciIqJOjn0eKU7s80hERNRFsM8jxYHN1kREREQUGcMjEREREUXG8EhE\nREREkTE8EhEREVFkHG1NRETUyXG0NcWJo62JiIi6CI62pjiw2ZqIiIiIImN4JCIiIqLIGB6JiIiI\nKDKGRyIiIiKKjKOtiYiIOjmOtqY4cbQ1ERFRF8HR1hQHNlsTERERUWQMj0REREQUGcMjEREREUXG\n8EhEREREkTE8EhEREVFknKqHiIiok+NUPRQnTtVDRETURXCqHooDm62JiIiIKDKGRyIiIiKKjOGR\niIiIiCJjeCQiIiKiyBgeiYiIiCgyhkciIiIiiozhkYiIiIgiY3gkIiIiosh4hRkiIqJOjleYoTjx\nCjNERERdBK8wQ3FgszURERERRcbwSERERESRMTwSERERUWQMj0REREQUGcMjEREREUXG8EhERERE\nkTE8EhEREVFkDI9EREREFBnDIxERERFFxvBIRERERJExPBIRERFRZAyPRERERBRZt2wvQFtVVVWh\npKQElZWVqKyszPbiEBER5azq6mpUV1ejvr4+24tCnYCx1mZ7GdJijCkHsGDBggUoLy/P9uIQERHl\njZqaGlRUVABAhbW2JtvLQ/mJzdZEREREFBnDIxERERFFxvBIRERERJExPBIRERFRZAyPRERERBQZ\nwyMRERERRcbwSERERESRMTwSERERUWQMj0REREQUGcMjEREREUXG8EhEREREkTE8EhEREVFkDI9E\nREREFFlOhEdjzOnGmCXGmKXGmAuzvTxEREREFKxbthfAGFMI4FYAJwBoAFBjjHnQWrslu0tGRERE\nRH65UHk8CsB71tp11toGAI8C+HyWl4mIiIiIAuRCeBwO4GPn548BjMjSshARERFREhmFR2PM8caY\necaYj40xLcaYaQGPucQYU2uMaTTGvGaMOTKT1yQiIiKi7Mm08tgbwEIA3wJg/b80xkyH9Ge8FsAR\nAN4B8LgxZpDzsDUA9nN+HrH3PiIiIiLKMRmFR2vtfGvtNdbavwEwAQ+pAnCXtfZea+0SABcD2AFg\nhvOYNwAcZowZZozpA+AUAI9nslxERERE1D7arc+jMaYIQAWAp/U+a60F8BSAyc59ewBcDuA5ADUA\nbuFIayIiIqLc1J5T9QwCUAhgve/+9QDGuXdYa/8O4O/pPHlVVRVKSkoS7qusrERlZWX6S0pERNTJ\nVFdXo7q6OuG++vr6LC0NdSZZn+exrWbPno3y8vJsLwYREVFOCiqo1NTUoKKiIktLRJ1Fe07VsxHA\nHgBDfPcPAbCuHV+XiIiIiNpJu1UerbVNxpgFAKYCmAcAxhiz9+dfZvr82mzNpmoiIqLktAmbzdYU\nByNjWNr4x8b0BjAWMtK6BsD3ADwLYLO1dpUx5isA5kJGWb8BGX19NoBDrLUb2via5QAWLFiwgM3W\nREREaXCarSustTXZXh7KT5lWHidBwqLd++/WvfffA2CGtfaBvXM63gBprl4I4OS2BkciIiIiyq6M\nwqO19nmk6Ddprb0TwJ2ZvA4RERER5Ya8HW3NPo9ERETRsM8jxSmjPo/ZwD6PREREbcM+jxSH9pyq\nh4iIiIg6GYZHIiIiIoqM4ZGIiIiIIuOAGSIiok6OA2YoThwwQ0RE1EVwwAzFgc3WRERERBQZwyMR\nERERRZa34THPWtuJiIiIOoW8HTDz3e9WoX9/DpghIiJKhQNmKE55O2DmxRcX4LjjOGCGiIgoKg6Y\noTjkbbP17t3ZXgIiIiKiridvw2NTU7aXgIiIiKjrydvw2Nyc7SUgIiIi6nryNjyy8khERETU8fJ2\ntPWNN1bh97/naGsiIqJUONqa4pS3o63vu28Bzj2Xo62JiIii4mhrikPeNluzzyMRERFRx8vb8Mip\neoiIiIg6Xt6GRw6YISIiIup4DI9EREREFBnDIxERERFFxvBIRERERJHl7TyP99xThVdf5TyPRERE\nqXCeR4pT3s7zeNVVC3DjjZznkYiIKCrO80hxyNtma87zSERERNTx8jY8ss8jERERUcfL2/DIScKJ\niIiIOl7ehkdWHomIiIg6HsMjEREREUWWt+GRA2aIiIiIOl7ehkf2eSQiIiLqeHk7Sfjzz1dh2jRO\nEk5ERJQKJwmnOOXtJOHTpi3A3/7GScKJiIii4iThFIe8bbbmgBkiIiKijsfwSERERESRMTwSERER\nUWR5Gx452pqIiIio4+VteOQ8j0REREQdL2/DI5utiYiIiDpe3obHXbuyvQREREREXU/ehsft27O9\nBERERERdD8MjEREREUWWt+GxoSHbS0BERETU9eRteNy1i4NmiIiIiDpa3oZHAOD13YmIiIg6Vrds\nL0DbVWH69BJcdFElKisrs70wREREOau6uhrV1dWoZ9WFYmCstdlehrQYY8oBLAAWoKamHEccke0l\nIiIiyg81NTWoqKgAgAprbU22l4fyE5utiYiIiCgyhkciIiIiiozhkYiIiIgiy9vw2K0b53okIiIi\n6mh5Gx4LCoA9e7K9FERERERdS96Gx8JCoLk520tBRERE1LXkdXhk5ZGIiIioY+V1eGTlkYiIiKhj\n5W14ZJ9HIiIioo6Xt+GRzdZEREREHS+vwyObrYmIiIg6Vl6HR1YeiYiIiDpW3obHggJWHomIiIg6\nWt6GR1YeiYiIiDpeToRHY8yDxpjNxpgHov4N+zwSERERdbycCI8AbgfwtXT+gFP1EBEREXW8nAiP\n1toXADSk8zfdurHySERERNTRciI8tgUrj0REREQdL+3waIw53hgzzxjzsTGmxRgzLeAxlxhjao0x\njcaY14wxR8azuB72eSQiIiLqeG2pPPYGsBDAtwBY/y+NMdMB3ArgWgBHAHgHwOPGmEHOY75ljHnb\nGFNjjOnRlgXnaGsiIiKijpd2eLTWzrfWXmOt/RsAE/CQKgB3WWvvtdYuAXAxgB0AZjjPcae19ghr\nbbm1dtfeu03I8wVyK4+NjQySRERERB2hW5xPZowpAlAB4Ca9z1prjTFPAZic5O+eBPBpAL2NMSsB\nfNla+3qy11q2rApr15Zg2jTgkUeA4cOBW26pRGVlZTxvhoiIKI9VV1ejuro64b76+vosLQ11Jsba\nVi3P0f/YmBYAX7TWztv78zAAHwOY7IY/Y8xPAUyx1oYGyDResxzAgsmTF2D48HL89a+A2VuvzOCt\nEBERdXo1NTWoqKgAgAprbU22l4fyU96PtmZgJCIiIuo4cYfHjQD2ABjiu38IgHVxvpBe23rHDu++\n7dvjfAUiIiIi8ou1z6O1tskYswDAVADalG32/vzLOF9r0aIqFBaW4N57KwFIP8faWuBTn4rzVYiI\niPKf9n9kn0eKQ9rh0RjTG8BYeCOjxxhjJgDYbK1dBeA2AHP3hsg3IKOviwHMjWWJ95o4cTYKC8sx\ndap337p1DI9ERER+lZUyoNTp80jUZm2pPE4C8CxkjkcLmdMRAO4BMMNa+8DeOR1vgDRXLwRwsrV2\nQwzLu49O1bN1q3ff2rVxvgIRERER+aUdHq21zyNFX0lr7Z0A7mzrQkVRWAjs2sXwSERERNSRYu3z\n2JFqaqqwe3cJHn1U+jwOGsTwSEREFIR9HilOeRsejz56Nj75pBwTJ8rPBx/M8EhERBSEfR4pTnk7\nz2O3bl6fx+7dgVGjGB6JiIiI2lvehkedJHzrVqBfP2DYMIZHIiIiovaWt83Wr7xShW3bSvDmm5Xo\n1auS4ZGIiCgE+zxSnPK28vjZz87G/vvPwyGHVKKoSCqPDQ3yj4iIiDyVlZWYN28eZs+ene1FoU4g\nb8OjXp6wqUn6Pw4bJvez+khERETUfvI2PBYWSp/H5maGRyIiIqKOktfhsblZ/mmzNQCsWZPd5SIi\nIiLqzPI6PO7Z4zVb9+sHDBkCLFqU7SUjIiIi6rzydrT1E09UYcOGEixdWomiokoYA5SXAzU12V4y\nIiKi3MLR1hSnvK08fuELs9Gv3zyMHFmJbnsjcHk5sGBBdpeLiIgo13C0NcUpb8Oj2+dRw+NBBwHr\n1gG7dmV32YiIiIg6q7wNj3qFGR0wAwADB8rt5s3ZWy4iIiKizixvw6N/wAwADBggt5s2ZW+5iIiI\niDqzvA6P7lQ9ACuPRERERO0tb0dbP/RQFXbvLsGKFZUYO7YSACuPREREQTjamuKUt5XH6dNnw9p5\nKCvzRlv37y+3rDwSERF5ONqa4pS34bGwUG537fKarbt1A0pKWHkkIiIiai95Gx4L9i75zp3egBlA\nmq5ZeSQiIiJqH3kbHrXy2NjoVR4BoKwMqK3NzjIRERERdXZ5Gx41MDY2JlYezzoLeOghYP367CwX\nERERUWeWt+FRA+OOHYmVxy99SeZ+fO+97CwXERERUWeWt1P1zJ1bBaAEdXWV6Natct/9OuK6ri7x\n8dbKv4K8jctERERtw6l6KE55G6W++c3ZAOahubkyodm6Xz+59YfH6dMTK5RERERdBafqoTjlbeVR\ng2BTU2Io7NYN6NsX8J9c/eUvHbdsRERERJ1V3lYe3WpjN18ELilpXXlUO3e23zIRERERdXZ5Gx7d\naqO/Obq0NDw8fvxx+y0TERERUWeXt+ExWeUxWXhctar9lomIiIios+uU4TFZszXDIxEREVHb5W14\nTNVs7Q6Y2bHD+//Spe27XPnkhReAd9/N9lIQERFRPsnb0dapmq3dULRpk9waAyxc2P7Llg9aWoAT\nTgB69UoM10RERETJ5G3l0Q2M/spjv37A1q3ezxoep05leFSvvy63fftmdzmIiIgov+RteLz++ioA\n0wBUt6o89u4NbN/u/az9H085RUZbv/RS8HPW1wPNze2xtLnntdfktrw8u8tBRETBamqA9evjea7q\n6mpMmzYNVVVV8TwhdWl5Gx5nzZIrzACVgeGxoUHmdBw/HnjmGbn/a1+TsDRrVvBzlpYCX/96Oy50\nDvnoI7ndvTu7y0FERK01NAAVFcDQocC2bZk/H68wQ3HK2/CYrNm6Tx+gsRF4/31g8WLgxz+W+/v3\nB6ZNk6qbtYl/09Iit/ff337LnEs0PDY2Znc5AOlWENfZdRysBX7xC2DLlmwvCRF1VU884f1fuxkR\n5YpOER6DKo9AYv/GXr0kZB59tISVZcsS/yZsap989eGHwJ494b/X8JgLV9wZNEjOrnPFkiXAd78r\n/4iIsuGZZ4CDDpKp57SbEVGuyNvwWFjo/d9fedTweMMN3n2lpXJ7zDHy+L//PfFvPvnE+7+/Kum3\neDGwa1d6y5uOPXuAe+9te//L5ctlp/O73wX/3lqgtlY+w2xXHpuasvv6Qf75T7l9443sLgcRdV3v\nvQcccQRw5JHAggXZXhqiRHkbHo3x/u+vPPbpI7crV3r3lZTIbWmpNF3/+MfSGVlt2OD9/+23w1/X\nWulHOXNm25Y7mW99S/pc/v73wPnnA48+2rbnefFFuW1oCP79tm0SfkePjic81tXJYKS2ND27O8VU\nob2jaHhcsSK7y5ErGhu9SjV1vD17EvdPUTz7bPj2ny1XXin7t1w8Ycw11kp4POwwYNiw9L9/ovaW\nt+HRFdZsDQCf+5zcangEgJtuAjZvBv73f737tPI4bBhw9dXhr6VTAN17L3DddW1e5EC/+Q1wzz3e\nAJ81a9r2PG++KbdlZcG/16mLRoxI3WxtbepQ98orwOOPy84uXR984P3/e99L/+/bg4bHxsbsV2Zz\nwZFHAgcemO2l6Lp+/nPZlqMOmti2DTjpJOBLX2rf5UrXrFmyf3v55eSP27FD+p7nyslkNvzrX7Kf\nPuwwOXa5F70gygWdIjz26pX4sxsejz9ebrXZGgAOPhg49NDEZuFPPpEQ+u//LtP5hNm82fv/9de3\nfZn93Im6tX9LW6+GoxOkhzV763sYMSJ1OLrqKqAgxVqiYastlQ73cpG33544xVK2vPMO8OlPy/81\naHdVmzbJwDOg81eM6upy8z1q9wk9qUxFT+KefDK3ph7T/XKqS8R+6UvAeee17WS0s7j0UuCAA4AT\nT2R4pNzUKcJj//6JP2uzNQBMnCi369YlPqZbN+9A0dAgTcYDBsgZ/saN4a/lhsc4aQADpM8ikFiV\nS8e//iW3UcJjqsrjzTenfj1d9rZMJ7F6tXzuSt97tmzZIge3k06Sn7t6eHSv1LR2berH/+MfwJgx\nuddkGsX48cDYsckHmilrgV/+sv32By7tovPrX0fryuKGLt0XZJu1Xj/x1auTP1ZHGX/604kjjruS\nDz8EzjlH9o0Mj5SLOkV4dMMHkFh5nDpVbmtrEx9TVOSFxyVL5PbCC2Xk74YN4U0m/oNFXPMkLlqU\n+PPRR6fX5+4PfwDmz5eDtlZOo4TH3buDD5ZbtsjBUelURkEyrTwec4w0zwDApz6V+uDSnvS9nHii\n3OZCeLzuOuBnP2u/53/xxfD1ffFi7//HHJP6ZOPCC2Vbe+qp+JavI+zeLeF45crk13vfulXWz5oa\n4LLLgFtvbf9l0/6mTz4JnH566se/+y4wcKD3/1ywfbu3P0pWeWxqSlwX//CH9l2uXGStFDuGDJGf\nS0rkxDzZPpioo3WK8Jis8lhcLKOO/WfsbnjUquS3vw0MHiz3u5c3dGnwuvNOuXUH5WTCXyE48sj0\nwtiMGcCppyY+T1gT3KZNUnkdPFh+3rVL/v7LX/Yec/fdcnBUYaFh1y4vfKcbHnfvlv5Po0YlVl7/\n+tf0nidOr70mJx+f+Yz8nAvh8frrgR/8oH2ee/58YMoUYN684N8vWeIdxNaubT3FlWvrVm9bautg\nr2xx35e/lULV10vLxP77A5MmyX2DB8sVq6qrpd/vCy/Eu1zWth6slKqryeuvywC2AQOA6dPTr1pt\n2ZK4PcZB50wtLEx+cugfGKLrXlfS0CDfsRserc3Paj51Xp0iPBYXJ/7cs2fizxddBBx3XOJ9RUXe\nmfC6ddI0NHiwF6jCRrdt3iw7wH/7N/k5rDq4Y4c859NPR3sPH3wg0zLo8o8Y0bb+fzU1Xh/FZJXH\nAQO8vqKNjXKG74Y2/1mu2yfTtXixV7lMt9n6ppukn9lRRyX2q8xmX6cXXgCOPVYqN4WFuREeVXtc\nDUiDSVjz69KlUnFUyZp0tXp+1FH5Ny+dW2ENmzXgtddaT9HV3Cz9qs85RwLbCSfEu1yrVkkoP/lk\n775kLRI7dsg+4NhjgS9+Ue5Lpwr81FPS+jJhQnyVrqoquboXIE3RySqPGty124g7hVpXoZ+Bzn2r\ngz3ZdE25pFOER3faHvdn/0Aal9vncd06CY3dusmOE5AdXhANXtosFDa5uI6U/u1v5XbVquQH/w8+\nACZPBubOlYOD//rcUT3+uFQtBwwIDo+VlTJ6c+BAL2QHVTLWrpXP49e/lp/DwuNtt8nnPWJE+mfG\nL70EnHaaTEsEyICA44/P7tUUXntNDrzGyGeYS+ExrlD9zDPy/urqvO/VP1eqWr0aGDlSAiGQvOr1\n7rtyEnDeeRIkw9aZTGzf3j4heskSacEoKQkPj+++K9vlo48Cd90l9+m0WMm4lwL93vdad1FJRkP4\nGWd49+kApiDvvCPb/THHSOtB//5ey0AUn/ucFxpPOSWe6apuvx14/nn5/+GHJ6886md/zz2yr6qu\nlsEjXYmGR7fyCDA8Um7pFOExyGWXAY89Fv57f7O1nuVp5fHvfw/uB6bhsW9f+Tms2qahwxg54I0c\nCVx7beJjmpulY7Quw4gREqTKyqSaunNntM77bkh86CFpvnbDsVq/XqYn2rpVXsutPPqtXSvNmRMm\nyM9BQaC+HvjjH+WAOGxYeuHRWqmQTJ7s3XfiiTLa/aOPsjNNR329fL8HHyw/Dx8efOB9882O68jv\nfu5z58bznDp5/rp1XsUx2UnQ8OHAfffJz2HdF7TiddBBEr5bWiTIxMla6ZLyla/E+7yAVB4PPVQO\n2GHhUefdO+00med15MjWA7zGjWv9NwceCDzwgJxIzp4t88xG9cYbMupWK3EAcPbZ4d+XVuqGD5d9\nz7hx0Wdt8O8vnnwSuOSS6MsaxN8F4NOflgGJYSch+viyMq/7kZ7AdhW6/rHySLksb8NjVVUVgGkA\nqgN/f/vtyZuQ/M3W/vAIBPcZrKuTaX+KiqRyF9Y3UnfixnjNRv7mmgcekJ1pc7McfN3mdx30E6V6\n4+5U/vM/ge9/P/H9Kbf5auZMLzwGVQLWrpVAqMsUtBy6oz/jDNnRp9Ns/d570g+qvDzx/lGj5LWy\nMSmufj/77y+306dLU35dnQxCqq2VvnHHHy/NiKl25uvWZf4+9O/Ly+Ob++6tt+TWHVy1ZYvM1+le\nVaexUe4fPjz5iQYg6+v//I8MeBo1Su4L6zvYVrpszz0X7/MCcpJwyCESHj/5BLjmGnk/rqVLJWCq\n3r1bV+b8g/d0oN7rr3vhf9my6FMCLVok+4hDD5Xt4uyz5f6w6p32LdR+4OmER53dYcgQmS5n0KDM\nm439V0Y5/HC5DVv+9evlM+zePTEgx9ECYK3sj5PNILFtm1xAIspJe1s1NCSvHm/cKBV8nV4urvBY\nXV2NadOm7T12EmUmb8Pj7NmzAcwDUNmmvw+rPBYWSqgDgg+U27Z5Vcd+/VKHR2u9+dncYArIgaex\nUSqTjY3B4TFK07XuZJ9+GrjjDnmebt1ah8cPPpBA+PzzchDSZuug6Ty04pQsPLpnyH36yAH4ooui\nzS33i19I9VNHw6vRo+XWPzq+I2h4HDlSbs84Qypt774L7LefTEHjTuWSamDPpElSQYl6AF60SPqp\nuYFT/3/22VIljONz0XXqscdksntAlvHYY2WUv9LwN2yYt66kGm19+OFexSjuDv7af9hdxlSeeEKW\nP1kYsFbWXbfy+OMfA9/8ZuLjtmzxuqsAso3W10tQu+IKadr3fz5ambztNglSOnG3hrxUamu9baJX\nL+knDISflGzZIsvVvbv8PHJk8nlrXdqc/v77wP/9H3DuuZl3PXBfu6jIey9h4dEdZTxrltelJY4B\nPBpAr7oq/DHz58uJQ5Sm/oULk4dA19y5wH/8h/z/7LPlJCvMjh3yHWr3q1StXFFVVlZi3rx5e4+d\nRJnJ2/AISL+eM89s29/6+zxqeASSV1kaGtILj1u2eDtK/5mjHgC2bJEDmNtHM1nl0V998lcbgODw\nuG2bnMVOmSI7Jg2GjzyS+Lj6ehlFPmpUtMrj0KHymbz9NjBnTrQR6AsWyLQjPXok3q9Vq2yEx5Ur\n5Yx/2DD5ecQIufVPF6Kfa7LLWL73nnfgfPLJaK//858Df/ubhJgLL5T7dM7RU06RW716UFu56477\nvf/lL3LrrkPabzdV5dEdWDFypASXoqLUB7trr028ylMqGm5ShdJHH/X6Rf7sZ7Kehs0f+vrr8p03\nNMg8j8OGhV+Ksb4+8UpVum0MGgT89KcyQt8/oMb/XOecI7dBA5QeecQ7cQXku1q+3AtcQOoBfVu2\nJH6HvXtHD4CffCLfnVZPe/XK/ApL7lWyBg3ytqmwQTPr13v74jFjvOnC4hg4oxXYZFfN0uprlBaD\nI46QEPjjHwePst++3ds2LrhAuvg0N3v7g2RdQNxjgZ64+dctomzK6/D46qvAww+37W/9lUcNDEDy\nA+W2bV5lpV+/8AOk7uw2bvRClr+fkj5Gz4ijVB7vuksOdm4w1Od1Dxru+3OXXYMvIDvnESOkf6dq\naZEA09QklbdU4bFnT/kc3OmRUlWnAHnP/kosIE01ffqEV0t0suE//jH+fpErV0pQ0stdlpbKuhB0\nYBg/PnmLJ6jAAAAgAElEQVQ15JVX5HsqKUk+vY36+GPgwQfl/5s2yfXNrZWDrzESKA84wGty9vvX\nv6S61tSU/IDvHoCWLZP3ceqpXhXZvQyhfgepKo/uev35z8ttnz7JQ15Tk/S9rEyj4UDDY9gJGyCh\n5PTTvUtdathzR1O77r7b+/9RR0mXhbDvyx8edRvVQXY9e7b+fPS5Pv952ba02dvfDHvFFdIXcvp0\n77716+X53PBYUiLbdjrhMerAO+3PrRWv4uLUwXP5cuA73wkfmb1mjbc/HTRInrN///Dt238i37ev\nnGDGGR6B8MqntsKkCo/uZ3rNNcB//3fi7996S7YB7WagFi3yPquw1/C3QhUVyXcSZb9K1FHyOjxm\nQvsENjTIjqCtlcf6+uBRsLqz27DBC49aebzxRpl7UnceWl0Kqjz6d/zf/37rZdPKo3sJxrDKoxse\nCwvlzP7LX5ZmNX3exx7zDqSpmq2HDJEdW79+ia8T5OabvSC2aVNiE6BLBwv5bdgggezoo6UJKO4J\nkNeskeZppaPI/WGiWzcZtPHPf4YH2Npa+fzGj/cGRSVz661ykHz2We++nTvlNcaOlWAyaVJweFy7\nVgb5/OpXMrDCP3WVy/1uNm+WZnU3bLjr1Ztvyvvv31/WlaKi4O9F1/UXXvAqS6nCozsIIspJwM6d\n0pTYq1fyiqZuCzrAR6eACguP7jYyaJDXZQFI/Bw1lCcLjz16tP581q+XOVQff9wbwQ8kVh4bGqSP\nNiDv76OPZF3U6rtW4wF5Dr2QQZCg8Lh7d7SuJBoeVZTw+I1vyHoXFsbWrPEG3elnO2SILP+aNa2/\ne92nKGNkHQ0bxJSOxYu9YBw2mCtq5dHfl9PfDK3rm17HW0++tMIPhAdif+XRGPl7Vh4pl3TZ8KjN\n1nrJtXTCo1t5rK6Wfl7+plo9iK1b5zXR1NVJkPjRj2TAijZBJqs8PvpoYqDQA6c7XYlWftwAFyU8\nAtIH64EHpAoJSFh98UVvsJHu9IIOIitXep/bpZd6/ZOCQkNDg5ydf/Wr8rnu2BEeHnv0CN5RLlwo\nt7rj//hjac695Zbg50mXv+oBeGHIrZKOGyeDK+rrw6tg2lftwAOjVR4XLpTuBJ/9rExWD8g6tHCh\nd4nNI4+Ug5a/yqNNo7W1Mv1RMv7lLStLDAx6srJokXyuJ5zgHXB79gzeJvRAW1bm3ZcsPG7cmDgV\nVqorCr3zjteEOn168sqjbgv19dIqof1Sw/qw6UFem0fd8Oh+zvqa7jam26iuG0GVx+3bE7c5/aw/\n/lj+7uGHJXQ3NclJZWOjrDOnnOJVysaOTXzOwYOjh8dkJ39+QeExVbO17veCKol6cYZx46Rp93e/\nk/vLymSfNmJEYpgCgrdBHcSUqddek/1daWnm4fH99+XE5PbbpUXAP32UFgw++UT+v3OnfG833ug9\nJuw9+SuPQPCJCVE2ddnwqM26/glZAS88Bu1w3QDmHhT8B8qGBgkPTU3yb9QoObBVO4PDNWAmqzze\neKM3TYfbb8vdWelIbXei7aBma7dq6qc7q6VL5UBw7LHys/aN9H8Wt9wi1R2dLH3UKC/EBVWGtOI4\ncaIXltMNj/7q0WmnSUXn5ZclgMyYkdkcgMnC4/jxEiZ69JAqg3ZzCBtRXFsrn8mYMdH6by5a5F2i\nUZsu6+rkIKeVm4kT5bP1n6joeuGGtzD+72bwYO9qOoAXHn/+c7l1L4fXq1fyyqMbsJOFR+0rql1O\nUg1O0O4BX/mKVMS3bg2vVrpN6P/+797/V62S9cR/wP7gA+AnP/ECuxsed+70wpO2GgRVHnUdCaoO\nuSebgPQp7NNHTrY2bpQr+7z9toQ23ZYAqaovWiTrkHu5VSC98JjOrA3+8Nirl7f/CqN9Gv19GJct\nkxNkQCrmV1/tbTNlZd6JoPt3u3bJ8vuvKhNH5XHXLjlZP/542Z709V2bNnn7pl/8onV10aXb92WX\nyTbe2OgNvBozRrqtqNNOk1sdnKaShUf/HMVBJyZE2dSlw2Nzc3B41CAVpfKo/Bv2tm2JB+Vx4+QA\n9Mor3nQVKqjy6D/zdEdtA4kHKf80P0D0yqPSg4xWydx+VkHh8Yc/lFt3At+gUbbWSqDTy9WNHu29\nX23u8ws6y25p8aYtcbsJlJbKgWXWLBnYkmwQSyr+vq+6vLqsxkhF6LTTvPVFK9cffZQ4al0HOgwe\nLO83WdPspk3yHjQ86sF/7VoJQxpowiam12Zx9/vW1/O/rlbQdDRuWZl3cAO873nrVpkw2u2TmKzy\nWFiY2G0iWXisqZHfn3KKnPCEDWYBJJzcd5+Emj//Wba5PXvCD6Rh8x9++KFcZcqdI7KlRR7vrof6\nvR5wgNzqAT4oPOo2OH683AYd4P3hUe/T0d9Dh8r3r5c9dM2f7z23a+jQ8JOWsMpjlH6PmzcH/21Y\n9XH3bu879lePdZ1csqT1qHX3JMcdde6f39B9fKaVx8WL5fs66ig5CQuqPLrb75YtEnrDRum7o+D1\ne7/hBunTWlsrJ0Y6Ddnbb8uJrQ56A2Q/HHYC4G+2dl+DKFd06fDY1CQ7bh3YoMKarVtaZCccFB79\nj21okMEXOpp4yhTZIb32WuJOBAiuPPrD4IoVidcfDqo8usLCo/9A5n893Un7rw/uhseWFgkl//M/\niTv6Hj0kROgBpblZAnS/fjIABPA+cyC88uiv4Fgrz3vXXbJz1pAFyPOvW+ctf1vnFtyzR967/8Cl\nkz7r9/jww9Lf0l95PPBAb3Lx7dvluUaPltCze3fyyo9WJrV5UkOY3q/VoLD53jTwu9OG7NwpVeuC\ngsQAqZXH4cPltqxMnvett2Rk8o4d8nidgsYVdgBbvVo+D7fyHRYem5ulkjhhgnym++2XPDx+8Yve\ndDuAt82FNV3X1SWO4L/hBuDyy73P0u1CsH27vFd3Oy4slMruH/8oP2t/w6DwqBVgDXg9esh65G53\nQeGxd2/Z1g87TJZBK3667uk15t99N3FeSTV8ePiAE3+fwXSm/ApqtgbCw6MbfvyVR525YMwY+Uxd\n7nbvjsbWz9MfoocNS3xcW7iTp0+YIOHWv35qk7WeOADhJyP+KZQaG72riamDDvK++7POktsrr5TB\nUcma4sOarePo81hTIxeSIMpUlw2P2udRq3HuJQ7DwqPugLV6d+qpXiUjqOLQt69Uydatk52oVjpO\nOCHxQBtUeSwoSAyTf/2rTOWh/Qr94dF/phpltLXLHx7dprI+fRIDS329vBd/+DNGnl8Dypw53vWA\n9fNpavLCctRmazd4VVQkPvYzn5EDpn6eYdOspLJhg7wnf+VRw6N/dGyfPvKZaeXRpWFo9GgvgCWb\n109/pwdu/Rt9L/pzWHDSA7cbHuvrvTkB3YFF+re6jBpOKirkvVsr6/3WrYlBCQifusU/nQyQuB6o\nhgZZL//0Jy8ojxolB/KwyqxWo3Tb089AD8Z+dXXyeT38sATiH/1I+oqq1au9ycaD+jECEl7862ZQ\neNQ+kXrS4J9SpalJ/u8Pjx99JOubjoTWwWMFBRKSqqu9Ef/uFZjUiBESHv2fmTupu8q02TrZ3+p3\nU1YmrSLuPmnFClmOoMte6nrRr19iKNQJ193wBsi+c9WqzMKTBt3BgyU8Wtt6wN2yZbLMr73mNTEH\nTU5urXyHbnjUE0bX0KHeunXEEXJ7000yrdPAgeHXk0+n8vjBB/KedJ+aysMPA7/5TbTHEiXTZcOj\nO9raH6iCwqO13sauB4MTTpC+MUBws3WfPnKQHDIksRqw336JTXy64fvPNt3Kz1VXyTLoRLPujjro\nTNWtPC5aJNPu1NWlbrYOqjyOHZvYpKPLG9Ts7FacamoSRyEec4ws96ZNUo3whxPlD4/uTtYfHkeP\nlp1tUGUpyO9+JyHXX5XVA6G/v5WGR38QN8ariPivROKOko0SHrW6oetEz57SrKzhUQ/oYeFRD8Du\nVCR1dV6gdueZ3LZNll2rt1OmeL9zg0Z9fetQ1aOHjJJetky+W63orVjR+oAfVHnUbQXwmuLLymTA\nlo6O9tMgpBVeXc9ffjk4cOoVoM4801tX/CcERx8tVS79HIPWw4MOSjzBCwqPf/iDLLeGRv90Rhp4\n/eGxrEw+a52D0Q1tw4bJtqHrp/v9qBEjZPvwhw93Xk6VTrN1uoNtdJu5806pJrvzhq5c2XqdUFqp\n/bd/ax0eBwxo/XkdeKA352UUv/udTDfm2rBB9uvFxd6Ji7/vsF4YYehQr8n5wQdbr2dr1shnpdtQ\nr17yXP59yrBhMtn617/eeh3s3z98nxC0Pw8Lj3/8o+yP3VkYdu70Tmx27wauu8772w8+CP9eiNLR\npcOjW3l06bQk7k7zllu8Jiz38XrAcIOmNm+7j3N3HkOHJoZHPbv1n21qODv1VNkJTJzo9RdKp9n6\nlVdkLsddu6JXHt1lOfRQb449d3mDKoduxWnnztb94LTZeuDAxGqvy9/n0T1I6k69okIep019WkVI\nVXnUioL/TD3sPZWUSHOujhR1jRgh64Xbp2vXLu8a5sOHe6EgVeXRne7IGDm4+CuPPXvK9+pvttYD\nsNs/6+WXvcDgNnFu3Srf0QsvyMHf/Q7cJs6gyqNWb666SroQ3HabDO7RwQMuf3hsaUn8DPUApgNV\nwqbS0e1KR0Mfcoj3u6CqkIZH1/jxsj0//bQMjunXD7j++vDKIyCP1zkgm5vlO+rZM7FJfP/95Sos\nyh8e9f2HdRXRyqO/4gfI9jpzZvBcqDpAx/1et23zQlFQ5TFVeNy9W7ZNf3cVIDw86r7iC1+QE+Kz\nz5bPFZCTzbCQctFFUkksL0+s3AedhADe3KNhJ4a7diU+z8yZ0t3BtXGj91n27SsnBv4mab0kK+Dt\ne6+8Ui4L6qqpkVutJvbs6VX/77/f6088erScMPsvMgDI9715s1Q5/Z9vUOUxqNl6z57gimOvXjL3\nJiCT8F9/vTdQc+nS1tsqUVswPIY05fqb6HS+LiBxBxs0eXJQxcENj4MGeQe4AQO8a5nqTsddBgD4\n2tekaezUU73HpGq2dsOjG77CwqP+vTanuVWX8eNlx64Hw2Th0Q0NO3fK5/OPf8h1tfUz37gxvMka\naN3nUZd//nzvc3v1VQlRGh41fETtG+UfvZnsPX3/+95URi5trjz0UJngGZCpR/TgUlAQvdm6tDTx\nMy8tbR0ejZFA51Yet21LDGkabqqr5Xs87DBZj196SfrSbd0qYWnAgNajszUsnHeeVPr8oUorP8OH\ne1WbN9+UEBNUeXSXc+nSxHCglccpU2QAWdjgmnXr5OD3jW/Iz337ep+v/0Rh7Vo5UPvX8UGDZHs5\n6SSZLuoHP5CKYdAUVy43eG3YEBzkXPrZ67obVnlUxcWJzdauL3xBAnqQoPDoDlhpS7O1LqvbXSXV\n9czXr5f1sWdPrzp3773yft54Q6adCmKMhE3/fK7LlweHx/32k8827ATjxhvlPX/wQXjzrfv9GSPb\nl3+bdGdbcMO8f57Wt9+W70v7Zvbq5b3/qVO9fZS/K4erf39Z1smTvUtWqqiVx+9+12v10H2kvied\nR1VPNJcvlxOd999n5ZHi0WXDo7/Po59/jjMND0cemdiJXQ8Y7oatB0L3ef2d8ktKZAdSUiI72169\nWlfi9Ln795cd1vXXB4fHoJ2N2+cxSnjU5928ufXBTpuZdEqVVJVHf3g89VTZqbqDlMJGWgPhzdZH\nHZX4/nr0aH0WvWaNnP2HXcZPA5p/YM3mzfK7sCARRJu0TzlFpiIBJCQDMlcm4B1Iwvo3Aa2bC/Xv\nNm1KvE4x0PqSmBqWteqk6+bTT0tYKi2V4HD88dJv9uOPU3dd0Lki/Z+FHiAHD5Yg6E4t4/8eDjtM\nlk3XGa3MnHii3LrBtbRUJprWS41aK9/F3Lly0Pc3+em26J8CSUf0h4UWdcghsv1oH7uw7hNu8IoS\nHuOsPCYzYoTsL775Te96324Vzf3eNACmqjwGhccolUf9HnXWhcMOk22gpUUCcDLuPspamRpHp6Vy\nFRTIPuTOO4MnO9epnO66yzuxABL73G7YkLjP6d8/eeXR7avpPyGtrZUTR91f62dcVCTriG6fycLj\ngAFeJfXxxxN/F3WqHneAju6T3X7P1nrr4A03SF/jpqbgE2GidHXZ8Kh9HqNWHhsaJDi+8Ubw7P/u\nY3Wn5R40/MGwtFR2NL17B4c/wDsYlZTI77t184JEOlP1uM17YQeyggL5TDSwuLSpUJuuN26Ux/iv\nS63L+s9/ynvy7wTd8Jis8hgUHrXq5ldc7N1/3HFykDjvPK8SGMYfHjdtkh16QRpbhFbP+vf3+ko+\n9ZQ0qR93nPxcVCTr169+Bcye3fo5duyQCZP9Ta0aJv2hIiw8apB1m3U//Wn5fNyD5IoVqSttKixU\n7dollUd3Kip/eDzzTHkPeu1qrZL94hfAPffIsil97/PmyTp7001y4Puv/5IQ4u+HWlIif+PvA7dq\nlVSg/JeK89Pn09G1qbpyRK08tiU8rlkjzY/Jtge/wkL5vles8KZZ0orT1Vcn7msKC2W5/OFxxgzg\nuee8n9sSHrds8dbPH/5QtrstW6RCOHRoYgU0iBseP/xQtsFjjgl+7GWXSaXZDUdK161lyxK717jz\nh7rN1kDrymNLS+J1tV3+1/SvC7qPGz5c9h/6/Sf7Tvv3TzxmuH0WozZbu6PYg8Lj4sXBs0/4p4oj\naoush0djzH7GmGeNMe8bYxYaY85O/VeZS9VsvW6dXE5PdzBBU24o/1lhUOXR7+CDpaKnO2j/zkKf\nF0g82LdXszUgO6ig6Xz69JGgpM1GOqI1yPe+Jzuwxx7zKo/usu/enbrZOig89u8fHuy0acadFHrd\nusR5MZUGKX+zdbrVH8Drf/m5zyVeOcSdSgiQz/ztt+Wz2b5dgq320aqqkhMSHV2rNFD5P+eSksQ+\njxoetSlz4kTvuQ45RNYLt0JXWxseHv0nIP7H6aCp1avlfbgHIf/0Kj16yPqt1RW9okq/fjLoyw04\nbnB++GGviqvrbdD30r9/676fq1YlXl4yjIbHf/1LwpJ/KhmVbrN1W8KjNv+ne0DXbVu7Tuh+6sor\ng1/HDY8bNkjz/oknyowIQPLwGNZs7d93lpXJc3/0UbTqVlGRhLY9e7yKoTsy3jVpkqwzf/tb4pV/\nrPUGin30kfzT9dbd7/mr+/7K48aNshz+KjfgVaiVf13Q713XvWeflUpfWJ9uIHGdLiiQftUqSrP1\ntm2J34u+1w8/9KaIe+KJ1lMoAcmPAURRZT08AmgGcJm19jAAJwO43RgTEKXilarZWpvXHnxQbpOF\nR/+VN4Iqj4D0Q9M5tm6+WUbi6Y4uKExpZc/dkeh9N90kO6eXXw6uPKbbbA14O0F/BQqQIKCDUsIq\npYBXOaivbx0e06k8+gfMJAt2WiGbOjXx/qlTpUO6Szv533pr4gE11TIFGTtWDl7HHJP4efgrZTrY\nA5AA/sgj3mTmWinxT1qsBzp/v8SgymNJiTeY5eyzvSB3yCGyXG54XL48fB3wv5b/aj1jx0rI0UCo\ngRUIrkLvv78EzTVrZMoc9/EuNzy6JwjaTB5UAfWHaEBeyx9ig7jhMVk3hUz7PKYKj7rO7L9/YsU4\nCh18pMu0ebOcnAWdhGq/6h/8QMKqe+WUiy6S26Dw2KOH7GPCKo/+fade+WbZMm+QSzJ6Iqz9oIuK\nwrfzPn1kfbj22sRR3atXy+d8wgmyni9b5s3w4G7f2tdX+SuP2q/RraDX1cnFBzZuTBxx7W8C189c\n1+/PflbW92TcIHvxxRI4rfWmykrVbK0nHXPnSlO/Vh4/+UTew3HHSSuIu+1/4xuJMy8QZSLr4dFa\nu85a+8+9/18PYCOANGtA6UvVbP3oo9KZ+corpTKRrK+Yv9k6rPL41a96owCNkQOlHsiCLi132WWy\nM3ebUnSHq80cxx0noS5qs3WqyiMQfLD7/Odl4MtLLwXv3NzX7d5dDjjJwmOyPo9BA2aShccXX5Tm\nUe0I7gYId6BTS4sXsj75xDtwRnmNVNx5Of2f31lneVPR6CAPPXDpeuOv1Gig8ldCSksTTwZ0epFZ\ns2Tk94EHyiTEgDRt9urVetRzWGDq3l0GPEycKH979NGtH9Orl9cU5m9q99tvPzm46wTx/oEHyq38\n+QMhEBwe/SEaiF557NNH3kfU8PjKK9HCo37/69fLtrd1q3d5z2TPP2VK8ipVkGOOkT6Puh5pE3LQ\n8wwfLk3UP/uZVHXdJlINeUHh0Rh5T1HD46BBsm5+8EG08Kj9Cpua5HsvKUn+Oeh6XVcn3/Xbb3sn\nh9Ony3r+1FNeeHSXe9u2xO/aX3lctEi2Ye3+AcjyjBwp+yJ3G/I3gev3HmXdU+6+5swz5TkXLfLW\n/6Bpstx9om6Dxx0n+w7dx2/YIMeSY46RlhftDwoA//mfif2UiTKR9fDoMsZUACiw1oZcPyE+uuOq\nqwuvDEyd6k0erZdUCxLWbB32eFey8HjssfJc/n6DftbG12wNBFceL7tMfl9Tkzw86t9v3x4cHnft\nCr56iX85dEe5YoWcPScLdocfLgeP0lJpwnLPrt0DyKZN0jT1f/8ng48efdSrKGzenH7l0U8/t6DP\nT6sSWnGsq5P1avFir/Lg0s/HHx5HjUqsJmh4PPhgmXcRkOfTqaI0uBjjHZiTrQNf+5os45IlwZ+H\nOzF6SYmcUPjn1FMaHrUC/qtfBT8uaFomd3LssMrjmjXynh59VJ5jxYrEa1OHMUa2u+bm5NcD18/u\n+9+X5Up17fAhQ2S7qKyUKs+GDd7k30F0GwoaJBKFO1dg0KArNXy41yzcq5esc8cdJ5VIXf+DwiMQ\nfGlS5e/iooFq48Zo08G44TFoiiW/WbPkdvduqdSWl0uwHzNGTtCGDZPfjR8v37G+J2u9KaqUv/K4\neLE8j3+/pu9JTzp37ZL3nWl4dPv8aveXpUu9UOjf7v3HGN0Ghw1LvNb5J5/Iz0ccIftpt4k/SqAn\niirt8GiMOd4YM88Y87ExpsUY02pogjHmEmNMrTGm0RjzmjEmpCdLwt8MAHAPgG+ku0xtoTuuzZvD\nD6a6Uauo4VGbrYNChF+y8BikoKB1/zgguNk6KDy6Yc4vWeWxoMCbWiOT8Kg77GQVHw2PTU1yEHr6\n6ehVwYMPluqBtXIwcA982mQ9dKgcfLZt86oPWvnIhH4HycKjHsTr6mSHv2OHjCQNGhQCtL5/7Fip\ngq9aJe/11VeDByb4+9KWlXkHvHRGlPv16pU4Yfapp4YPTtp/f1kH5s6VaubXvx78OLeismGDVJA1\nKADB22e/fsDrr8v/f/lLOSFobAROPz3a+9CDc9AACeX/Hv3dIvwKC70A98QTiaORg2hQSLfJWqUT\nHtX27VIBPugg2abWrpWuDGHhMeyqQkDryqO7rgb1HfTTffDu3dG2P2O8fYNu1889J0G4rEy6ftx1\nlwzc0QnYV62Sit6ePa3Do1t5XLw4+FKQbiAGEq9Uo3QfF9YtI4i7DQ4cKJ/FunXphcd+/WQ7d691\nruvcxInys9udJFU4J0pHWyqPvQEsBPAtAK2u8WCMmQ7gVgDXAjgCwDsAHjfGDHIe8y1jzNvGmBpj\nTA9jTHcADwG4yVr7ehuWKW0awKwND4U6CaxK1ufR32ztnysxjB680jmga9N1ZaV3cApqtm5q8kY9\nq2TNQsnCI+A1zwcN0HHp/HVB4VGbV1JVQHfulAOwakuTsr9q4l5OTZu4tTO8fmeZ0HUqanh0qwd+\n2pTr/y50Sp7f/16aXXWEcRhdL4YO9T7DTDrM6/deWBjeHKv05GvlyuQh6uSTvf+vXCkhyD04B21H\n/fp5gWfLFuDnP5cmOR1Akop+/8nCo7uOn3RStGqaNqVPmJC6qVu/N7cKlY4BA+T1dBLzKOFx1SpZ\nb8aO9Ub8jh4tn2VhYeu5ZlNVHt11ya28JftcldvnMUrlEWgdohYulC41gHzWM2dKGNN90MiRXjO2\nfwoj93nWrAnuL6vdazQ06v4i08ojIK0N998v++ShQ2V/oCHQ//n5+4GvW5d4ArR+vQRkbbYeM0bm\nv1yyRKboufji9JaNKJWAGlZy1tr5AOYDgDGBUaQKwF3W2nv3PuZiAF8AMAPAz/Y+x50A7tQ/MMZU\nA3jaWvundJenrdzm37DQ4J9MNVmfR3/lMUqTNeDt8MOu7RtE+xTqdD+ffBLebJ1sfkG/ZM3WgPc+\nGxuTBxA96w8abe2/PnjY69TVSThWbQ2PbnB2L0Go38/y5XKWvn175uFRv8Og5ykulvu1guGGx6AD\nrS63/3vV8Oj2H0x20NK/Hzq09fWh20IDY6r+aYCErdtuk1Hmyfq4Tp8u38fpp8v3ETSBuZ9bpdI5\nPdMZDBAlPLqhVedTTOXOO2U0eWlp6srjzJkSeNs6abPuOwYNkm1XL13qp9/T4MFSnauvl/XIfX9b\ntsj66f9Ok4VH/6Vd3ZOgdCqPbp/HVILmO/zc51o/Lmi53WXVmR9U2HflD4/33SdhXCt7gMyucNJJ\nrWdZSMWdj1Srhxp8/ccPf7/6tWu9dXfYMAmOf/6zfDaDB8v3eNVV8vtRoxL3pURxiLXPozGmCEAF\ngH27WmutBfAUgMkhf3MsgC8D+KJTjUxzM0yfGx7Dqmh6DWQd2RjUXAwE93mMWt3Rs2//GX+Uvykt\n9ZoQ3Tn3AK/ZOp3wqEEv1ajyTJqtVbJwrSHWneQ3jsqjhuw+feRA0aOHN1dgHOFRhb039z3cfbc3\nlZC/aRqQIFVa2rqDe1mZPP8bb3j3JQsfbuVRq3JxVB6jNvHra/qvDe6n/SuXL5dQlKxPLOAFYHed\n8rcUJKPVuFSDYNL1ta/J5Ng7dnj9z8IUFHgnA22h30V9vbyW/5J86itfkfBw+eXe/mL06MTPeMWK\n8JOeoGbr5mbZvt11yR34FKX/sNtsXVcXbZ3q0aP18gSFPv/0REDrKc927/ZGOGvFLuj1+vXzTvqe\neGo8QhoAACAASURBVEKu1OQeC4YNk5OLTLq9DBvmVR6HDm0d4g8+WPaHetJYW+tVSjWo66Uyo/T7\nJcpU2pXHFAYBKATgm0UP6wGMa/1wwFr7cluWo6qqCiW+rbWyshKVEU+x3INOsua3wkLv4Kx95vz8\nI1rTqTxOnSrTOugVGqLQcFVaKmebn3ziXQVGabN10PV/Uz1vsspj0OTffr17y2fQ1BQeHpMFmKB5\n9+Jqti4r8waPVFTI6MzLLpPHRf3OUgn7/AYMSJx37ZZb5CAbdOIwalTwJQ2NkcCxcKF3X7Lw6FYe\n9YCeTpXbz608RnHQQXLrv5qHnz7f2rXRJmvXx3/hCzI3JJDeOqKfizugIMjo0cAFF0R/XsCbmD0s\nkMTFDcs6wjbIiBHSdPnAA959Q4cmbmdh4TFstLWe2IVtx1G67Pgrj1GbraPs04qLE08+gdaVR0BC\ncEND8sFTetK8Z490q4jaNSIdQ4fKKPiCguCq7Yknyj59/nwZZf/uuzJICEh8/AUXJBYSqqurUa0X\ntt6rPmhKA6I0xR0eO8zs2bNR7h/Rkgb3zDFZEAK8Du16IPTr1csLlqtWyeWmonae7tZNJpRNhy57\naWnriqP7mLY2W6caGJQqPBYXezv4toRH98oh48bJKMS2VMv8B5D16xOrfJWVMkn36tUSqNqz2RoI\nDjdRmvf80gmPbuVRr3qTSbUt3cqjXqIt1UAWtyoUpb+cPt4Nj+lMd3P88XI7ObA9xOO/fnYUxcWy\nTkWZ3icTQ4Z4fQWnTQuf7Fy54Wjw4MQKXrLKY7LwmMkJl7/PY9Rma3fS7rDA2bt36ysQBV1soanJ\n23eHfVdapfz4Y9mnJrvsYFsNHSr9oWtqpFXCr29faSp/+22Zy3L7dq/p3N2nnXNO4nYQVFCpqalB\nRUVF/G+CupS4p+rZCGAPAH9D3BAAARdKyp6olUfAm+w4bLSoO2fYpz4lB5y4qlhBdHRqsqY9t9k6\n6kE1rvDYu3dm4VErKu+9J5dRA4KvaZuKv2ri79f05S9L5emee7zlzkRbwmOUoOTnn3Ij2UHXrTxO\nmiQnN6eckv5r+p8vanjU6Zm+/e3kj3O/F73aSvfuwSNgATk5KyqS0d5tccgh8n21dbBKMsXF0rzY\n3Nz+TYglJcBPf+pN05SM+xn37CnVbQ0gS5YEV97Cmq3DKo9LliR2qUimrX0e3WtN33pr8OOKi70r\nIqmgyuPu3V54DKs86vy0GkbbIzy6fYLDuh/sv78EWJ3uS6d46tHD28+GbS9EcYu18mitbTLGLAAw\nFcA8YN+gmqkAfpnsb9OlzdbpNFW73ICUqvIIJK8kDhjghSUdbZlsSpxMaXNmsmY6rTxu2iRn5489\nFjxHpCtVs7WOKm9reNQdtl5vN8yXviQhtUcP+XfNNeEV1mSCmq3dsDBkiDQHaatOXH0eU4XHu+6S\nA/5HH0W7Ioqf21x55pnJH+tWHoH0R4SGPV9bZgdIxl03NTzW14c3f372s/L5pTM9SkcpLvYGWES5\nTF+mdPLsVPzhqHt3qXR17x5eUQtrtg6by3ZcYOekYPqdb9gg+5QoVfiePeVEHpDvPyzI9e7tPU65\nRYJ0wqNWHnV+1bYOcErG3ZeHVVP320/6Sb/0krQ+uJXSN94AfvOb5DMvaBM2m60pDmmHR2NMbwBj\nAWg9a4wxZgKAzdbaVQBuAzB3b4h8AzL6uhjA3FiWeK9Mm63dg1+qymMqAwZIhU87VQOtr34RJ60E\nJJtcWDuW65VTgq4W4pdqwEzPnvK+Mm227tMneTVU53MDZCfpH10ZlU7X8fOfy9UV/M3W+vwvvST/\n76gBM0ccIaFv9uy2hcevfEWqKhMmAGeckfyx2sczyjQzUegghEyql2GmTAFeeMEbRJLsBMMYLwjf\nckvrPr/Z5O5P4vrc4xB0smmMVL3WrQte1rBma90HZbLv1P2BXu4ySpW2Rw9vP5tsew1aLnef4w+P\n3bqFhzatPK5aJYEtSrEhXe4Ao7DuBzrp/ksveV1Q1OGHy0j/ZLTQwmZrikNbKo+TADwLmePRQuZ0\nBGSC7xnW2gf2zul4A6S5eiGAk621G2JY3ti4TSSZ7gwGDpSAc/313n3p9DVsq2TTn/TvL82F778f\nvdoUZcDM+vXRKo86ujbo6jhxhbRUioulGe2KK+QAETQdx4ABXjeAjmq2HjrUW//aEh6NkcvMRVFe\nLgfnuELMBRdI898558TzfK6HH5bJmsNmNQgTpcm2I+n3P2xY5iemcQqr4mp4DKripQqPmew7NcCl\nEx579vSm2EnWNUi/gwMPlOf31xn84XHw4PDPRyuPejWn9hBldPqIEVKNX7gwdTcQovaWdp9Ha+3z\n1toCa22h798M5zF3WmtHWWt7WWsnW2vfSvac2RB35REA7rjD26jbMzyWlaXe0Wp/yH/8I3wUpl+q\nPo/ahLVrV+rw6H9OwAuP7XHmHsT9XteulYDvrzy61ZhM+6lqn6qwysHw4XIgKivzDoAd0ewaZx+t\nkSNl3sZ0r8UcRf/+beuekGt0veuIJuu28PeV1isEBZ1khl1hRgNlHJXHDz+UE4Yo/X+1Gq3X3Q6j\ny3X00TLf4UMPJf7eHx6TjYrXyuPatdkNj+73owO+iLIlb0dbZ9rn0Q01mYYZN4DMnCnX8E3VFy0T\nK1dGXyZr4wuPOnk3kPygEXYtbv2/+9m3J3cZ3QnCXe53l2nl8ZFHZOqfMF/5igxQ6NHDC5iZXhKR\nco+ud+0xsCJTS5e2Xud+9COpgAeNPG/PyqPbbL3ffqlHiwNeeAya0Nw1fTrwhz/IPKlBUy25c0ym\nCo9u5VGvVhO3KNNMTZrk/b8t84OyzyPFKW/DY6Z9Ht0dT5SdVjLuhv+pT0m/sPYMSFGe260u6ACE\nqM+brNlaB+skO2gEBUb3/8ma2+PkhkedXNdf3YgzPI4eDXwjyZXZu3f3Dj7/9V/yeUfpi0r5JZcr\nj0FzFPbsGX75ul69pBVlzhzgwgu9+zVQZjIwUPcHtbWpp0xylxVI3UowaVJiH3Q/f+UxWfcRt/Ko\nl0KMW5STyN69ZYDTmjVtq/yzzyPFKe6perokDSBa1SouzjyQZsoNRen2eUzWbJ1JeNQddkeNkHXD\n4/LlEtx0zk7lfk4d2T+tXz9pTsv2ekLxy+XwmK5jj5Vb3zzT+/o9Z9J9wZ1rMeqURqn2Uem+9u7d\nqSdz795duuqsXdu2eVmj0M8xVWHgiCNkblOibGN4jEH//nJd07lzs70kHrfyGLUKqmf1YSGqZ09v\nUEiyamZYeNSBEJlOFxOVv3/SBRe0Pti54THKVTGIUtEKdmcIjxUVcgUm/7Q3jY2Zn2y5+4ao4dFt\nts5E0ICZMEVFEjCbm4MvJRqXOXOA119vv+cnilPeNlvnkoICmdA6l6Sa0zHIxIkyBUtYiNId96RJ\nyauHYeFRp+/pqPDov9ZxUBOxhseOakqnzu/ww4HTTks+lVY+GTNGmpZbWrx9w44dmfcVLyyUkzlr\n0w+PcVUed+yQ/VKqyqM2gaczv2m6ZsxI/RiiXJG34THTATPU2oknyr8werDQ0ZlhwsKjVkDbq9O5\nn39UqT9MAl5onD27/ZeHuoYhQ4BHH832UsTnwAO9ASN64pdquq6oiorkubNVedSr1aQaba0nvm25\nTGqu4IAZilPehsdMB8x0Bb17p55EOh16sEg1kjwsPH75yzJgZcqU+JYplZ/8RC7pNXZscDNb9+5e\nUzwRtaZzhC5fnhge4+gjrCOZsxUetTk+VeVRp15rz8pje+OAGYpT3obHOLz0kncZsc5ILyEWl1NP\nlbksU43eDguPxnRscASA//7vjn09os5G+wNq9Q2Ip9ka8PYPUSfL12tx6+wJbZVOeCwqkiZ7IL8r\nj0Rx6tLhUUcSUjRlZcAll6R+XFh4JKL8o/2C3fAYV+WxqEguCxi1oqcnn5meGKdbeVQMj0SiS4dH\nah9BI6yJKD916ybzEG7aBNx8s8wqsWdP62mv2qKoKNqVZdzHP/FE9GbuZM8DSHgsLk7eDO7uzzId\nqEPUWeTtoZ0DZnKXv6maiPLbgAESHl9+GfjgA7kvji7n3bunHwQ/97nMX9cY2U+tXp286gh4lcde\nvfL7ZJgDZihOebspcMBM7mJTNVHnMnCgDBrZudO7L44+j716AQcckPnztEX37kB9fepL/en+LJ8H\nywAcMEPxytvwSLnL7SNERPlv4ECpPOq17YF4+jz+/vcdd8Upv+7d5VKyqd6H7s/Y35HIw/BIsWPl\nkahzGThQprzSy5MCmU+XA2T32u66n0p1BS59HMMjkYcXZKPYMTwSdS4DBwLPP5844to/CX++0Yqi\nzh2Z6nEMj0QehkeKHcMjUedywQWt7yst7fjliJOGwqiVx5KS9l0eonySt83WHG2duxgeiTqXI44A\nDjsMeP997758D48aGqNWHocNa9/laW8cbU1xytvwyNHWuYvhkajz0evAq3xvttaBMlErj/keHjna\nmuLEZmuKHcMjUeczcKDcatjK98qjTjWUqvJYWCi3/vBM1JUxPFLsGB6JOp//+A+51csV5nt4jFp5\n3LpVbhkeiTwMjxQ7hkeizufMMwFrvVHHnSU8pqo8btwot1p5JSKGR2oHDI9EnVdzs9x2lvCYqvKo\nFcdsXQmHKBfl7YAZyl35fP1XIkpOQ2MclyfMpqh9Hi+9FDjqKODgg9t/mYjyBQ/zFDtjsr0ERNRe\nHn4YeOGF/N/Oo1YeCwuByZPbf3mI8knehkfO80hE1PH23x8499xsL0XmovZ57Cw4zyPFKW/DI+d5\nJCKitop6hZnOgvM8Upw4YIaIiLocnb+xq1QeieLE8EhERF1Owd6jX1epPBLFieGRiIi6HA2PWoEk\nougYHomIqMvR8GhtdpeDKB8xPBIRUZejFceWluwuB1E+YnikdnP00dleAiKiYFp53LMnu8tBlI/y\ndqoeym11dRzFSES5a9w4ud1vv+wuB1E+ytvwyEnCc1tJSbaXgIgo3BlnAEuXdp3LDnKScIqTsXnW\nW9gYUw5gwYIFCzhJOBERURqcScIrrLU12V4eyk/s80hEREREkTE8EhEREVFkedvnkYiIiNrsEGNM\ntpeBctNGa+3KZA9geCQiIuoi1q5di4KCArS0tNyf7WWh3FRQULDTGDMuWYBkeCQiIuoi6urq0NLS\ngvvuuw+HHnpotheHcszixYtx3nnn9QQwCADDIxEREYlDDz2UM5ZQm3HADBERERFFxvBIRERERJEx\nPBIRERFRZAyPRERERBQZwyMRERERRcbwSERERESR5W14rKqqwrRp01BdXZ3tRSEiIspp1dXVmDZt\nGm699dZsL0qHmDNnDk4//XR0794d06ZNS/jdRx99hKlTp6KgoAATJkzAnXfemaWlbJsdO3bguuuu\nw3e+8x1MmTIF559/PtavX5/0b2bOnIlXX301tmUw1trYnqwjGGPKASxYsGAB56giIiJKw/3334/z\nzjsPXeEY+uqrr+LBBx/E7bffjtraWuy3334Jvz/nnHPwpz/9KUtL13ZXXnklvvOd72DYsGEAgFNO\nOQWrVq3CwoULUVRU1Orxzz33HE466SQ899xzmDJlStLnrqmpQUVFBQBUWGtrwh6Xt5VHIiIiojCv\nvfYarrnmGgwZMgR33313wu82bNiAI488MktL1na7du3CHXfcgTlz5uy77/LLL8fixYsxb968wMc/\n88wziPs65gyPRERE1Ok0NDSgb9+++PrXv445c+bAbWl96aWXcOyxx2Zx6dpmz549GDRoELZv377v\nvgMOOAAAsGzZslaPv+OOO3DppZci7lZmhkciIiLqdLTaNmPGDKxZswZ///vf9/3urbfe0ubZvFJc\nXIza2lrcfPPN++5bvnw5AGD06NEJj33vvfcwZMgQlJWVxb4cvLY1ERERBdqxA1iypH1f45BDgOLi\neJ/zvffew/jx4wEAY8aMwWc/+1n89re/xRlnnAEAaGpqQmFhYcrnaW5uRrduuR2VqqurMW7cOHzx\ni1/cd5+1Fvfff39CyIxTbn8iRERElDVLlgDtXaBbsACIe+zOiy++iC996Uv7fr7wwgtx/vnnY/Xq\n1SgpKUHfvn1b/c0DDzyARYsWYfjw4SguLkb37t3xyiuv4PbbbwcAvPDCC7jiiitwwAEH4OSTT0Zj\nYyPefvttXHHFFXj00Ufxwx/+ED/96U9xzjnnoKysDD/+8Y/xl7/8BVdddRWmT58OAGhpacFZZ52F\nXbt2AUCr5mStllprUVpamnJGmXfeeQcPP/wwnnzyyYTBMnPmzMGFF17Yhk8uGoZHIiIiCnTIIRLu\n2vs14rZ69WoMGTJk389nnXUWLr30Utx9992YPHkyJk+evO93LS0tuOCCC3Dcccfhuuuu23f/rbfe\nihEjRuz7ecqUKdizZw8uv/xyHHXUUQCAX//617j66qvx7W9/G8cddxy++93v7nv8V7/6VRx77LE4\n6aST9t1XUFCAhx56KJb32NDQgJkzZ+LBBx/EpEmTEt57Y2Mjxo4dG8vrBGF4JCIiokDFxfFXBbOh\nR48eOPfcczFnzhw0Nzfjyiuv3Pe7H/3oR+jTpw++8Y1vJPzN5MmTUVpauu/nbdu2YenSpQlTHK1c\nuRJ9+/bFM888kxASAeCVV17BV7/61XZ6R8A3v/lN3HLLLTj++OMBALW1tRg9ejTmz5+PN954AzNm\nzACAfVXOWbNmYd68ebjlllsyfm2GRyIiIuo0VqxYgZEjR7a6/6KLLsKvf/1rvPLKK+jduzcAYNOm\nTbjtttvw4Ycftnr8Zz7zmYSfX3rpJUycOHFfH8h169bhhRdewP/+7//i/PPPx6xZsxIev2PHDvTo\n0SPhPn+zdZhUzdY/+clPcP755+8LjitXrsTzzz+P0aNH46KLLsJFF12U8HlUV1fjyiuv3Pf4TDE8\nEhERUafx8MMP4+ijj251/4QJE1BeXo4JEybsu+/FF1/E6NGjE5qn//a3v+Hpp59GU1MTLr/88n3N\nv88//zwGDRqEefPmYefOndi+fTueeOIJFBUVYdGiRfuaslVBQesJbeJotv7zn/+MZ599Ft26dcOC\nvX0K3n//fVx88cWBj29ubgYg0/zEheGRiIiI8l5NTQ1++MMf4rnnnsOYMWNw9dVX47zzzkt4zMyZ\nMxOmrikoKED//v0THnPmmWfiwQcfxIknnpjQb/D555/HNddcg1NPPTXh8W+99RY+9alPJYTFl19+\nOaFfZVw2b96MGTNmYOfOnXj22Wf33W+MwW233dbq8T/5yU/w5z//GcYYzJw5EyeffDJ+9atfZbwc\nDI9ERESU98rLy/HEE08kfczMmTMTfj7ppJNQVVWFFStW7JtsWyuKV1999b7HNTQ04N1338UJJ5zQ\n6jnLysrQvXv3fT83NjbizTffTBg8E5cBAwYkTBCeylVXXYWrrroq9uVgeCQiIqIuqU+fPnjkkUdw\nzTXXYMKECRg4cCCampowa9YsHHTQQQCkovn73/8eJSUleOyxx3DWWWclPMfIkSNRWVmJm2++eV9V\n85JLLunw99KRGB6JiIioyxo/fjzuueee0N+Xl5ejvLwcd9xxR+hjzj///PZYtJzFyxMSERERUWRZ\nD4/GmBJjzJvGmBpjzD+NMRel/isiIiIiyoash0cAWwEcb60tB3A0gP82xvRP8TedSqrLD+WTzvRe\nAL6fXNaZ3gvA95PLOtN7IYpD1sOjFTv3/thr763J1vJkQ2faMXWm9wLw/eSyzvReAL6fXNaZ3gtR\nHLIeHoF9TdcLAawE8HNr7eZsLxMRERERtZZ2eDTGHG+MmWeM+dgY02KMmRbwmEuMMbXGmEZjzGvG\nmCOTPae1tt5aOxHAaADnGmMGp7tcRERERNT+2lJ57A1gIYBvAbD+XxpjpgO4FcC1AI4A8A6Ax40x\ng5zHfMsY8/beQTL7Lvxord2w9/HxXHyRiIiIiGKV9jyP1tr5AOYDgDEmqG9iFYC7rLX37n3MxQC+\nAGAGgJ/tfY47Ady59/dlxpgd1toGY0wJgCn6uxA9AWDx4sXpLnrOqq+vR01NTbYXIxad6b0AfD+5\nrDO9F4DvJ5d1pvdSW1sLoHMdQyk+UdcLY22r4mFkxpgWAF+01s7b+3MRgB0AztL79t4/F0CJtfbf\nA57jSAC/1R8B3GGtvTvJa54D4P42LzQREVEXVlBQgJaWlmwvBuWogoKCnS0tLeOstSvDHhP3FWYG\nASgEsN53/3oA44L+wFr7JqR5O6rHAZwLYDmAnckfSkRERI6eLS0tFQDeA1Cf7YWh3NPS0rIxWXAE\n8vDyhNbaTQD+lO3lICIiylOvZHsBKL/FPVXPRgB7AAzx3T8EwLqYX4uIiIiIOlis4dFa2wRgAYCp\net/eQTVTwTMdIiIioryXdrO1MaY3gLHwrgIzxhgzAcBma+0qALcBmGuMWQDgDcjo62IAc2NZYiIi\nIiLKmrRHWxtjTgDwLFrP8XiPtXbG3sd8C8AVkObqhQC+ba19K/PFJSIiIqJsSrvZ2lr7vLX/3965\nB2lVl3H880UUzYaoQdpM8i5RcqlFExPTMCxDiZpRphmdxmGyi2E1jZcuM4VdkD82VNwuWk2i3XQS\nR6OciKYxRIxwgNDQWiochIZEro0QPP3x/BbOHt7dfaF33z2/7fnM/Gb2nPO873m+79nz/J7zO7+L\nDTKzo0rl2oJNu5mdYmbHmdnERiWOh7tyTTOQdIukpyRtl7RZ0kOSzqphN1vSRkm7Jf1a0hml40Mk\n3SVpi6Qdkh6UNKJk81pJ90vaJmmrpHtSS3Bfabs5rSLUlqsWSSdKWpB82S1plaS356hH0iBJt0rq\nSL7+RdIXa9hVUo/qW52qKb5LGinpF5J2Sdokaa6kuuNhT1okDZZ0m6TVknYmmx9KekMVtfSmp4bt\nt5PNrJz1SBot6WFJL6frtFzSSVXT05sWScdLmi9pQ7pv1kq6rmRTCS3BAMLMsijAVfjUPNcAbwa+\nA7wEDO9nvxYBVwOjgTHAo/g0QscVbG5Kvk4FzgYWAn8FjinYfCt97l341EVPAI+XzvVLYCUwATgf\neA64r490nQN0AE8DbTlqAYYB64F7gFbgZOAS4NRM9Xwe+CfwXuBNwAeB7cD1OehJfs8GpuED664o\nHW+K7/hD8xp82q8xwKXpd/1qI7QAQ9N3fwg4EzgXeBJ4qvQdldBSz7Up2E3HY8IGYFaueoDT8QGe\n3wDG4kvjTqVQn1RFTx1avpvOOwmPCzOBvcDUqmmJMnBKvztQt6MefG8vbAt4Abixv30r+Tkc2A9c\nUNi3EfhMYXso8G/gysL2K8D0gs2o9D3npu3RafttBZtLgf8ALQ3W8GpgHfBuvItCW45agDnA73qx\nyUnPI8DdpX0PAvfmpid9f7kSbIrvwPvwyrWYKFwHbAUGN0JLDZsJeMV/UpW19KQHeCPwj+TXegrJ\nY256gB/jXa26+0wl9XSjZQ3whdK+FcDsKmuJknfJorlZvnJNK/Cbzn3m/7mLgYn95Vc3DMP7g74E\nIOlUoIWuvm8HlnPQ9wn44KWizTo8UHfanAdsNbOnC+danM71jgZruAt4xMyWFHdmqOVyYIWkn8m7\nFKyUNDNjPU8AkyWdmfwfB7wTb/3OUc8Bmuz7ecAaM9tSsHkMeA3w1gZJKtMZF15O261kpEWSgHuB\nuWZWa/2ybPQkLe8Hnpf0qxQbnpQ0LUc9eFy4QtKJSd/FeIv3YxlqCTIhi+SRnleuaWm+O7VJQWke\n8HszeybtbsFvwJ58fz2wJ1WW3dm04K8IDmBm+/AktWG/gaQZwHjglhqHs9ICnAZ8HG9FnYK/urlD\n0tUFP3LSMwf4KfBnSXvwabHmmdlPCn7kpKdIM31v6eY80Af6JA3Br92PzGxn4Tw5abkZ93d+N8dz\n0jMCf7tyE/7g9R7gIeDnkiYVzpWLnk8BzwIvpLiwCPikmS0tnCcXLUEmZLfCTMVpB96CtwZlR+os\nPg+4xHzOztwZhPcz+1LaXiXpbOBjwIL+c+uIuQr4MDADeAZP8m+XtNHMctQz4JE0GHgAT4w/0c/u\nHBGSWoFZHN4yslWms9FkoZndkf5eLel8PDY83j9uHTGz8NbBqXhr4oVAe4oLS3r8ZBAcIbm0PFZ+\n5RpJ84HLgIvM7MXCoU14/8yefN8EHCNpaC825dFxRwGvo3G/QStwArBS0l5Je/EO1jekJ9rNGWkB\neBF/Ii/yLN6pvNOPnPTMBeaY2QNmttbM7ge+ycFW4tz0FGmm75u6OQ80UF8hcRwJTCm0OnaeJxct\nF+BxYUMhLpwMtEnqKJwrFz1b8L58vcWGyuuRdCzwNeCzZrbIzP5kZu34G4rP5aQlyIsskker+Mo1\nKXGcBlxspcXEzWw9fmMVfR+KPyl2+v5HPJgVbUbhgWxZ2rUMGCap+PQ/Ga9wlzdIymJ8FN14YFwq\nK4D7gHFm1pGRFoCleMfwIqOAv0N21wZ8sv19pX37SfdxhnoO0GTflwFjJA0v2EwBtuEtuv8zhcTx\nNGCymW0tmWSjBe/rOJaDMWEcPrhpLj6oIis9qT75A4fGhrNIsYF89BydSjku7ONg/Z6LliAn+nvE\nTr0FuBLYTdepev4FnNDPfrXjo80m4U9hneXYgs2NydfL8eRsIfA8XacgacdHMF6EtwAu5dCpFBbh\nydw5+KvxdcCCPtZXHm2djRZ8gMUreMvc6fgr3x3AjEz1/AB/LXUZ3vIzHe+n9PUc9ADH44nHeDzp\n/XTaHtlM3/FKdRU+NclYPAHaDNzaCC14d6CH8URkDF3jwtFV01LPtalh32W0dW56gA/gU7/NxGPD\n9cAeYGLV9NSh5bfAavwt0SnAR/C68qNV0xJl4JR+d+CwnPU+Q3/Dp+9YBkyogE/78ae8crmmZPdl\n/Gl9Nz5C7YzS8SHAnfgrlR14q8WIks0wvBVwG56w3g28qo/1LaGQPOamBU+0Vidf1wLX1rDJMhqF\n5wAAAPNJREFUQk+qRNpSJbALT6y+QmmajKrqwSu3WvfL95vtO57kPQrsxCvA24BBjdCCJ/blY53b\nF1ZNS73XpmTfwaHJY1Z68CTrOfxeWklhXsQq6elNC/66+Xv43Ju78FbAG6qoJcrAKYe9PGEQBEEQ\nBEHw/0sWfR6DIAiCIAiCahDJYxAEQRAEQVA3kTwGQRAEQRAEdRPJYxAEQRAEQVA3kTwGQRAEQRAE\ndRPJYxAEQRAEQVA3kTwGQRAEQRAEdRPJYxAEQRAEQVA3kTwGQRAEQRAEdRPJYxAEQRAEQVA3kTwG\nQRAEQRAEdRPJYxAEQRAEQVA3/wXdy3oPIUftdAAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAGPCAYAAACUIDA3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3XuYXXV97/H3N8jdGIvhrpKIStMqlxmRwy1QQMAqNxXp\nFFTqpQVENHhpe6QHa9VjQUGs9/YoWnWsFRC0IqIBQ7nqDBGqqYAQLhEooIaAIJF8zx9rTdiZW3Z+\ne+/s2Zn363nmmdlr/fb6fTNPdviw1u8SmYkkSVKvmNHtAiRJktaF4UWSJPUUw4skSeophhdJktRT\nDC+SJKmnGF4kSVJPMbxIkqSeYniRJEk9xfAiSZJ6iuFFkiT1FMOLJEnqKdMqvETEsyPiioj4aUQs\njojXdLsmSZK0bmI6bcwYEdsB22TmTRGxLTAEvCAzH+tyaZIkqUnT6s5LZt6XmTfVP98PPAhs1d2q\nJEnSunhauy8YEYcDLwLuBi7MzJXt7qMdIqIfmJGZy7pdiyRJal7RnZeIOCUibo+IfUcd/zrwH8A/\nAl8FroqIzUqLi4j9I+KSiFgWEasi4shx2rw1Iu6IiMci4rqI2LOJ624FfBF4S2ltkiSpO0ofGx0D\nbAFcO3KgvuPyGmAZ8GHgBmBPWgsIWwKLgVOAMYNzIuI44KPAmcAewE+AyyJidkObUyLixogYjohN\nI2IT4CLgQ5l5fQu1SZKkLigasBsRdwG3ZOYhDcfOB14H7J2ZN9R3XO4ElmbmXi0XGrEKODozL2k4\ndh1wfWa+vX4dVI+rPp6ZZ01wnUFgSWa+v4k+nwUcBiwFHm/1zyBJ0jSyGTAHuCwzH2rnhUvHvGwN\nLBp17ADg7sy8ASAzH4+Ia4B9R7+5HSJiY6Af+NDIsczMiPg+sPcE79kXOBa4KSKOobqb87rM/OkE\n3RwGfKWthUuSNL0cTzWUpG1Kw8tyoPHRzFxgJ+BLo9o9SvXopxNmAxsB9486fj+wy3hvyMyrWbc/\n81KAL3/5y8ybN6+gxA3XggULOPfcc7tdxqS6UWOn+mzndVu5Vul71/V9zbbvhb+H3dALv5cN6fPZ\nzmu3ep2S93fq87lkyRJOOOEEqP9b2k6l4eU2YH5EPDcz7wL+kuouxndHtXs2cF8L9XXb4wDz5s2j\nr6+v27VMKbNmzZryv5Nu1NipPtt53VauVfredX1fs+174e9hN/TC72VD+ny289qtXqfk/Z36fDZo\n+7CL0gG7n6Z6lnVTRAwB7wEeAL490iAiNgdeAvys1SIn8CDwJLDtqOPb0tuBqScMDAx0u4S16kaN\nneqznddt5Vql713X9/XC36+prBd+fxvS57Od1271OiXv78XPZ/EKuxFxNnAqsClwD/CGzLyi4fwb\ngC8Ap2fmx1outPkBu3dRDdg9uw199gFDQ0NDU/7/YqTp6Mgjj+SSSy5Ze0NJ693w8DD9/f0A/Zk5\n3M5rFy9Sl5nvjogzgGdk5gPjNFlINX35F6V9RMSWwPOBqA89LyJ2A36VmXcD5wDn13d/bgAWUE3h\nPr+0T0mSNLW1tMJuZv6O6nHReOfuppq23IqXAFdQjadJqjVdoFpg7o2Z+fV6TZf3Uz0uWgwcNkGY\nkrSBmQq3ryWtf23fHqCdMvOHrGVcTmZ+CvjU+qlI0lRieJGmp+LwEhHbUK18Ox/Ynmrsy3gyM3cu\n7UeSJKlRUXiJiHnAD4Fn8dR4FEmSpI4rnSp9NtUicRdSjUt5RmbOmOirbdVKkqRpr/Sx0f7Az4HX\nZulca0mSpAKld0UCWGxwkSRJ61tpePkx1V5GkiRJ61XpY6P3AT+IiCMy81ttrGdKWrBgAbNmzWJg\nYMCpmZIkTWJwcJDBwUGWL1/esT6a2h4gIuaPc/hI4O1U21xfTrVFwKrx3p+Zi1qosWvcHkCSpDJT\nYXuAK6lWuB0tgNcBJ6zl/RutQ02SJEkTaja8fInxw4skSdJ61VR4ycwTO1yHJElSU1xATpIk9ZSi\n8BIRT4+IXesdnSdqM7tus2V5eZIkSWsqvfNyOnAjMNmGizvXbd5e2IckSdIYpeHlCOC2zLx+ogb1\nuV8ARxf2IUmSNEZpeHke8N9NtFsCzC3sQ5IkaYzS8LI58FgT7R4Dnl7YhyRJ0hil4eVuYM8m2u0J\n/LKwD0mSpDFKw8tlwJyIWDBRg4h4O9Ujo+8W9iFJkjRG6caMZ1FtC/CRiDgY+BzV4FyoZhn9JfBy\n4OG6rSRJUlsUhZfMvCcijgQuAP6UKqg0CuBB4NjMvLO1EiVJkp5SeueFzLwqInYB3gIcDDynPnU3\n8H3gXzLz162XKEmS9JTIdL/FiUREHzA0f/58Zs2axcDAAAMDA90uS5KkKWtwcJDBwUGWL1/OokWL\nAPozc7idfRheJjESXoaGhujr6+t2OZIk9Yzh4WH6+/uhA+Gl+LERQERsC7wR2B/YsT68DFgEfCEz\n72+tPEmSpDUVh5eIeDXweapF6KLh1IuBw4C/iYg3ZeYFrZUoSZL0lNJdpV8CDAJbAhcBxwB7ALtT\n7WV0IVWo+WrdVpIkqS1K77z8LbAR8JrMvGjUuZuASyLiGKqp1H8DvKa8REmSpKeUrrC7H3DNOMFl\ntfrc1VTjYSRJktqiNLzMAu5qot1ddVtJkqS2KA0v91GNcVmb3eu2kiRJbdHKxoy7RMSHImKj0Sej\n8gHgD3FjRkmS1EalA3b/AXgV8NfAQER8HVhan9sJOBaYAzwEfKC1EiVJkp7SysaMBwFfAV4EvBsY\nWap3ZM2Xm4HjM/OelquUJEmqtbIx483ArhFxINWMoh3qU78ErsrMK1uuTpIkaZSWtgcAqEPKlS1X\nIkmS1ISWw8uIiPiD+sffpLs9SpKkDimdbQRARBwZEd+LiEeAB+uvFfWxo9pSoSRJUoPSvY0iIj5P\nta/RIcAWwPL6a4v62IURcX5ExMRXkiRJWjeld17eDpwI3AucDDwzM7fKzK2oVtQ9qT73urqtJElS\nW0TJ8JSI+BnwXODFmXnHBG3mUk2Xvisz/6ilKrskIvqAofnz5zNr1iwGBgYYGBjodlmSJE1Zg4OD\nDA4Osnz5chYtWgTQn5nD7eyjNLw8BnwvMycd1xIRFwOHZubmhfV11Uh4GRoaoq+vr9vlSJLUM4aH\nh+nv74cOhJfSx0YPAE800W4l1SBeSZKktigNLxcBBzVMjx4jIrYCDgK+WdiHJEnSGKXh5QzgdmBh\nvU3AGiLiT4DLgV8A/7u8PEmSpDWVLlJ3MdVjo37g8oj4FXBnfe65wLPqn68DLh41Wzoz8+DCfiVJ\n0jRXGl4ObPg5qMLKs8Zpt/c4x1x9V5IkFSsNL3PbWoUkSVKTisJLZt659laSJEnt19LeRpIkSetb\nqxsz/lFEnBsRV0fEzyPirIZz+0TEafWUaUmSpLYoHfNCRJwOfLjhGgnMHtXsXOB3wGdL+5EkSWpU\nuqv0K4CPAHcDrwK2oZp1tFpmXkO1Eu+kWwhIkiSti9I7L6cDjwIvy8zbAUat5TJiMbBLYR+SJElj\nlI556QeuGwkuk3gQ2K6wD0mSpDFKw8smwIom2m0D/L6wD0mSpDFKw8sdwG6TNYiITYBdgVsK+5Ak\nSRqjNLxcAsypZxxN5D3A1sCFhX1IkiSNUTpg9yzgeODsiNgLuKg+vm1EHAMcU5+/A/hEy1VKkiTV\nSrcH+HVEHAJ8AzgWeE196vD6K4CfAUdnZjNjYyRJkppSvEhdZt4SEbsDRwCHAnOoHkPdA1wOXJCZ\nT7ajSEmSpBHF4QUgM1cBF9dfG6wFCxYwa9YsBgYGGBgY6HY5kiRNWYODgwwODrJ8+fKO9RGZ2bGL\n97qI6AOGhoaG6Ovr63Y5kiT1jOHhYfr7+wH6M3O4ndd2V2lJktRTDC+SJKmnGF4kSVJPMbxIkqSe\nYniRpA2MEzG0oTO8SNIGYMWKFZx52mkcMncuRz/nORwydy5nnnYaK1a4Tqg2PC2t8yJJ6r4VK1bw\n6r335vQlS3jfqlUEkMBln/wkr164kAuuvZaZM2d2u0ypbdbpzktEbBQR+0fEcRFxQERsvJb2+0TE\n61srUZI0mY+8972cvmQJh9fBBao9Wg5ftYoFS5bw0TPO6GZ5Uts1HV4iYh/gNuBK4KvAQuDuiHjL\nJG97C/CFVgqUJE3u6m99i8NWrRr33OGrVnH1JZes54qkzmoqvETEHOBSYCfgYeDHwAPANsBnIuIb\nEbFph2qUJE0gM9ly5crVd1xGC2CLlSsdxKv1ZmT81UmvfGXH+mj2zstfAzOp7qJsn5l7AdsDrwMe\nAo4BLouIZ3SkSknSuCKCRzfemImiSQKPbrwxERPFG6l9RsZf7f3JT/Lpe+/tWD/NhpeXUe0WfVJm\nPg6Qla8A/cAwMB9YGBGzO1KpJGlc+x5xBJfNGP+f8+/OmMF+Rx65nivSdDXe+KtOaDa8PBu4ITNX\njj6RmXcDBwDfBfqARRGxY/tKlCRN5l0f/CDnzJvHpTNmrL4Dk8ClM2Zw7rx5vPMDH+hmeZpGJht/\n1U7NhpeHgY0mOpmZvwWOBL4G/CFwVUTs3Hp5kqS1mTlzJhdcey3Xn3oqh86Zw1E77sihc+Zw/amn\nOk1a683axl+1U7PrvNwJvHSyBpn5+4g4Hvg1cDKwCFjaUnWSpKbMnDmT9513Hpx3HpnpGBetd43j\nrzr9t6/ZOy83ANtFxAGTNarHwbwV+BDVgN7/1WJ9kqR1ZHBRt0w2/qqdmu3hIqog9bfNNM7MM4B3\nlhYlSZJ6z3jjrzqh2fDyA+AFwCnNXjgzz6UawHtQQV2SJKnHNI6/OmX77TvWT7hw0cQiog8YGhoa\noq+vr9vlSJLUM4aHh+nv7wfoz8zhdl7bXaUlSVJPMbxIkqSeYniRJEk9pdl1Xqa1BQsWMGvWLAYG\nBhgYGOh2OZIkTVmDg4MMDg6yfPnyjvXhgN1JOGBXkqQyDtiVJEmqFYWXiDgnIv6u3cVIkiStTemd\nl1OBXdtZiCRJUjNKw8s9LbxXkiSpWGkA+SZwQES4z7okSVqvSsPLmcBdwHciYo821iNJkjSp0nVe\nLgZ+B+wL/Dgi7qUKM4+P0zYz8+DCfiRJktZQGl4ObPg5gB3qr/G4kIwkSWqb0vAyt61VSJIkNako\nvGTmne0uRJIkqRlOd5YkST2lpfASEX8UEedGxNUR8fOIOKvh3D4RcVpEbNV6mZIkSZXiXaUj4nTg\nww3XSGD2qGbnUs1K+mxpP5IkSY1K9zZ6BfAR4G7gVcA2VLOOVsvMa4AHgKNarFGSJGm10jsvpwOP\nAi/LzNsBImK8douBXQr7kCRJGqN0zEs/cN1IcJnEg8B2hX1IPSfTZY0kqdNKw8smwIom2m0D/L6w\nD6knrFixgjNPO41D5s7l6Oc8h0PmzuXM005jxYpmPiKSpHVV+tjoDmC3yRpExCbArsAthX1IU96K\nFSt49d57c/qSJbxv1SqCauT6ZZ/8JK9euJALrr2WmTPdv1SS2qn0zsslwJx6xtFE3gNsDVxY2Ic0\n5X3kve/l9CVLOLwOLlCNXD981SoWLFnCR884o5vlSdIGqfTOy1nA8cDZEbEXcFF9fNuIOAY4pj5/\nB/CJlqvssiVLul2BpqpLv3E7R67ajeFxzm2zCi79919w5BvWe1mS1HWd/G9nlA4wjIgXAt8AXkR1\np3zkjjn1zz8Djs7M29pQZ1dERB8wBENAX7fLkSSphwxTze+hPzPH+3+8YsWL1GXmLRGxO3AEcCgw\nh+ox1D3A5cAFmflkO4rsti9/GebN63YVmopOeuUr+fS9v2S8hQISOHn7HfjMt7+9vsuSpK5bsgRO\nOKEz1y4OLwCZuQq4uP7aYM2bB33eeNE4Xv6a5/HAJy/l8FWrxpy7dMYM/vTY/f27I0ltVrrC7ucj\n4o1NtDsxIj5f0ofUC971wQ9yzrx5XDpjxupnpkkVXM6dN493fuAD3SxPkjZIpbONTgT2a6LdvoDD\nFbXBmjlzJhdcey3Xn3oqh86Zw1E77sihc+Zw/amnOk1akjqkpcdGTdgE2CDGvUgTmTlzJu877zw4\n7zwyc6KtMiRJbVJ652WtovoXvI9qc0ZpWjC4SFLnNX3nJSIWjjp0+DjHGq+7M9W+Rv9aWJskSdIY\n6/LY6MCGn5MqmEy26eJK4NvAu9a9rKllwYIFzJo1i4GBAQYGBrpdjiRJU9bg4CCDg4MsX768Y300\nvUhdROw08iNwO9UCde+eoPkTwIOZubLlCrtoZJG6oaEh+pzvKklS04aHh+nv7+IidRHxeuC2zLym\nfv33wI2ZeWc7i5EkSVqbZgfsng+8ueH1iTQ3VVqSJKmtmg0vq1jzLs1OwOz2lyNJkjS5ZsPL/wAv\n7mQhkiRJzWh2ttH3gRMi4hfAyDiXyaZKN8rMPLioOkmSpFGaDS+nA88EXg7Mpbmp0iOam84kSZLU\nhKbCS2Y+CBwZERsD2wNLmXyqtCRJUkes095G9botd0XEXcBSp0pLkqT1rWhjxsyc0+Y6JEmSmtKx\njRklSZI6oejOy4iI2A84CngBMJNq64DRnG0kSZLapii8REQA/w94A08FlmTN8DLy2tlGkiSpbUof\nG51EtUXAEPAy4ML6+C5U06nPp1qV92zgeS1VKEmS1KD0sdGJwKPAyzPzoYg4ASAzbwVuBS6LiO8A\n/wZcw1ML20mSJLWk9M7LPOCazHyofp0AEbHRSIPM/AbVnZl3tVShJElSg9LwMgN4qOH1b+vvfzCq\n3a24J5IkSWqj0vCyDNih4fXIY6E9RrV7IfD7wj4kSZLGKA0vw8AfNTwm+h7VzKKzIuIPI2JmRLwb\n6AdubEOdkiRJQHl4uQSYDbwCIDN/AnwN2A34KfAb4MNUd13e23qZkiRJldLtAQYj4kLWfCT0BuAm\n4GiqsS+3AGdl5g0tVylJklQrXmE3M3836vVKqrstH261KEmSpIm4t5EkSeophhdJktRTDC+SJKmn\nGF4kSVJPMbxIkqSeYniRJEk9xfAiSZJ6StvDS0TMioiXRMR27b62JElSUXiJiEMj4vMRsceo428D\n7gOuB+6JiHPbUKMkSdJqpXde3gwcC9w6ciAiXgx8DNgIuA54GDgtIo5qtUhJkqQRpeGlD1icmY80\nHDtx5Htm7ku1o/QTwCnl5UmSJK2pNLxsC9wz6tjBVLtJfw0gM+8AfgjMK65OkiRplNLw8iSw2ciL\niNgKeBFwVWauamj3ALB1eXmSJElrKg0vS4F9ImLj+vWrgAAuH9XuWcBDhX1IkiSNURpe/o3qjsqi\niPgocBawEvjmSIOICKpxL7e3WqQkSdKIpxW+7+PAUcBe9dcq4N2ZuayhzUFUAeczLVUoSZLUoCi8\nZOajEbEPMJ8qoCzOzFtHNXsSWAB8q7USu2/BggXMmjWLgYEBBgYGul2OJElT1uDgIIODgyxfvrxj\nfURmduzivS4i+oChoaEh+vr6ul2OJEk9Y3h4mP7+foD+zBxu57VLHxtNKCLmAX8M3J2Z17f7+pIk\naXor3R7guIhYGBF7jTp+NvBfVAN6r4mIiyJiozbUKUmSBJTPNjoB2B24ceRAPQbmncAKqoXqlgJH\nAse3VqIkSdJTSsPLi4CbMvOJhmOvAxJ4bWYeD+wJPEK1D5IkSVJblIaXbYBlo479CfA/mfk9gMz8\nFbAIeH55eZIkSWsqDS+PAc8YeRER2wMvpNrLqNFvgD8o7EOSJGmM0vByO7B/RDyzfn081SOj741q\ntx3wP4V9SJIkjVEaXs6nuvMyFBEXAB+gGt9y8UiDet+jlwC3tFijJEnSaqXrvPwz1RiXVwNzgUeB\nv8rMxk0YXwnMAha2VKEkSVKD0u0BVgLHRsQcqu0B/jszV4xqdgdwDHBdKwVKkiQ1ammF3cxcSrWe\ny3jnFgOLW7m+JEnSaC1vDxARm1AtWLdjfWgZ1UaNT0z8LkmSpDLF4SUiNgPeD/wV8PRRpx+JiM8A\nZ2bm4y3UJ0mStIai8BIRmwLfB/auD91E9fgogTnAbsC7gH0j4uDM/F3LlUqSJFE+VXoBsA9wNbB7\nZu6Rmcdk5qsys48qvFxFFW7e0Z5SJUmSysPLAPAA8IrMvHn0ycz8L6qp0g/ixoySJKmNSsPL84Er\nx5kevVpmPgJcCexc2IckSdIYpeHl98AWTbTbom4rSZLUFqXh5WbgoIh43kQNImIucBDVYF5JkqS2\nKA0vnwU2B66MiDdFxOYjJyJi84j4C6pHRpsBn2m5SkmSpFrp9gD/GhH7AW8BPgd8LiIerE/Prr8H\n8NnM/ErrZUqSJFVK77yQmX8FHAv8J7CSao+jreufrwKOzcyT21GkJEnSiFb3NroAuCAingY8qz78\nUGY6SFeSJHVEy3sbAdRh5f52XEuSJGkyxY+NJEmSuqGpOy8RsbCFPjIzD27h/ZIkSas1+9jowBb6\nyBbeK0mStIZmw8vcjlYhSZLUpKbCS2be2elCJEmSmuGAXUmS1FMML5IkqacYXiRJUk8xvEiSpJ5i\neJEkST3F8CJJknqK4UWSJPWUovASEQsj4j1NtHtXi1sLSJIkraF0V+kDgaVNtNsFOKCwD0mSpDE6\n/dhoM+D3He5DkiRNIx0LLxHxDGAf4N5O9SFJkqafph8bRcTtow69JiIOnOS629bfP1FWmiRJ0ljr\nMuZlTsPPCTy9/hrPSuCXwCXA3xZVJkmSNI6mw0tmrn7EFBGrgPMz840dqUqSJGkCpbON/gK4rZ2F\nSJIkNaMovGTmF9tdiCRJUjNamm0UEYdGxEURsSwifhcR/6/h3GERcU5E7NB6mZIkSZXi8BIR5wGX\nAkcBM4GNgWhoci/wDuC4VgqUJElqVLo9wOuBtwFDQF9mPmN0m8y8CbgbOKKlCiVJkhqUDtg9GfgN\n8IrMfGCSdjcBLy7sQ5IkaYzSx0YvAq5ZS3ABWE61WJ0kSVJbtDJgN5toswPwWAt9SJIkraE0vNwK\n9EXExhM1iIiZwO7ATwv7kCRJGqM0vPw7sD3w4Una/F9gFvC1wj4kSZLGKB2w+zHgz4B3RMQ+wMX1\n8Z0jYgFwDLAfMAz8c8tVdtmCBQuYNWsWAwMDDAwMdLscSZKmrMHBQQYHB1m+fHnH+ojMZoaujPPG\niK2B84GXU41/iVFNLgdOaGJQ75QVEX3A0NDQEH19fd0uR5KknjE8PEx/fz9Af2YOt/PapXdeqEPJ\nKyJiN+BQql2nZwD3AJdn5g1tqVCSJKlBcXgZkZk/AX7ShlokSZLWqqW9jcYTEbMi4iURsV27ry1J\nklS6PcChEfH5iNhj1PHTgPuA64F7IuLcNtQoSZK0WumdlzcDx1Kt9wJARLwYOBfYCLgOeBg4LSKO\narVISZKkEaXhpQ9YnJmPNBw7ceR7Zu4L9ANPAKeUlydJkrSm0vCyLdWsokYHU23W+DWAzLwD+CEw\nr7g6SZKkUUrDy5PAZiMvImIrqs0ar8rMVQ3tHgC2Li9PkiRpTaXhZSmwT8PeRq+iWqTu8lHtngU8\nVNiHJEnSGKXh5d+o7qgsioiPAmcBK4FvjjSIiKAa93J7q0VKkiSNKF2k7uPAUcBe9dcq4N2Zuayh\nzUFUAeczLVUoSZLUoCi8ZOaj9YaM86kCyuLMvHVUsyeBBcC3WitRkiTpKa3sbbQKuHKS81dOdl6S\nJKlE27cHkCRJ6iTDiyRJ6imGF0mS1FMML5IkqacYXiRJUk8xvEiSpJ5ieJEkST3F8CJJknpK0SJ1\nEdHsfkVPUG3MeCPwlcy8tqQ/SZKkEaUr7M5Zx/Z7AydHxD9l5jsK+5QkSSp7bJSZM4DzgOXAB4Fd\ngWfWXy8GPgD8Bvgn4LnAnwO/BN4WEX/WetmSJGm6Kn1sdApwMvC/MnPxqNM/Bf5PRFwA3ADcmpmf\niIj/Bn4MvBn4Wgs1S5Kkaax0wO7JwJXjBJfVMvMnwELgpPr1YmAI2KOwT0mSpOLwsjPw6yba/QZ4\nXsPrO4CnF/YpSZJUHF5+DewXEZtM1KA+tx9rhpxnUAUaSZKkIqXh5RJgR+BrEfHc0Scj4tnAILAD\ncHHDqXlAs9OsJUmSxiidKn0GcDBwNPDKiPgRcHd97jnAS4CNgduAvwOIiD5gO+DzrRQsSZKmt6Lw\nkpkPRcTewD9STYPeu/4a8TjwJeBvMvOh+j3DwOatlStJkqa70jsv1KHkzRHxdqAf2L4+dS8wlJmP\ntqE+SZKkNRSHlxF1SFnUhlokSZLWyo0ZJUlSTym+8xIRmwIDwHyqR0abTtA0M/Pg0n4kSZIalW4P\nsCPwA+AFQKyleZb0IUmSNJ7SOy9nAy8ErgHOAW4BVrSrKEmSpImUhpfDgLuAQzLz8TbWI0mSNKnS\nAbubAtcbXCRJ0vpWGl5uBma3sxBJkqRmlIaXfwTmR8RL21mMJEnS2pSOeRmmGqj7g4g4B7gcuAdY\nNV7jzLyrsB9JkqQ1lIaXpVRToINqk8YzJmmbLfQjSZK0htJQsQjXb5EkSV1Quqv0gW2uQ5IkqSnu\nbSRJknqK4UWSJPWUph4bRcTr6x8vyswVDa+bkplfWufKJEmSxtHsmJfzqQboXke1h9HI67WJup3h\nRZIktUWz4eX9VCHkwVGvJUmS1qumwktmvm+y15IkSeuLA3YlSVJPMbxIkqSeUrxsf0RsCgwA84Ht\ngU0naJqZeXBpP5IkSY2KwktE7Aj8AHgB1YyiyTiwV5IktU3pnZezgRcC11DtLn0L1RRqSZKkjioN\nL4cBdwGHZObjbaxHkiRpUqUDdjcFrje4SJKk9a00vNwMzG5nIZIkSc0oDS//CMyPiJe2sxhJkqS1\nKR3zMkw1UPcHEXEOcDlwD7BqvMaZeVdhP5IkSWsoDS9LqaZAB3BG/TWRbKEfSZKkNZSGikW4fosk\nSeqCovCSmQe2uQ5JkqSmuLeRJEnqKYYXSZLUU5p6bBQRr69/vCgzVzS8bkpmfmmdK5MkSRpHs2Ne\nzqcaoHsd1R5GI6/XJup2hhdJktQWzYaX91OFkAdHvZYkSVqvmgovmfm+yV5LkiStLw7YlSRJPcXw\nIkmSekpLy/ZHxHOBI4AXADOpBuiOlpn5plb6kSRJGlEcXiLi/wB/x5p3b0bCSza8TsDwIkmS2qLo\nsVFEHAfgIxIOAAAMuUlEQVS8D7gb+EuqXaUBDgNOBn5IFVzOAQ5quUpJkqRa6Z2XU4AngD/JzDsj\nYj+AzBwJMZ+NiAXAWcA3Wy9TkiSpUjpgd1fgmsy8s36dABGxesxLZp4L/Bw4o6UKJUmSGpSGl02B\n+xpeP15/f+aodj8B9izsQ5IkaYzS8HIvsE3D62X19z8e1e7ZwEaFfUiSJI1RGl5uBnZpeH0l1QDd\nv4+ILQEi4rXA/sBPWylQkiSpUWl4+RawY0QcBJCZVwNXAH8C/DoiHgQGqcbC/EM7CpUkSYLy8PJl\nYB6wuOHYMcDngF9RLVj3M+B1mfndliqUJElqUDRVOjN/RzWTqPHYw8BJ9ZckSVJHlC5Sd2FEfLLd\nxUiSJK1N6WOjPwWe1c5C1oeImBURP4qI4Yi4KSLe3O2aJEnSuikNL3cAW7azkPXkYWD/zOwD9gL+\nd0T8QZdrklRocHCw2yVI6oLS8DIIHBAR27WzmE7LysiCepvX38fbCVtSDzC8SNNTaXj5v8BVwA8j\n4piI2LiNNXVU/ehoMXAXcHZm/qrbNUmSpOaVhpefU62m+3zgG8BjEfHLiLh9nK9flBYXEftHxCUR\nsSwiVkXEkeO0eWtE3BERj0XEdREx6XYEmbk8M3cH5gLHR8TWpfVNZ73wf7zdqLFTfbbzuq1cq/S9\n6/q+Xvj7NZX1wu9vQ/p8tvParV6n5P29+PksDS9zgOdSPXKJ+jrb1cdHf81tob4tqdaSOYV688dG\nEXEc8FHgTGAPqr2ULouI2Q1tTomIG+tBupuOHM/MB+r2+7dQ37Q1Ff7yrs2G9I+j4UXrohd+fxvS\n57Od1za8NKd0nZfS0LOu/XwX+C6suWN1gwXAZzPzS3Wbk4BXAG8Ezqqv8SngU/X5bSLit5n5SETM\nAuaPnJvAZgBLlixpzx9oA7J8+XKGh4e7XcakulFjp/ps53VbuVbpe9f1fc2274W/h93QC7+XDenz\n2c5rt3qdkvd36vPZ8N/OzdapoCZE5pgbGlNSRKwCjs7MS+rXGwO/BV49cqw+fj4wKzOPGecae1Kt\nAgzVHaNPZOa/TNLnnwNfadsfQpKk6ef4zPxqOy9YdOclIj4P/Gdmfn4t7U4E5mfmG0v6WYvZVDtW\n3z/q+P2suWnkapn5I6rHS826DDgeWAo8PnlTSZLUYDOq4SOXtfvCReEFOLH+Pml4AfYF3kD1GKfn\nZOZDQFvToiRJ08g1nbhop8eubAI82aFrP1hfe9tRx7cF7utQn5Ikqcs6Fl7qAbZ9wAOduH5mrgSG\ngINH9XkwHUp6kiSp+5p+bBQRC0cdOnycY43X3Zlq+vS/FtZGRGxJtZbMyEyj50XEbsCvMvNu4Bzg\n/IgYAm6gmn20BXB+aZ+SJGlqa3q2UT3bZ0Sy9mX1V1JNc35TZj5YVFzEAcAVjF3j5Ysjg4Aj4hTg\nPVSPixYDb8vMH5f0J0mSpr51CS87jfwI3E61su67J2j+BPBg/WhHkiSpbZoe85KZd9ZfS4G/B77c\ncGz0173TIbhExCsj4r8j4ucR8aZu1yPpKRFxYUT8KiK+3u1aJK0pIp4dEVdExE8jYnFEvGad3t8r\ni9RNNRGxEfAz4ADgEWAY2Cszf93VwiQBEBHzgZnAGzLztd2uR9JTImI7YJvMvCkitqWagPOCzHys\nmfevl2X+N1AvBf4rM+/LzEeA/wAO7XJNkmqZuYjqfywkTTH1fztvqn++n2r5k62afb/hpdwOwLKG\n18uAHbtUiyRJPSki+oEZmblsrY1r0zK8RMT+EXFJRCyLiFURceQ4bd4aEXdExGMRcV29L5KkDvPz\nKU1t7fyMRsRWwBeBt6xLDdMyvABbUk2rPoWx07CJiOOAjwJnUu2F9BPgsoiY3dDsl8CzG17vWB+T\n1Jp2fD4ldU5bPqMRsQlwEfChzLx+XQqY9gN2R+9WXR+7Drg+M99evw7gbuDjmXlWfWxkwO6BwArg\nR8A+DtiV2qf089nQ9kDgrZl57PqrWpo+WvmMRsQgsCQz37+u/U7XOy8TioiNgX7gByPHskp43wf2\nbjj2JPBO4EqqmUYfMbhIndXs57Nueznwb8DLI+KuiNhrfdYqTUfNfkYjYl/gWODoiLgxIoYj4o+b\n7ad0V+kN2WxgI+D+UcfvB3ZpPJCZ3wa+vZ7qkrRun8+Xra+iJK3W1Gc0M6+mhQzinRdJktRTDC9j\nPQg8SbVXUqNtgfvWfzmSGvj5lKa29fIZNbyMUm9rMAQcPHKsHmx0MHBNt+qS5OdTmurW12d0Wo55\niYgtgefz1M7Yz4uI3YBfZebdwDnA+RExBNwALAC2AM7vQrnStOLnU5rapsJndFpOlY6IA4ArGDs/\n/YuZ+ca6zSnAe6hudS0G3paZP16vhUrTkJ9PaWqbCp/RaRleJElS73LMiyRJ6imGF0mS1FMML5Ik\nqacYXiRJUk8xvEiSpJ5ieJEkST3F8CJJknqK4UWSJPUUw4skSeophhdJktRTDC+SJKmnGF4kSVJP\nMbxIaquIODIi/jMiFkfE7yJiVUQ8HBHPnKD9thFxdUQ8UrddFRErImIoIl62vuuXNPW5q7SkjoiI\nAB4CZtWH3p2Z50zSfivgFuDbwMmZ+Vjnq5TUiwwvkjoiInYDrgCeALYBbsvMF67lPbcBe2TmivVQ\noqQe5WMjSZ2yP3Al8Nn69c4RcehEjSNiDrDM4CJpbQwvkjplPrAQ+DSwsj721knaHwD8sNNFSep9\nhhdJnbI/sDAz7wO+DgTwpxHxnAnaz8fwIqkJhhdJbRcRL6AaU/ez+tB59fcZwEkTvG0f4JpO1yap\n9xleJHXCfKrxLgBk5o+Ba6nuvrwpIp7W2DgidgB+7QwjSc0wvEjqhPlUM40afaz+vjVw7DjtW35k\nFBEbt3oNSVOf4UVSJ4wM1m10IXB3/fMpo84dACxa20UjYs+I+FS9qN33IuLrEXFmRGwWETsBXxjV\nfpOI+E5E3FYvfresfv2diLgiIm6JiH+PiHkRcUxELIyIR+u2P6v72L7hel+LiOX1+Tsi4sPr/JuR\n1DLXeZHUVhHxbODazBwzMDci3gN8GEhg98y8uT5+E7DvRNOkI+JZwCeBFwPvycz/aDjXD7wD2AMY\nzMwPjvP+44CvAq/OzG82HN8YGAQOquu5KyKuA7bLzDkT1HI08KrMfP1afxmSOsI7L5LabbxHRiP+\nGRgZ13IKQETMBh6fJLjsAlwPbAn0NwYXgMwcAm4A5k3S73719/8c9d6VVHdrnkk1FmcLoG+S6wDM\nrf8ckrrE8CKp3SYML5n5a+BfqQbuHh8RT6/bXzVe+4jYGvgu8DDw2sx8fII+vwM8ShVyxrMf1Qq/\nD45z7un195lUM56exuTjb/aapB9J64HhRVK77c/kdy5Gpk1vCZzI5IN1Pwc8B/jLtcxEehi4OjOf\nHH0iIp5B9bhpomnYB1I9xrqmriUnqqfer2mTzHxiklokdZjhRVLb1GNTNsvMpRO1ycwlwOX1y5Op\n7oqMufMSEfsCRwGX1VOtJ/MIcOYE5/ah+rfu6nH6mA0cB1yRmd+gCjLLMvOOCa61K3DzWmqR1GGG\nF0nttMb6LpP4GNWjoz8EnlY/ThrtL6jugnxhnHNryMzHMvOGCU7vz1N3VlarF9K7GPgecEREbArs\nyeSPjJqaFSWps5629iaS1LRDmPyREQCZeWlE3AK8gInDwAH19ytbrGl/qr2V3lY99SGBTajGyLwz\nM68DiIj5wKZMHl72wcG6UtcZXiS1RUQcBrwBWNzkW/6p/pooLOwIPDzeINuIeBnw18A2wObAKuAO\n4Ih6BtFIu02o7qZck5knr6WeSce71DZ3FWCp+3xsJKklEfGtiFhKNeNnc+DTEXFzRJy6lreeD9zH\nxHdeHgZ+O96JzLw8Mw8B3gnsDHwtMw9vDC61PanupjTzqGdX4KHMvHW8kxGxNzDcxHUkdZh3XiS1\nJDOPKHzfo8AOkzS5Hnh5RGxZtx3PPlR3Sy6d4PzIeJdmth7YAhg3uNQW1F+Susw7L5KmqnOp/o16\ny3gn62nLr6Eau/KjCa4xMt7l2ib6uxGYPUFfpwHfy8xlTVxHUocZXiRNSZl5JfBe4IMR8RcRsfrf\nq3qK82eA26jGs4y3vsvGVHdmhidZ3K7RJ4AtI+L4hmtsFhFnAttk5r+09AeS1DbubSRpSouIA6ke\n1+wE3A/8Bvgf4OPAXcBLM/OqhvY7Af9St98ZWAH8GPhE475GE/T1XOAsqjswvwOeAD6TmZe1908l\nqRWGF0mS1FN8bCRJknqK4UWSJPUUw4skSeophhdJktRTDC+SJKmnGF4kSVJPMbxIkqSeYniRJEk9\nxfAiSZJ6iuFFkiT1FMOLJEnqKf8fnVLzp9np1dwAAAAASUVORK5CYII=\n", "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "for (i,loss_arr) in enumerate(loss_arrays):\n", - " plt.semilogy(np.array(range(len(loss_arr)))*node_counts[i],loss_arr,label=r\"$N_{{GPU}} = {{{}}}$\".format(node_counts[i]))\n", - "plt.legend(loc=(1,0))" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAGPCAYAAACUIDA3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3XmcXXV9//HXB2Q3joWwqwRRaayCzEgpW6CgoFU2Be0U\nFOrSCq7Bpe1PWq1Va0FBrVq1LcZ1qBUQtCKiAYOEpc4QoZoKCIEQhRLEEJAlks/vj3Mm3MyWm++9\nd2Yu83o+Hvcxc8/53vP9zMgj8/ac7xKZiSRJUrfYZKoLkCRJ2hiGF0mS1FUML5IkqasYXiRJUlcx\nvEiSpK5ieJEkSV3F8CJJkrqK4UWSJHUVw4skSeoqhhdJktRVDC+SJKmrzLjwEhEXRMSvI+LrU12L\nJEnaeDMuvAAfB14z1UVIkqQyMy68ZOYi4IGprkOSJJV5UrsvGBEvAZ4HLAcuyMw17e5DkiTNXEV3\nXiLitIi4NSIOHHH868B/Af8EfA24MiK2LC0uIg6OiIsjYkVErI2Io8do8+aIuC0iHoqIayJi39L+\nJEnS9Ff62Og4YGvg6uED9R2X44EVwEeA64B9gTe2UN82wBLgNCBHnoyIVwMfA94H7AP8BLg0ImY3\ntDktIq6PiKGI2KKFWiRJ0jQQmaMywYY/FHEHcFNmvqjh2AKqgbD7Z+Z19R2X24Flmblfy4VGrAWO\nzcyLG45dA1ybmW+v3wfV46pPZuaZE1zrUODNmXnCBvrcDjgSWAY83OrPIEnSDLIlMAe4NDPvbeeF\nS8e8bA8sGnHsEGB5Zl4HkJkPR8Ri4MCRH26HiNgM6AM+PHwsMzMivg/sP8HnLgP2ArapQ9gJmXnt\nOM2PBL7avqolSZpxTqQaStI2peFlFdD4aGZ3YDfgSyPaPUj16KcTZgObAnePOH43sOd4H8rMF29E\nH8sAvvKVrzB37tyNrU8z0Pz58znnnHOmuoyuN1N+j934c07Hmqeypsnsu5N9deLaS5cu5aSTToL6\nb2k7lYaXW4B5EfGMzLwD+AuqMSnfHdHuacBdLdQ31R4GmDt3Lr29vVNdi7pAT0+P/620wUz5PXbj\nzzkda57Kmiaz70721eGfo+3DLkoH7P4L1bOsGyJiEHgPcA/w7eEGEbEV8ELgZ60WOY6VwGPAjiOO\n70h3ByZ1sf7+/qku4Qlhpvweu/HnnI41T2VNk9l3J/uajv+7TqRowC5ARJwFvAXYArgTODkzL284\nfzLwBeD0zPx4y4U2P2D3DqoBu2e1oc9eYHBwcHDa/T8NSZKms6GhIfr6+gD6MnOondcuXqQuM98d\nEWcAT8nMe8ZospBq+vIvSvuIiG2AZwFRH3pmROwN/DozlwNnAwvquz/XAfOppnAvKO1TkiRNby2t\nsJuZj1A9Lhrr3HKqacuteCFwOdV4mqRa0wXgi8DrMvPr9ZouH6B6XLQEOHKcMCVJkp4A2r49QDtl\n5g/ZwLiczPwM8JnJqUiSJE214vASETtQrXw7D9iZauzLWDIz9yjtR5IkqVFReImIucAPge14fDyK\nJElSx5VOlT6LapG4C6jGpTwlMzcZ79W2aiVJ0oxX+tjoYODnwKuydK61JElSgdK7IgEsMbhIkqTJ\nVhpefky1l5EkSdKkKn1s9H7gBxFxVGZ+q431TEvz58+np6eH/v7+rltCWZKkyTQwMMDAwACrVq3q\nWB9NbQ8QEfPGOHw08Haqba4vo9oiYO1Yn8/MRS3UOGXcHkCSpDLTYXuAK6hWuB0pgNcAJ23g85tu\nRE2SJEnjaja8fImxw4skSdKkaiq8ZOYpHa5DkiSpKS4gJ0mSukpReImIJ0fEXvWOzuO1mV232aa8\nPEmSpPWV3nk5HbgemGjDxT3qNm8v7EOSJGmU0vByFHBLZl47XoP63C+AYwv7kCRJGqU0vDwT+N8m\n2i0Fdi/sQ5IkaZTS8LIV8FAT7R4CnlzYhyRJ0iil4WU5sG8T7fYFflnYhyRJ0iil4eVSYE5EzB+v\nQUS8neqR0XcL+5AkSRqldGPGM6m2BfhoRBwOfJ5qcC5Us4z+AngpcH/dVpIkqS2Kwktm3hkRRwPn\nA39CFVQaBbASOCEzb2+tREmSpMeV3nkhM6+MiD2BNwKHA0+vTy0Hvg/8W2be13qJkiRJj4tM91sc\nT0T0AoPz5s2jp6eH/v5++vv7p7osSZKmrYGBAQYGBli1ahWLFi0C6MvMoXb2YXiZwHB4GRwcpLe3\nd6rLkSSpawwNDdHX1wcdCC/Fj40AImJH4HXAwcCu9eEVwCLgC5l5d2vlSZIkra84vETEK4FzqRah\ni4ZTzweOBP46Il6fmee3VqIkSdLjSneVfiEwAGwDXAgcB+wDvIBqL6MLqELN1+q2kiRJbVF65+Vv\ngE2B4zPzwhHnbgAujojjqKZS/zVwfHmJkiRJjytdYfcgYPEYwWWd+txVVONhJEmS2qI0vPQAdzTR\n7o66rSRJUluUhpe7qMa4bMgL6raSJElt0crGjHtGxIcjYtORJ6PyQeD3cWNGSZLURqUDdv8BeAXw\nV0B/RHwdWFaf2w04AZgD3At8sLUSJUmSHtfKxoyHAV8Fnge8Gxheqnd4zZcbgRMz886Wq5QkSaq1\nsjHjjcBeEXEo1YyiXepTvwSuzMwrWq5OkiRphJa2BwCoQ8oVLVciSZLUhJbDy7CI+L3629+kuz1K\nkqQOKZ1tBEBEHB0R34uIB4CV9Wt1feyYtlQoSZLUoHRvo4iIc6n2NXoRsDWwqn5tXR+7ICIWRESM\nfyVJkqSNU3rn5e3AKcCvgFOBp2bmtpm5LdWKum+qz72mbitJktQWUTI8JSJ+BjwDeH5m3jZOm92p\npkvfkZnPbanKKRIRvcDgvHnz6Onpob+/n/7+/qkuS5KkaWtgYICBgQFWrVrFokWLAPoyc6idfZSG\nl4eA72XmhONaIuIi4IjM3Kqwvik1HF4GBwfp7e2d6nIkSeoaQ0ND9PX1QQfCS+ljo3uAR5tot4Zq\nEK8kSVJblIaXC4HDGqZHjxIR2wKHAd8s7EOSJGmU0vByBnArsLDeJmA9EfHHwGXAL4D/V16eJEnS\n+koXqbuI6rFRH3BZRPwauL0+9wxgu/r7a4CLRsyWzsw8vLBfSZI0w5WGl0Mbvg+qsLLdGO32H+OY\nq+9KkqRipeFl97ZWIUmS1KSi8JKZt2+4lSRJUvu1tLeRJEnSZGt1Y8bnRsQ5EXFVRPw8Is5sOHdA\nRLytnjItSZLUFqVjXoiI04GPNFwjgdkjmp0DPAJ8rrQfSZKkRqW7Sr8M+CiwHHgFsAPVrKN1MnMx\n1Uq8E24hIEmStDFK77ycDjwIvDgzbwUYsZbLsCXAnoV9SJIkjVI65qUPuGY4uExgJbBTYR+SJEmj\nlIaXzYHVTbTbAfhdYR+SJEmjlIaX24C9J2oQEZsDewE3FfYhSZI0Sml4uRiYU884Gs97gO2BCwr7\nkCRJGqV0wO6ZwInAWRGxH3BhfXzHiDgOOK4+fxvwqZarlCRJqpVuD3BfRLwI+AZwAnB8feol9SuA\nnwHHZmYzY2MkSZKaUrxIXWbeFBEvAI4CjgDmUD2GuhO4DDg/Mx9rR5GSJEnDisMLQGauBS6qX09Y\n8+fPp6enh/7+fvr7+6e6HEmSpq2BgQEGBgZYtWpVx/qIzOzYxbtdRPQCg4ODg/T29k51OZIkdY2h\noSH6+voA+jJzqJ3XdldpSZLUVQwvkiSpqxheJElSVzG8SJKkrmJ4kSRJXcXwIkmSuorhRZIkdZWN\nWqQuIjYFDgB2Ae4CFmfmmgnaHwA8KzO/1FKVkiRJtabvvNRB5BbgCuBrwEJgeUS8cYKPvRH4QisF\nSpIkNWoqvETEHOASYDfgfuDHwD3ADsBnI+IbEbFFh2qUJElap9k7L38FzKK6i7JzZu4H7Ay8BrgX\nOA64NCKe0pEqJUmSas2GlxdT7Rb9psx8GCArXwX6gCFgHrAwImZ3pFJJkiSaDy9PA64ba3BuZi4H\nDgG+C/QCiyJi1/aVKEmS9Lhmw8v9wKbjnczM3wJHA+cBvw9cGRF7tF6eJEnS+pqdKn078IcTNcjM\n30XEicB9wKnAImBZS9VJkiSN0Oydl+uAnSLikIka1eNg3gx8mGpA7x+1WJ8kSdJ6mg0vFwIB/E0z\njTPzDOCdpUVJkiSNp9nHRj8Ang1ksxfOzHMiYiHw1JLCJEmSxtJUeMnMBH6xsRfPzJ9sdEWSJEkT\ncGNGSZLUVQwvkiSpqxheJElSV2l2wO6MNn/+fHp6eujv76e/v3+qy5EkadoaGBhgYGCAVatWdayP\nqMbiaiwR0QsMDg4O0tvbO9XlSJLUNYaGhujr6wPoy8yhdl7bx0aSJKmrFIWXiDg7Iv623cVIkiRt\nSOmdl7cAe7WzEEmSpGaUhpc7W/isJElSsdIA8k3gkIiY1c5iJEmSNqQ0vLwPuAP4TkTs08Z6JEmS\nJlS6zstFwCPAgcCPI+JXVGHm4THaZmYeXtiPJEnSekrDy6EN3wewS/0aiwvJSJKktikNL7u3tQpJ\nkqQmFYWXzLy93YVIkiQ1w+nOkiSpq7QUXiLiuRFxTkRcFRE/j4gzG84dEBFvi4htWy9TkiSpUryr\ndEScDnyk4RoJzB7R7ByqWUmfK+1HkiSpUeneRi8DPgosB14B7EA162idzFwM3AMc02KNkiRJ65Te\neTkdeBB4cWbeChARY7VbAuxZ2IckSdIopWNe+oBrhoPLBFYCOxX2IUmSNEppeNkcWN1Eux2A3xX2\nIUmSNEppeLkN2HuiBhGxObAXcFNhH5IkSaOUhpeLgTn1jKPxvAfYHrigsA9JkqRRSgfsngmcCJwV\nEfsBF9bHd4yI44Dj6vO3AZ9quUpJkqRa6fYA90XEi4BvACcAx9enXlK/AvgZcGxmNjM2RpIkqSnF\ni9Rl5k0R8QLgKOAIYA7VY6g7gcuA8zPzsXYUKUmSNKw4vABk5lrgovolSZLUcaUr7J4bEa9rot0p\nEXFuSR+SJEljKZ1tdApwUBPtDgROLuxDkiRplJZ2lW7C5oDjXiRJUtt0LLxEtdlRL9XmjJIkSW3R\n9IDdiFg44tBLxjjWeN09qPY1+nJhbZIkSaNszGyjQxu+T6pgMtGmi2uAbwPv2viyppf58+fT09ND\nf38//f39U12OJEnT1sDAAAMDA6xatapjfURmNtcwYrfhb4FbqRaoe/c4zR8FVmbmmpYrnEIR0QsM\nDg4O0tvbO9XlSJLUNYaGhujr6wPoy8yhdl67qTsvEfFa4JbMXFy//3vg+sy8vZ3FSJIkbUizA3YX\nAG9oeH8KzU2VliRJaqtmw8ta1r9Lsxswu/3lSJIkTazZ8PJ/wPM7WYgkSVIzmp1t9H3gpIj4BTA8\nzmWiqdKNMjMPL6pOkiRphGbDy+nAU4GXArvT3FTpYc1NZ5IkSWpCU+ElM1cCR0fEZsDOwDImniot\nSZLUERuzSB31ui13RMQdwDKnSkuSpMm2UeFlWGbOaXMdkiRJTen0rtKSJEltVXTnZVhEHAQcAzwb\nmEW1dcBIzjaSJEltUxReIiKAfwdO5vHAkqwfXobfO9tIkiS1TeljozdRbREwCLwYuKA+vifVdOoF\nVKvyngU8s6UKJUmSGpQ+NjoFeBB4aWbeGxEnAWTmzcDNwKUR8R3gP4DFPL6wnSRJUktK77zMBRZn\n5r31+wSIiE2HG2TmN6juzLyrpQolSZIalIaXTYB7G97/tv76eyPa3Yx7IkmSpDYqDS8rgF0a3g8/\nFtpnRLvnAL8r7EOSJGmU0vAyBDy34THR96hmFp0ZEb8fEbMi4t1AH3B9G+qUJEkCysPLxcBs4GUA\nmfkT4Dxgb+CnwG+Aj1DddXlv62VKkiRVSrcHGIiIC1j/kdDJwA3AsVRjX24CzszM61quUpIkqVa8\nwm5mPjLi/Rqquy0fabUoSZKk8bi3kSRJ6iqGF0mS1FUML5IkqasYXiRJUlcxvEiSpK5ieJEkSV3F\n8CJJkrpK28NLRPRExAsjYqd2X1uSJKkovETEERFxbkTsM+L4W4G7gGuBOyPinDbUKEmStE7pnZc3\nACcANw8fiIjnAx8HNgWuAe4H3hYRx7RapCRJ0rDS8NILLMnMBxqOnTL8NTMPpNpR+lHgtPLyJEmS\n1lcaXnYE7hxx7HCq3aTPA8jM24AfAnOLq5MkSRqhNLw8Bmw5/CYitgWeB1yZmWsb2t0DbF9eniRJ\n0vpKw8sy4ICI2Kx+/woggMtGtNsOuLewD0mSpFFKw8t/UN1RWRQRHwPOBNYA3xxuEBFBNe7l1laL\nlCRJGvakws99EjgG2K9+rQXenZkrGtocRhVwPttShZIkSQ2KwktmPhgRBwDzqALKksy8eUSzx4D5\nwLdaK3HqzZ8/n56eHvr7++nv75/qciRJmrYGBgYYGBhg1apVHesjMrNjF+92EdELDA4ODtLb2zvV\n5UiS1DWGhobo6+sD6MvMoXZeu/Sx0bgiYi7wB8DyzLy23deXJEkzW+n2AK+OiIURsd+I42cB/0M1\noHdxRFwYEZu2oU5JkiSgfLbRScALgOuHD9RjYN4JrKZaqG4ZcDRwYmslSpIkPa40vDwPuCEzH204\n9hoggVdl5onAvsADVPsgSZIktUVpeNkBWDHi2B8D/5eZ3wPIzF8Di4BnlZcnSZK0vtLw8hDwlOE3\nEbEz8ByqvYwa/Qb4vcI+JEmSRikNL7cCB0fEU+v3J1I9MvreiHY7Af9X2IckSdIopeFlAdWdl8GI\nOB/4INX4louGG9T7Hr0QuKnFGiVJktYpXeflX6nGuLwS2B14EPjLzGzchPHlQA+wsKUKJUmSGpRu\nD7AGOCEi5lBtD/C/mbl6RLPbgOOAa1opUJIkqVFLK+xm5jKq9VzGOrcEWNLK9SVJkkZqeXuAiNic\nasG6XetDK6g2anx0/E9JkiSVKQ4vEbEl8AHgL4Enjzj9QER8FnhfZj7cQn2SJEnrKQovEbEF8H1g\n//rQDVSPjxKYA+wNvAs4MCIOz8xHWq5UkiSJ8qnS84EDgKuAF2TmPpl5XGa+IjN7qcLLlVTh5h3t\nKVWSJKk8vPQD9wAvy8wbR57MzP+hmiq9EjdmlCRJbVQaXp4FXDHG9Oh1MvMB4Apgj8I+JEmSRikN\nL78Dtm6i3dZ1W0mSpLYoDS83AodFxDPHaxARuwOHUQ3mlSRJaovS8PI5YCvgioh4fURsNXwiIraK\niD+nemS0JfDZlquUJEmqlW4P8OWIOAh4I/B54PMRsbI+Pbv+GsDnMvOrrZcpSZJUKb3zQmb+JXAC\n8CNgDdUeR9vX318JnJCZp7ajSEmSpGGt7m10PnB+RDwJ2K4+fG9mOkhXkiR1RMt7GwHUYeXudlxL\nkiRpIsWPjSRJkqZCU3deImJhC31kZh7ewuclSZLWafax0aEt9JEtfFaSJGk9zYaX3TtahSRJUpOa\nCi+ZeXunC5EkSWqGA3YlSVJXMbxIkqSuYniRJEldxfAiSZK6iuFFkiR1FcOLJEnqKoYXSZLUVYrC\nS0QsjIj3NNHuXS1uLSBJkrSe0l2lDwWWNdFuT+CQwj4kSZJG6fRjoy2B33W4D0mSNIN0LLxExFOA\nA4BfdaoPSZI08zT92Cgibh1x6PiIOHSC6+5Yf/1UWWmSJEmjbcyYlzkN3yfw5Po1ljXAL4GLgb8p\nqkySJGkMTYeXzFz3iCki1gILMvN1HalKkiRpHKWzjf4cuKWdhUiSJDWjKLxk5hfbXYgkSVIzWppt\nFBFHRMSFEbEiIh6JiH9vOHdkRJwdEbu0XqYkSVKlOLxExCeAS4BjgFnAZkA0NPkV8A7g1a0UKEmS\n1Kh0e4DXAm8FBoHezHzKyDaZeQOwHDiqpQolSZIalA7YPRX4DfCyzLxngnY3AM8v7EOSJGmU0sdG\nzwMWbyC4AKyiWqxOkiSpLVoZsJtNtNkFeKiFPiRJktZTGl5uBnojYrPxGkTELOAFwE8L+5AkSRql\nNLz8J7Az8JEJ2vwj0AOcV9iHJEnSKKUDdj8O/Cnwjog4ALioPr5HRMwHjgMOAoaAf225yik2f/58\nenp66O/vp7+/f6rLkSRp2hoYGGBgYIBVq1Z1rI/IbGboyhgfjNgeWAC8lGr8S4xochlwUhODeqet\niOgFBgcHB+nt7Z3qciRJ6hpDQ0P09fUB9GXmUDuvXXrnhTqUvCwi9gaOoNp1ehPgTuCyzLyuLRVK\nkiQ1KA4vwzLzJ8BP2lCLJEnSBrW0t9FYIqInIl4YETu1+9qSJEml2wMcERHnRsQ+I46/DbgLuBa4\nMyLOaUONkiRJ65TeeXkDcALVei8ARMTzgXOATYFrgPuBt0XEMa0WKUmSNKw0vPQCSzLzgYZjpwx/\nzcwDgT7gUeC08vIkSZLWVxpedqSaVdTocKrNGs8DyMzbgB8Cc4urkyRJGqE0vDwGbDn8JiK2pdqs\n8crMXNvQ7h5g+/LyJEmS1lcaXpYBBzTsbfQKqkXqLhvRbjvg3sI+JEmSRikNL/9BdUdlUUR8DDgT\nWAN8c7hBRATVuJdbWy1SkiRpWOkidZ8EjgH2q19rgXdn5oqGNodRBZzPtlShJElSg6LwkpkP1hsy\nzqMKKEsy8+YRzR4D5gPfaq1ESZKkx7Wyt9Fa4IoJzl8x0XlJkqQSbd8eQJIkqZMML5IkqasYXiRJ\nUlcxvEiSpK5ieJEkSV3F8CJJkrqK4UWSJHUVw4skSeoqRYvURUSz+xU9SrUx4/XAVzPz6pL+JEmS\nhpWusDtnI9vvD5waEf+cme8o7FOSJKnssVFmbgJ8AlgFfAjYC3hq/Xo+8EHgN8A/A88A/gz4JfDW\niPjT1suWJEkzVeljo9OAU4E/yswlI07/FPi7iDgfuA64OTM/FRH/C/wYeANwXgs1S5KkGax0wO6p\nwBVjBJd1MvMnwELgTfX7JcAgsE9hn5IkScXhZQ/gviba/QZ4ZsP724AnF/YpSZJUHF7uAw6KiM3H\na1CfO4j1Q85TqAKNJElSkdLwcjGwK3BeRDxj5MmIeBowAOwCXNRwai7Q7DRrSZKkUUqnSp8BHA4c\nC7w8Iv4bWF6fezrwQmAz4BbgbwEiohfYCTi3lYIlSdLMVhReMvPeiNgf+CeqadD7169hDwNfAv46\nM++tPzMEbNVauZIkaaYrvfNCHUreEBFvB/qAnetTvwIGM/PBNtQnqQWZSURMdRmS1FbF4WVYHVIW\ntaEWSW2wevVqPvre93LVt77FNmvW8OBmm3HgUUfxrg99iFmzZk11eZLUspbDi6TpY/Xq1bxy//05\nfelS3r92LQEkcOmnP80rFy7k/KuvNsBI6nrF4SUitgD6gXlUj4y2GKdpZubhpf1Iat5H3/teTl+6\nlJesXbvuWAAvWbuWXLqUj51xBu//xCemrkBJaoPS7QF2BX4APJvq38aJZEkf08nSpVNdgdScS75x\nK0ev3ZuhMc7tsBYu+c9fcPTJk16WpBmok387S++8nAU8B1gMnA3cBKxuV1HTzUknTXUFUrO+zQsn\nOv0r6OubrFokqTNKw8uRwB3AizLz4TbWMy195Sswd+5UVyFt2Jte/nL+5Ve/HPN2aAKn7rwLn/32\ntye7LEkz0NKlnfs//6XhZQvg+zMhuEAVXHp7p7oKacNeevwzuefTl6w35mXYJZtswp+ccLD/LUvq\neqXbA9wIzG5nIZJa964PfYiz587lkk02WTfYLKmCyzlz5/LOD35wKsuTpLYoDS//BMyLiD9sZzGS\nWjNr1izOv/pqrn3LWzhizhyO2XVXjpgzh2vf8hanSUt6wih9bDRENVD3BxFxNnAZcCcw+l41kJl3\nFPYjaSPNmjWrmg79iU+4wq6kJ6TS8LKM6m50UG3SeMYEbbOFfiS1wOAi6YmoNFQs4gmwfoskSeo+\npbtKH9rmOiRJkppSOmBXkiRpShheJElSV2nqsVFEvLb+9sLMXN3wvimZ+aWNrkySJGkMzY55WUA1\nQPcaqj2Mht9vSNTtDC+SJKktmg0vH6AKIStHvJckSZpUTYWXzHz/RO8lSZImiwN2JUlSVzG8SJKk\nrlK8bH9EbAH0A/OAnYEtxmmamXl4aT+SJEmNisJLROwK/AB4NtWMook4sFeSJLVN6Z2Xs4DnAIup\ndpe+iWoKtSRJUkeVhpcjgTuAF2Xmw22sR5IkaUKlA3a3AK41uEiSpMlWGl5uBGa3sxBJkqRmlIaX\nfwLmRcQftrMYSZKkDSkd8zJENVD3BxFxNnAZcCewdqzGmXlHYT+SJEnrKQ0vy6imQAdwRv0aT7bQ\njyRJ0npKQ8UiXL9FkiRNgaLwkpmHtrkOSZKkpri3kSRJ6iqGF0mS1FWaemwUEa+tv70wM1c3vG9K\nZn5poyuTJEkaQ7NjXhZQDdC9hmoPo+H3GxJ1O8OLJElqi2bDyweoQsjKEe8lSZImVVPhJTPfP9F7\nSZKkyeKAXUmS1FUML5Ikqau0tGx/RDwDOAp4NjCLaoDuSJmZr2+lH0mSpGHF4SUi/g74W9a/ezMc\nXrLhfQKGF0mS1BZFj40i4tXA+4HlwF9Q7SoNcCRwKvBDquByNnBYy1VKkiTVSu+8nAY8CvxxZt4e\nEQcBZOZwiPlcRMwHzgS+2XqZkiRJldIBu3sBizPz9vp9AkTEujEvmXkO8HPgjJYqlCRJalAaXrYA\n7mp4/3D99akj2v0E2LewD0mSpFFKw8uvgB0a3q+ov/7BiHZPAzYt7EOSJGmU0vByI7Bnw/srqAbo\n/n1EbAMQEa8CDgZ+2kqBkiRJjUrDy7eAXSPiMIDMvAq4HPhj4L6IWAkMUI2F+Yd2FCpJkgTl4eUr\nwFxgScOx44DPA7+mWrDuZ8BrMvO7LVUoSZLUoGiqdGY+QjWTqPHY/cCb6pckSVJHlC5Sd0FEfLrd\nxUiSJG1I6WOjPwG2a2chkyEinhYRl0fETyNiSUQcP9U1SZKkjVO6wu5twDbtLGSS/A54e2beEBE7\nAoMR8V+Z+dBUFyZJkppTeudlADgkInZqZzGdlpl3ZeYN9fd3AyuBbae2KkmStDFKw8s/AlcCP4yI\n4yJiszbWNCkiog/YJDNXbLCxJEmaNkrDy8+pVtN9FvAN4KGI+GVE3DrG6xelxUXEwRFxcUSsiIi1\nEXH0GG0G/56iAAAKRElEQVTeHBG3RcRDEXFNRGxwO4KI2Bb4IvDG0tqksQwMDEx1CU8IM+X32I0/\n53SseSprmsy+O9nXdPzfdSKl4WUO8AyqVXWjvs5O9fGRr91bqG8bqrVkTqPe/LFRRLwa+BjwPmAf\nqr2ULo2I2Q1tTouI6yNiKCK2iIjNgQuBD2fmtS3UJo3Sbf8ATFcz5ffYjT/ndKzZ8DK9r90Jpeu8\nlIaeje3nu8B3Yf0dqxvMBz6XmV+q27wJeBnwOuDM+hqfAT4z/IGIGAB+kJlfa6KELQGWLl3awk+h\nmWTVqlUMDQ1NdRldb6b8Hrvx55yONU9lTZPZdyf76sS1G/52btnWCwOROeqGxrQUEWuBYzPz4vr9\nZsBvgVcOH6uPLwB6MvO4Ma5xIPBD4AaqO0ZJtQrwmPsvRcSfAV9t848iSdJMcmKTNwyaVnTnJSLO\nBX6UmeduoN0pwLzMfF1JPxswm2rH6rtHHL+b9TeNXKfeg2ljfuZLgROBZcDDG1+iJEkz1pZUw0cu\nbfeFS9d5OaX+OmF4AQ4ETqZ6jNN1MvNeoK1pUZKkGWRxJy7a6bErmwOPdejaK+tr7zji+I7AXR3q\nU5IkTbGOhZd6gG0vcE8nrp+Za4BB4PARfR5Oh5KeJEmaek0/NoqIhSMOvWSMY43X3YNq+vSXC2sj\nIrahWktmeKbRMyNib+DXmbkcOBtYEBGDwHVUs4+2BhaU9ilJkqa3pmcb1bN9hiWPB4rxrKGa5vz6\nzFxZVFzEIcDljF7j5YvDg4Aj4jTgPVSPi5YAb83MH5f0J0mSpr+NCS+7DX8L3Eq1su67x2n+KLCy\nfrQjSZLUNk0/NsrM24e/j4i/B65vPDYTRcQFwKHA9zPzVVNcjiRJXSEinkY1rGQHqic1H8zMbzT9\n+W5ZpG46ioh5wCzgZMOLJEnNiYidgB0y84aI2JFqAs6zM/OhZj4/Kcv8P1Fl5iLggamuQ5KkbpKZ\nd2XmDfX3d1Mtf7Jts583vEiSpCkTEX3AJpm5otnPzMjwEhEHR8TFEbEiItZGxNFjtHlzRNwWEQ9F\nxDURse9U1CpJ0nTSzr+hEbEt8EXgjRtTw4wML8A2VNOqT2P0NGwi4tXAx4D3AfsAPwEujYjZk1mk\nJEnTUFv+hkbE5sCFwIcz89qNKWDGD9gduVt1fewa4NrMfHv9PoDlwCcz88wRnz8UeHNmnjB5VUuS\nNPVa+RsaEQPA0sz8wMb2O1PvvIwrIjYD+oAfDB/LKuF9H9h/RNvLgP8AXhoRd0TEfpNZqyRJ00mz\nf0Mj4kDgBODYiLg+IoYi4g+a7ad0V+knstnApsDdI47fDezZeCAzXzxZRUmS1AWa+huamVfRQgbx\nzoskSeoqhpfRVgKPUe2V1GhH4K7JL0eSpK4xKX9DDS8j1PsxDQKHDx+rBxsdDiyeqrokSZruJutv\n6Iwc8xIR2wDP4vGdsZ8ZEXsDv87M5cDZwIKIGASuA+YDWwMLpqBcSZKmjenwN3RGTpWOiEOAyxk9\nP/2Lmfm6us1pwHuobnUtAd6amT+e1EIlSZpmpsPf0BkZXiRJUvdyzIskSeoqhhdJktRVDC+SJKmr\nGF4kSVJXMbxIkqSuYniRJEldxfAiSZK6iuFFkiR1FcOLJEnqKoYXSZLUVQwvkiSpqxheJElSVzG8\nSGqriDg6In4UEUsi4pGIWBsR90fEU8dpv2NEXBURD9Rt10bE6ogYjIgXT3b9kqY/d5WW1BEREcC9\nQE996N2ZefYE7bcFbgK+DZyamQ91vkpJ3cjwIqkjImJv4HLgUWAH4JbMfM4GPnMLsE9mrp6EEiV1\nKR8bSeqUg4ErgM/V7/eIiCPGaxwRc4AVBhdJG2J4kdQp84CFwL8Aa+pjb56g/SHADztdlKTuZ3iR\n1CkHAwsz8y7g60AAfxIRTx+n/TwML5KaYHiR1HYR8WyqMXU/qw99ov66CfCmcT52ALC407VJ6n6G\nF0mdMI9qvAsAmflj4Gqquy+vj4gnNTaOiF2A+5xhJKkZhhdJnTCPaqZRo4/XX7cHThijfcuPjCJi\ns1avIWn6M7xI6oThwbqNLgCW19+fNuLcIcCiDV00IvaNiM/Ui9p9LyK+HhHvi4gtI2I34Asj2m8e\nEd+JiFvqxe9W1O+/ExGXR8RNEfGfETE3Io6LiIUR8WDd9md1Hzs3XO+8iFhVn78tIj6y0b8ZSS1z\nnRdJbRURTwOuzsxRA3Mj4j3AR4AEXpCZN9bHbwAOHG+adERsB3waeD7wnsz8r4ZzfcA7gH2Agcz8\n0BiffzXwNeCVmfnNhuObAQPAYXU9d0TENcBOmTlnnFqOBV6Rma/d4C9DUkd450VSu431yGjYvwLD\n41pOA4iI2cDDEwSXPYFrgW2AvsbgApCZg8B1wNwJ+j2o/vqjEZ9dQ3W35qlUY3G2BnonuA7A7vXP\nIWmKGF4ktdu44SUz7wO+TDVw98SIeHLd/sqx2kfE9sB3gfuBV2Xmw+P0+R3gQaqQM5aDqFb4XTnG\nuSfXX2dRzXh6EhOPv9lvgn4kTQLDi6R2O5iJ71wMT5veBjiFiQfrfh54OvAXG5iJdD9wVWY+NvJE\nRDyF6nHTeNOwD6V6jLW4riXHq6fer2nzzHx0glokdZjhRVLb1GNTtszMZeO1ycylwGX121Op7oqM\nuvMSEQcCxwCX1lOtJ/IA8L5xzh1A9W/dVWP0MRt4NXB5Zn6DKsisyMzbxrnWXsCNG6hFUocZXiS1\n03rru0zg41SPjn4feFL9OGmkP6e6C/KFMc6tJzMfyszrxjl9MI/fWVmnXkjvIuB7wFERsQWwLxM/\nMmpqVpSkznrShptIUtNexMSPjADIzEsi4ibg2YwfBg6pv17RYk0HU+2t9NbqqQ8JbE41RuadmXkN\nQETMA7Zg4vByAA7Wlaac4UVSW0TEkcDJwJImP/LP9Wu8sLArcP9Yg2wj4sXAXwE7AFsBa4HbgKPq\nGUTD7TanupuyODNP3UA9E453qW3lKsDS1POxkaSWRMS3ImIZ1YyfrYB/iYgbI+ItG/joAuAuxr/z\ncj/w27FOZOZlmfki4J3AHsB5mfmSxuBS25fqbkozj3r2Au7NzJvHOhkR+wNDTVxHUod550VSSzLz\nqMLPPQjsMkGTa4GXRsQ2dduxHEB1t+SScc4Pj3dpZuuBrYExg0ttfv2SNMW88yJpujqH6t+oN451\nsp62fDzV2JX/Hucaw+Ndrm6iv+uB2eP09Tbge5m5oonrSOoww4ukaSkzrwDeC3woIv48Itb9e1VP\ncf4scAvVeJax1nfZjOrOzNAEi9s1+hSwTUSc2HCNLSPifcAOmflvLf1AktrGvY0kTWsRcSjV45rd\ngLuB3wD/B3wSuAP4w8y8sqH9bsC/1e33AFYDPwY+1biv0Th9PQM4k+oOzCPAo8BnM/PS9v5Uklph\neJEkSV3Fx0aSJKmrGF4kSVJXMbxIkqSuYniRJEldxfAiSZK6iuFFkiR1FcOLJEnqKoYXSZLUVQwv\nkiSpqxheJElSVzG8SJKkrvL/Ae0pS0+bx7nsAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -357,7 +383,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 79, "metadata": { "collapsed": false }, @@ -365,18 +391,18 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 24, + "execution_count": 79, "metadata": {}, "output_type": "execute_result" }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj4AAAGPCAYAAABcTNrhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3XlYlFX7wPHvAU3ABcs9d8EVlxKzzDRJlFIbLUWjRQVL\nS9ywXNJyS3JNTC21XtescU/pp+bSm5ZmmuCaYuZGroW7mblwfn8M8IKgDjAzzzBzf65rruLMmee5\nh5uB2+c5i9JaI4QQQgjhDjyMDkAIIYQQwlGk8BFCCCGE25DCRwghhBBuQwofIYQQQrgNKXyEEEII\n4Tak8BFCCCGE25DCRwghhBBuQwofIYQQQrgNKXyEEEII4Tak8BFCCCGE25DCRwghhBBuQwqfbFJK\neSuljimlxhsdixBCCCGyRwqf7BsKbDU6CCGEEEJknxQ+2aCU8geqA2uMjkUIIYQQ2SeFT/ZMBN4F\nlNGBCCGEECL73KLwUUo1UUrFKqVOKqWSlVKmLPpEKqWOKqX+UUr9rJR67I7nTcBBrfXvqU2OiF0I\nIYQQtuMWhQ9QENgF9AT0nU8qpToBHwHDgUeB3cBapVTxdN2eAF5SSh3BcuXndaXUe/YOXAghhBC2\no7TOVAe4NKVUMtBOax2bru1nYJvWum/K1wr4A5iitc40e0sp1QUI0FoPvMd5igEhwDHguk3fhBBC\nCOHavIBKwFqt9TlbHjifLQ+WFyml8gOBwIepbVprrZTaADTKxaFDgC9zGZ4QQgjhzl4BvrLlAd2+\n8AGKA57A2Tvaz2KZwZWJ1nqeFcc9BrBgwQJq1qyZm/hsIioqipiYGKc4XnZea03f+/W51/N3ey6r\ndlt/D3PDHrHk9Ji2zuf9+kk+7XvM7L7Onp9RW7UbQX7n5u4zeuDAAV599VVI+VtqS1L42M91gJo1\na1K/fn2jY8HX19emceTmeNl5rTV979fnXs/f7bms2m39PcwNe8SS02PaOp/36yf5tO8xs/s6e35G\nbdVuBPmdm/vPaAqbDxWRwgeSgNtAqTvaSwFncnvwqKgofH19CQsLIywsLLeHyzFbnzs3x8vOa63p\ne78+93r+bs9l1X7mTK5/HGzGHj9LOT2mrfN5v36ST/seM7uvs+dnNLvtrpxTd/mdazabMZvNXLp0\n6b5x5JQMbuaug5sTsQxunpDD89QH4uLi4pzmXyAid8qWLcvJkyeNDkPYiOTT9UhOXUd8fDyBgYEA\ngVrreFse2y2u+CilCgL+/G/tnSpKqXrAea31H8AkYK5SKg7YDkQBPsBcA8IVTirlQyhchOTT9UhO\nhTXcovABGgDfY1nDR2NZswdgHhChtV6csmbPKCy3uHYBIVrrv3J7Yme51SVyT/LnWiSfrkdymvfJ\nra48TG51CSHsLTExkaSkJKPDECJHihcvToUKFbJ8Tm51CSGEyCAxMZGaNWty7do1o0MRIkd8fHw4\ncODAXYsfe5HCRwgrhYeHM2fOHKPDEDaS1/OZlJTEtWvXnGatMCGyI3WdnqSkJCl8XI2M8XEdLVu2\nNDoEYUOukk9nWStMCFtwxBgfKXzsLCYmRn4puQgpXF2L5FMI55N6kSDdGB+bc5fd2YUQQgghpPAR\nQgghhPuQwsfOoqKiMJlMmM1mo0MRubR582ajQxA2JPkUwvmYzWZMJhNRUVF2O4cUPnYWExNDbGys\njCdwAePHjzc6BGFDkk8hnE9YWBixsbE23dn+TlL4CGGlhQsXGh2CsCHJZ941YsQIPDzkz5fIGfnJ\nEcJKPj4+RocgbEjymXcppbDsJZ0906dPZ968eXaISOQlUvgIIYSbsOcWRXlh+6NPP/1UCh8hhY+9\nyeBmIYSRrly5wvA+fQiuXJl25csTXLkyw/v04cqVK059bOGeHDG4Ga21POzwAOoDOi4uTgvX8M47\n7xgdgrChvJ7PuLg4fb/fMZcvX9YtAgL0Gg8PnQxag04GvcbDQ7cICNCXL1/O8fnteez0fvzxR92g\nQQPt5eWl/f399cyZM/WIESO0h4dHWp/Zs2frZ555RpcsWVIXKFBA16pVS0+fPj3DcSpVqqSVUhke\nQUFBWmutz58/r99++21dp04dXahQIV2kSBH93HPP6d27d9vkPYjM7vfzm/o8UF/b+O+zrNwshJUc\nvZ+MsC93yOfEoUPpf+AAzyYnp7Up4NnkZPSBA3z03nuM+Phjpzt2qn379hESEkLJkiUZNWoUN2/e\nZMSIEZQsWTJDvxkzZlC7dm3atm1Lvnz5+Oabb+jZsydaa9566y0APv74Y3r16kXhwoV577330FpT\nqlQpAI4cOUJsbCyhoaFUrlyZs2fPMnPmTJo1a8b+/fspXbp0rt6HcDK2rqTkIVd8hBD2Z80Vn+aV\nKqVdjbnzkQw6uFKlHJ/fnsdO1a5dO+3j46NPnDiR1paQkKDz5cuX4YrP9evXM7322Wef1f7+/hna\nateunXaVJ70bN25kajt+/Lj28vLSo0ePzs1bEHdh5BUfGeMjhBAuSGtNwZs3udvcJwX43LyZ+g81\npzl2quTkZNatW8cLL7xA2bJl09qrV69OSEhIhr4FChRI+//Lly9z7tw5mjZtypEjR6wab5Q/f/4M\n5z1//jw+Pj5Ur16d+Pj4HL8H4Zyk8BFCCBeklOLv/Pm5W+mhgb/z58/RtHB7HjvVX3/9xT///IO/\nv3+m56pXr57h6y1bthAcHEyhQoUoWrQoJUqUYOjQoQBW7fKttSYmJoZq1apRoEABihcvTsmSJdm7\nd69ddwkXxpDCx85kVpfrSEhIMDoEYUPukM/Gzz/P2rss9PethwdPmUxOeezsOHz4MMHBwZw/f56Y\nmBhWr17Nhg0b0mYFJacbg3Q30dHRvP322zRr1owvv/ySdevWsWHDBmrVqmXV64XtyKyuPPxAxvi4\nnOeff97oEIQN5fV8ZmdW1+o7Zl6ttuGsLnscW2utb9++rX18fPTLL7+c6blWrVqljfGJiYnRHh4e\nGcYBaa31kCFDtIeHhz5+/HhaW506dbIc4/PII4/o5s2bZ2ovV65clv1F7skYHyHygGnTphkdgrAh\nd8hn4cKFWbZ1K9t69aJlpUq0LVuWlpUqsa1XL5Zt3UrhwoWd8tgAHh4ehISEsGLFCk6cOJHWfuDA\nAdatW5f2db58lsnJ6a/MXLp0iblz52Y6ZsGCBbl48WKmdk9Pz0zjkZYsWcLJkydz9R6Ec5Lp7EJY\nyR2mP7sTd8ln4cKFLdPKP/4YrXWuxt048tgAI0eO5Ntvv+Wpp56iZ8+e3Lx5k2nTplG7dm327NkD\nQMuWLcmfPz9t2rShR48eXLlyhf/85z+UKlWKM2fOZDheYGAgM2bMIDo6Gn9/f0qWLElQUBBt2rTh\ngw8+ICIigieffJK9e/fy5Zdf4ufnZ9P3I5yDFD5CCOEmbF2Y2PvYderUYd26dfTv35/hw4dTrlw5\nRo0axalTp9IKn2rVqrFs2TLee+89BgwYQOnSpenZsyfFihWjW7duGY43bNgwEhMTmTBhAleuXOHp\np58mKCiIIUOGcO3aNb766isWL15MYGAgq1evZvDgwXb9ngljqDsv7wnbUErVB+Li4uKoX7++0eEI\nIVxMfHw8gYGByO8YkRfd7+c39XkgUGtt0zUFZIyPEFYaN26c0SEIG5J8CuGe5FaXnUVFReHr60tY\nWBhhYWFGhyNy4dq1a0aHIGxI8imE8zGbzZjNZruunyS3uuxEbnUJIexJbnWJvExudQkhhBBCOIAU\nPkIIIYRwG1L4CGGlpKQko0MQNiT5FMI9SeEjhJUiIiKMDkHYkORTCPckhY8QVhoxYoTRIQgbknwK\n4Z6k8BHCSjJzxrVIPoVwT1L4CCGEEMJtSOEjhBBCCLchhY8QVpo1a5bRIQgbknyK9ObOnYuHhweJ\niYlpbc2aNSMoKMjAqIzTtWtXKleunKPXNmvWjGeeecbGEdmOFD52FhUVhclkwmw2Gx2KyKX4eJsu\nHioMJvkU6SmlMu3ErpTCw8M9/0xm9f3Izmtzymw2YzKZiIqKyvEx7kf26rKzmJgYGUTpIj755BOj\nQxA2JPkU97N+/XqjQ3A7qftaptuywuak8BFCCCGykC+f/Il0Re55DU8IIYRTu3r1Kv369aNy5cp4\neXlRqlQpWrZsya5duzL027ZtG61ateKhhx6iUKFC1KtXjylTpqQ9v3fvXsLDw/Hz88Pb25syZcrQ\nrVs3zp8/f98Y7hyrsmnTJjw8PFiyZAnR0dGUL18eb29vgoODOXz4cKbXf/LJJ/j5+eHj48MTTzzB\n5s2brR7/4uHhQZ8+fVi6dCkBAQH4+Pjw5JNPsm/fPgBmzpxJ1apV8fb2JigoKMPYpFRLliyhQYMG\n+Pj4UKJECV577TVOnTqVqd+KFSuoXbs23t7e1K1blxUrVmQZk9aayZMnp/UtXbo0b775JhcvXrzv\n+3EmUs4KIYRwOj169GD58uX07t2bmjVrcu7cOTZv3syBAwd45JFHAMutqOeff56HH36Yfv36Ubp0\naQ4cOMCqVavo06dPWp+jR48SERFB6dKl+fXXX5k5cyb79+9n69at94zhbmNVxo4di6enJwMGDODS\npUuMGzeOV199NcPxpk+fTu/evXn66afp378/x44do127djz44IOUL1/equ/BDz/8QGxsLJGRkQB8\n+OGHtGnThoEDBzJ9+nQiIyO5cOEC48aNIyIigg0bNqS9du7cuURERPD4448zduxYzp49y+TJk/np\np5/YuXMnRYoUAWDdunV06NCB2rVrM3bsWM6dO0d4eDjlypXLFE/37t2ZP38+ERER9O3bl6NHjzJ1\n6lR27drFli1b8PT0tOp9GU5rLQ87PID6gI6Li9NGW7xY6/h4o6PI+55//nmjQxA2lNfzGRcXp7Pz\nO+bvv7WOi7Pv4++/bff+ihYtqnv37n3X52/fvq0rV66sq1Spoi9fvnzXftevX8/UtnDhQu3h4aE3\nb96c1jZ37lzt4eGhjx8/ntbWrFkzHRQUlPb1xo0btVJKBwQE6Fu3bqW1T5kyRXt4eOhff/1Va631\njRs3dPHixfUTTzyhb9++ndZv/vz5WimV4Zh3o5TS3t7eOjExMa3ts88+00op/fDDD+u/032zhwwZ\nkiH2mzdv6lKlSul69erpf//9N63fqlWrtFJKjxgxIq3tkUce0WXLltVXrlxJa9uwYYNWSunKlSun\ntf34449aKaUXLlyYIc5169ZppZQ2m813/b5l5X4/v6nPA/W1jf8+yxUfF6c1jB17ifj4CJo3H8y0\naY9Ro4bRUeVNvXr1MjoEYUPuls+EBLDTWNE0cXFgq7kcRYsWZdu2bZw+fZoyZcpken7nzp0cO3aM\njz/+mMKFC9/1OAUKFEj7/3///ZerV6/y+OOPo7UmPj6exo0bZzu2iIiIDFc3mjRpgtaaI0eOUKtW\nLXbs2MG5c+cYN25chllhL7/8Mv369bP6PMHBwRmuDj3++OMAdOjQAR8fn0ztR44coUKFCuzYsYM/\n//yTUaNG8cADD6T1a9WqFTVq1GDVqlUMHz6cM2fOsHv3boYMGUKhQoXS+jVv3pxatWpx7dq1tLal\nS5dStGhRmjdvzrlz59LaH330UQoVKsT333/PSy+9ZPV7M5IUPi5OKZg9+wStWiXw3XcNqVmzA+3a\njWbSpOrkcIkGt9WyZUujQxA25G75rFHDUpjY+xy2Mn78eLp27Ur58uUJDAykVatWdO7cOW1tmcOH\nD6OUIiAg4J7HuXDhAiNGjGDRokX8+eefae1KKS5dupSj2O68VfXggw+mnQvg+PHjKKXw8/PL0M/T\n05NKlSrl+Dy+vr4AmW5D+fr6orXOdP5q1aplOmaNGjXYsmVLWj8Af3//TP2qV6/Ozp07074+dOgQ\nFy9epGTJkpn6KqUyfG+dnRQ+bqBevQASE/cwe/YXDBo0jBUrAli5MoLXXhvOmDFlefhhoyMUQtib\nj4/trsY4QmhoKE2bNuXrr79m3bp1TJw4kXHjxvH1118TEhKSreP8/PPPDBw4kHr16lGoUCGSk5MJ\nCQkhOTk5R7HdbSyLtgxzsJm7ncdR508vOTmZUqVK8dVXX2V5nhIlStjt3LYms7rchKenJ2+80ZVT\np35j7NgJeHsvZ/58fypUGESvXudJd+VSCCGcQqlSpXjzzTdZvnw5R48epVixYkRHRwPg5+eH1jpt\nllNWLl68yH//+1/effddhg0bRtu2bWnevHmOVyS2VsWKFdFa8/vvv2dov337NseOHbPrudOf/+DB\ng5meO3jwIBUrVkzrB5arOVn1S8/Pz49z587x5JNP8swzz2R61KlTxw7vxD6k8HEzXl5eDBoUxenT\nRxgwYAAeHp8wa9Z/qVwZRo6Ey5eNjtB53W2Kp8ibJJ/OKzk5mct3/DIqXrw4Dz/8MP/++y8A9evX\np3LlykyePPmut6xSr4zceWUnJiYmV6sL30+DBg0oVqwYn3/+eYZzL1iwIO12lD01aNCAkiVLMmPG\nDG7evJnWvmbNGg4cOECbNm0AKF26NI888gjz5s3jypUraf3Wr1/P/v37MxyzY8eO3Lp1i1GjRmU6\n3+3bt3N829AIcqvLTRUpUoTx40cxYEBvtC7O+PEwdixMnQqDBkFkpOXSuPgfs9lMu3btjA5D2Ijk\n03lduXKFcuXK0aFDh7TbU+vXr2fHjh1MmjQJsIwrmT59OiaTiUceeYTw8HDKlClDQkIC+/fvZ82a\nNRQuXJimTZsyfvx4bty4QdmyZVm3bh3Hjh2z622h/PnzM2LECPr06UNQUBAdO3bk2LFjzJkzB39/\nf7sWXWBZeDF1invTpk0JCwvjzJkzTJkyhSpVqmQYYD1mzBjatGlD48aNiYiI4Ny5c0ybNo3atWtz\n9erVtH5NmzalR48ejB07ll27dtGyZUvy58/Pb7/9xtKlS5kyZQovvviiXd+XrcgVn2xQSvkqpX5R\nSsUrpfYopV43OqbcKlGiBCVLKiZOhN9/h9BQGDIE/P3h00/hxg2jI3QeixYtMjoEYUOST+fl4+ND\nZGQku3fvZsSIEfTv359Dhw4xffp0+vbtm9avZcuWfP/991SvXp1Jkybx9ttv89///heTyZTWx2w2\nExISwqeffsqQIUMoUKAAa9assXovqqz277KmX2RkJFOmTOGPP/5gwIABaWvy+Pr64uXlZdV5szrX\nvdrT69KlC4sWLeLmzZsMHjyYzz//nPbt2/Pjjz+mreEDEBISwpIlS0hOTmbIkCGsWLGCuXPnEhgY\nmOmY06dP57PPPuOvv/5i6NChDBkyhI0bN9K5c+dMs+PsXdzlhrJn1etqlCWTBbTW15VS3sCvQKDW\nOtO1S6VUfSAuLi4uz+3VdeSI5bbXF19AhQqaESMUr74Ksnq7EM4jdS+jvPg7xl1prSlRogTt27dn\n5syZRodjqPv9/KbbqytQa23THYXlik82pKyrdD3lS++U/zpvWZtDVarAvHmwbx+UKLGA8PBg/P1/\nYckSyOEkCCGEcCupY5HSmzdvHufPnycoKMiAiEQqKXyyKeV21y4gEZigtb7/hi95VK1a8OGHpalS\n5TTHjzekY8dQAgIOsmqVZWFEIYQQWfv555+pX78+Y8aM4bPPPqNHjx688cYb1K1blw4dOhgdnltz\nm8JHKdVEKRWrlDqplEpWSpmy6BOplDqqlPpHKfWzUuqxO/torS9prR8BKgOvKKXyzuIFOdCiRQt+\n+20Pc+bMoWTJbSQkBNCmTXcaNDjJxo1GR+dY4eHhRocgbEjyKeypUqVKVKhQgalTp9KnTx+++eYb\nunbtyoYNG2TXd4O5TeEDFAR2AT2x7P+RgVKqE/ARMBx4FNgNrFVKFc/qYFrrv1L6NLFXwM7C09OT\nrl27cvz4b3z00QSKFFnOzp3+BAUNIijoCtu3Gx2hY7jbSr+uTvIp7KlixYqsWLGCU6dOcf36dU6d\nOsXnn39O8eJZ/kkRDuQ2hY/W+lut9TCt9UqyHpcTBczUWs/XWicAbwLXgIjUDkqpkkqpQin/7ws0\nBTKvEOWivLy86N8/ij/+OMLQoQMoXXoZZ8548Pjj0LYt7NljdIT2FRYWZnQIwoYkn0K4J7cpfO5F\nKZUfCAS+S23TluluG4BG6bpWBH5USu0ENgEfa61/vdexW7VqhclkyvBo1KhRpsXT1q1bl2EKZqrI\nyEhmzZqVoS0+Ph6TyURSUlKG9uHDhzNu3LgMbYmJiZhMJhISEjK0T506lQEDBmRou3btGiaTic2b\nN2doN5vNGW4LFClShA8+GEWTJvWJjl7P/PmWgdCPPAJBQeto3jxvvI9UnTp1ytP5kPch70OIvMxs\nNqf9bSxdujQmk4moqCi7nc8tp7MrpZKBdlrr2JSvywAngUZa623p+o0DmmqtG2V9pHueI89OZ8+J\nmzdh9mz44AM4cwbCw2HYMLhjjz0hhI3IdHaRl8l0dpHn5c8PPXrAoUMwfjysWAF+flfp1w/OnjU6\nOtu481/6Im+TfArhnqTwsUgCbgOl7mgvBZzJzYGjoqIwmUyYzebcHCbP8PaG/v3h8GFNuXImpk0L\npmLFXxg6FBywRY1djR8/3ugQhA1JPoVwPqm3vex5q0sKH0BrfROIA5qntqWs0twc+Ck3x46JiSE2\nNtbtBlIWLgyTJ/ejatUz/PtvQ8aN60DFigf58ENIt/1LnrJw4UKjQxA2JPkUwvmEhYURGxtLTEyM\n3c7hNoWPUqqgUqqeUuqRlKYqKV+njkKZBLyhlOqslKoBzAB8gLkGhJvnKaUwmUzs27ebOXPmUKrU\ndq5eDeD997tTqdJJJk+G69fvfxxn4iO7troUyacQ7sltCh+gAbATy5UdjWXNnnhgJIDWejHwDjAq\npV9dICRlvZ4cc7dbXXdKXQPo8GHLGkC+vsu5eNGf/v3fp2pV+Pxzy8BoIYRIb+7cuXh4eJCYmHjf\nvpUqVSIiIuK+/XLDEefICQ8PD0aNGpX2dXa+b87IEbe63Gb5SK31Ju5T6GmtPwU+teV5Y2JiZMYF\nljWAoqKi6NatGxMnTuTs2Ztcvgzdu8O4cTBqFLz0Eni4UykuhLgra3dPT+1rb86823h62fm+OaOw\nsDDCwsLSz+qyOfkzIxyqSJEijBo1ipkzx2A2w+7dlj3BXnkF6tWDlSuddx+wO9d1EXmb5NO5de7c\nmX/++YcKFSoYHUqeIt+3+5PCRxiqbl2IjYWtW6FUKWjXDh5/HNavd74CSH6RuBbJp3NTSvHAAw8Y\nHUaeI9+3+5PCx87cfYyPtZ54AjZsgO++A09PaNnyIEFBsGWL0ZH9T+/evY0OQdiQ5NO53W2syujR\noylfvjwFCxakefPm7N+/P8vXX7p0iX79+lGhQgW8vLyoWrUq48eP585FeydOnEjjxo0pXrw4Pj4+\nNGjQgGXLluU47oULF9KgQQOKFCmCr68vdevWZcqUKZlii4qKonLlynh5eVG+fHm6dOnC+fPnAbh5\n8ybDhg2jQYMGFC1alEKFCtG0aVM2WrEzdFbft0qVKmEymdiyZQuPP/443t7e+Pn58cUXX2R6/Z49\ne3j66afx8fGhfPnyREdHM2fOHIeNG5IxPi5AxvhkzzPPgNl8HH//2sTFPc1TT42hVavHGD0aHn3U\n6OiEEI6S1ViV999/n+joaNq0acNzzz1HfHw8LVu25OYdMyT++ecfmjZtyunTp3nzzTcpX748P/30\nE++++y5nzpxh0qRJaX2nTJlC27ZtefXVV7lx4wYLFy6kY8eO/N///R/PPfdctmJev349L7/8Mi1a\ntEhbJ+rAgQP89NNP9OnTB4C///6bp556ioMHD9KtWzceffRRkpKSiI2N5cSJEzz00ENcvnyZ2bNn\nExYWRvfu3bly5QqzZs3i2WefZfv27dStWzdb3zelFIcOHSI0NJRu3brRtWtXZs+eTXh4OA0aNKBm\nzZoAnDp1iqCgIDw9PRk6dCg+Pj785z//4YEHHnDYuCFHjPFBay0POzyA+oCOi4vTInuSk5P1ypUr\ndUBAgAZ0oUIdNCTo0FCtDxwwOjohnENcXJx25d8xc+fO1R4eHvr48eNaa63/+usvXaBAAW0ymTL0\nGzp0qFZK6fDw8LS2Dz74QBcuXFgfPnw4Q993331X58+fX584cSKt7fr16xn63Lp1S9epU0cHBwdn\naK9UqVKGc2SlX79+umjRovfsM2zYMO3h4aFXrlx51z7Jycn65s2bGdouXbqkS5curV9//fUM7Uop\nPXLkyLSv7/y+pcbu4eGht2zZktb2119/aS8vLz1gwIC0tt69e2tPT0+9Z8+etLYLFy7oYsWKZTpm\nbt3v5zf1eaC+tvHfZ7nVJZxO6hpAu3fvZu7cuTz00HY8PAJYvbo7tWqdoGtXOHbM8XHduZGlyNvc\nMZ+nT58mPj7+ro+73TZKb//+/Vm+9vTp03aNfcOGDdy8eTPTLcp+/fpl6rt06VKaNGmCr68v586d\nS3s0b96cW7du8cMPP6T1LVCgQNr/X7x4kQsXLtCkSRPi47O/PVTRokX5+++/Wbt27V37LF++nHr1\n6mW5WW0qpRT58lluyGituXDhAjdu3KBBgwY5igugVq1aPPnkk2lfFy9enOrVq3PkyJG0trVr19Ko\nUSPq1KmT4T298sorOTqns5LCx85kjE/OeXp60qVLFw4ePMhHH03Ey2s5+fJV5euv51CtGkRGgp1/\n12YwcOBAx51M2J075nPmzJkEBgbe9REaGnrfY4SGhmb52pkzZ9o19uPHjwPg7++fob148eI8+OCD\nGdoOHTrEt99+S4kSJTI8WrRogVKKP//8M63v//3f/9GoUSO8vb156KGHKFmyJNOnT+fSpUvZjrFn\nz55Uq1aNVq1aUb58ebp165apCDp8+DC1a9e+77HmzZtHvXr18PLyolixYpQsWZJVq1blKC7IejD/\ngw8+yIV0ewkdP3480/cXMn/P7UnG+LgAGeOTe15eXvTr14+IiAg++ugjWrSoy5YtlvV/5syBXr1g\n0CAoVsy+cUybNs2+JxAO5Y757NGjxz2vNHh5ed33GEuWLOF6FsuulylTJlex2VJycjItWrRg0KBB\nmQYzA1TpwnbiAAAgAElEQVSrVg2AH3/8kbZt29KsWTOmT59OmTJlyJ8/P7Nnz87RP1ZLlCjBrl27\nWLt2LWvWrGHNmjXMmTOHzp07M3fuXKuPs2DBAsLDw3nxxRcZOHAgJUuWxNPTkw8//DDDFZrs8PT0\nzLI9q++PkRwxxkcKH5FnFClShJEjRwLw1FPw5pswaZLlMWMGvP02REVBkSL2Ob9Mf3Yt7pjPMmXK\n5LpAqVWrlo2iyZ6KFSsClqs5lSpVSmtPSkrKcNUCwM/Pj6tXrxIUFHTPYy5fvhxvb2/Wrl2bdmsJ\nYNasWTmOM1++fLRu3ZrWrVsD8NZbb/HZZ58xbNgwqlSpgp+fH/v27bvnMZYtW4afnx9Lly7N0D5s\n2LAcx2WNihUr8vvvv2dqP3TokF3P62hyq0vkWb6+MHIkHDkCb7wBY8ZAlSowYQJcu2Z0dEIIWwoO\nDiZfvnxMnTo1Q3tWm1l27NiRrVu3sm7dukzPXbp0ieTkZMByFUQpxa1bt9KeP3bsGCtXrsxRjKnT\n0dNLHS/z77//AtC+fXt27959z3NkdXVm27ZtbN26NUdxWSskJIStW7eyZ8+etLbz58/z1Vdf2fW8\njiZXfESeV6IEfPSR5WrPsGFXGDx4GzExwbz3Hrz+OshaXkLkfcWLF+edd95h7NixtGnThlatWrFz\n5860sTzpDRgwgNjYWNq0aUPXrl0JDAzk77//Zs+ePSxfvpxjx47x0EMP0bp1ayZNmkRISAgvv/wy\nZ8+e5dNPP6Vq1aoZ/vhb6/XXX+f8+fM888wzlCtXjmPHjjFt2jQeffTRtCnjAwYMYOnSpYSGhhIe\nHk5gYCDnzp3jm2++YebMmdSpU4c2bdqwfPly2rVrR+vWrTly5AgzZ84kICCAq1ev2uT7mZWBAwey\nYMECgoOD6d27NwULFuQ///kPFStW5MKFC3l6K4z05IqPncngZscpVw6eeMJMcnILtA4mMvIXqleH\nefPg9u3cH3/cuHG5P4hwGpLPvCc6OpqRI0eya9cuBg4cyNGjR1m3bh0FCxbM8EfZ29ubH374gYED\nB7Jp0yb69evHuHHjOHz4MKNGjcLX1xeAoKAgZs+ezdmzZ4mKimLRokWMHz+edu3aZTq3NXtgvfba\na3h7ezN9+nQiIyP54osvCAsLY/Xq1Wl9ChYsyObNm3nrrbdYs2YNffv2ZcaMGdSsWZNy5coB0LVr\nV8aMGcOePXvo27cv69ev58svvyQwMDDLNXruF9e9+qRvL1euHBs3bqRWrVqMGTOGyZMn89prr9G1\na1fAujFgueWIwc2Gr3fjqg9kHR9DJCcn6xUrVqStAVS2rGUNoJo1tV6yROvbt3N+7GHDhtkuUGG4\nvJ5PV1/HRziPvn37ah8fH52cnGyzY8o6PkLYiFKKtm3bsnv37pRl1rfh6RnA5ctvEBp6ggYNYPXq\nnO0DljqwWrgGyacQmd05Y+/cuXMsWLCAJk2ayK0uIZyZp6cnXbt25bfffmPChAlcv/41DzxQFa13\n0bo1NGkCmzYZHaUQQjiXRo0aERUVxWeffcaoUaMIDAzkypUrvP/++0aHZjNS+AiX5uXlRVRUFEeO\nHGHChHHs2FGXNWvgn3+gWTMICYFffjE6SiGEcA6tW7dmzZo19O/fnwkTJlCpUiW+/fZbGjdubHRo\nNiOFj3ALRYoUoU+fPnh6evDss7BjByxdCn/8AQ0bwgsvwH2W1iApKckxwQqHkHwKkdno0aNJSEjg\n6tWrXLlyhY0bN953PaS8RgofO5NZXc5JKWjfHvbuhfnzYfduqFsXXn0Vsli/C4CIiAjHBinsSvIp\nhPNxxKwuKXzsLCYmhtjYWMLCwowORWTB0xNeew0SEqB37/WsWbOeGjWgRw84cSJj3xEjRhgSo7AP\nyacQzicsLIzY2NgsF6a0FSl8hMCyyOH58/M5f74llSsHs2jRDvz9oX9/+OsvSx/Zc821SD6FcE9S\n+AiRYv78+axcuZICBc5w6dJj+PmF8tlnB6lcGd5/Hy5eNDpCIYQQuSVbVgiRQimFyWSidevWLFiw\ngGHDhnH9egABARFMnDicTz4py8CB0Ls3FCxodLRCWBw4cMDoEITINiN/bpV2si3pXYVSqj4QFxcX\nJ5fU86jr168zY8YMRo8eTYECPlSr9h5btnTnoYdgyBDLOKACBYyOUuTUrFmz6Natm9Fh5FhiYiI1\na9bkmuzIK/IoHx8fDhw4QIUKFTI9Fx8fT2BgIECg1jrelueVKz5C3IWXlxf9+vUjIiKCX3/9lQUL\nFjBnjmVH+Kgoy8aow4ZBly6QTz5JeU58fHyeLnwqVKjAgQMHZFp+OmPHjmXw4MFGhyGsVLx48SyL\nHnuTKz52knrFp2nTpvj6+hIWFiYzu1xIQgIMHw6LF0PVqjBqFHTsCB4yak4IIXLMbDZjNpu5dOkS\nP/zwA9jhio8UPnYit7rcw86d8N57lv2/6taF0aOhTRvLOkFCCCFyxp63uuTfp0LkwqOPwqpV0KlT\nf7Rej8kETz4J//2v0ZEJIYTIihQ+QuTS1atXSUz8mb17W/Loo8FcuvQLzZtD8+bw889GRyeEECI9\nKXyEsJLJZMqyvVChQmzZsoWVK1dy48YZDhxoSKNGofzxx0EaNQKTCfbscXCw4r7ulk+Rd0lOhTWk\n8BHCSr169brrc6lrAO3evZs5c+Zw8uR2jhwJICioO3v3nqRePQgLg99+c2DA4p7ulU+RN0lOhTWk\n8BHCSi1btrxvH09PT7p27crBgweZMGECe/YsJyQkms8+g82boVYteP11SEx0QMDinqzJp8hbJKfC\nGlL4CGEHXl5eREVFceTIEaKjP+CNN+DQIZg4EWJjLVPg+/aFs2eNjlQIIdyLFD5C2FGRIkUoVqwY\nAF5e0K8fHDliWfhw3jyoUgXefRfOnzc4UCGEcBNS+AhhpRUrVtjkOIUKwdChcPSo5arPlCmWAmj0\naLhyxSanEFawVT6F85CcCmtI4SOElcxms02P9+CD0L37MQICGhMUtJ4PPgA/P4iJgevXbXoqkQVb\n51MYT3IqrCErN9uJbFkhrLFv3z66d+/O1q1bady4OcWKjWHVqscoU8ZyO6xrV8if3+gohRDCMWTL\nijxMtqwQ1tJaExsby5AhQ9i/fz8hIR3Il280q1ZVx8/PsinqSy+Bp6fRkQohhGPIlhVCuDClFG3b\ntmXPnj3MmTOH/fu38e23AbRv3x0/v1O8+io88gisWAHy7xQhhMgdKXyEcBKpawD99ttvTJgwgU2b\nvmbkyES2boVSpeCFF+Dxx2HdOimAhBAip6TwEcJK4eHhDjlP6hpAiYmJPPHEEzzxBGzYAN99Z7nd\nFRICQUGwZYtDwnFZjsqncBzJqbCGFD5CWMnRq8J6e3tn+PqZZ+CnnywLIF64AE89BW3awK5dDg3L\nZcgqv65HciqsYfXgZqVUKOB9346284/WeokDz2dTMrhZ2FNyMpjNNxkxwpPff/cgNBRGjYIaNYyO\nTAghcs+eg5vzZaPvROCILU9+H5WBPFv4CGFPHh6QmDiRIkWWERU1hqVLWxAQAJ07w/DhUKmSZbaY\nUsroUIUQwqlkp/A5r7UOslskd1BK7XTUuYTIi5o0acI333xDTExLgoKa07HjGObPD2T+/GQeLvgV\ntQpO5rbXBRo//zzvREdTuHBho0MWQgjDyRgfIay0efNmo0PI4KmnnmLLli2sWLGCs2dP89FHDeF6\ncdomR3L1yvP8eGYz9Y+9ScA0M+0bNeKK7IeRgbPlU+Se5FRYIzuFzyS7ReEc5xPinsaPH290CJmk\nXwPIFBwMVy4Qy2e0pRSRjOFTInlD/06p/e35cOAHRofrVJwxnyJ3JKfCGrJys53I4GbXc+3aNXx8\nfIwO466CK1fmm2PHmAGsBL4DzlGCsQzmU3qS7PE3YycU4623wNuR0xSclLPnU2Sf5NR15LmVm5VS\nYUqpj5VST9jj+EZRSpVTSn2vlPpVKbVLKdXB6JiE4zjzL1StNQVv3sQbiAK+BzyBkvzFJN7md/x5\n2GsNAwdq/P1hxgy4ccPYmI3mzPkUOSM5FdaweeGjlHoDaAO8heUfniilHlZKrVFKXVVK7VNKdbP1\neR3kFtBXax0AhACTlVLyb2dhOKUUf+fPT+r12zvncpXlJP4l3ychQdGsGfTsaZn6/sUXcPu2g4MV\nQggD2eOKzytAV6ATMCalbSHQEss/Qv/BUjBMt8O57UprfUZrvSfl/88CScBDxkYlhEXj559nrUfW\nH+lvPTx4ymTC3x++/BLWrz9F3brJdO4MdevC8uWyDYYQwj3Yo/C5pbW+qbX+Wms9WSlVG3gKSAaC\ntNaPAf5AI6VUMzuc3yGUUoGAh9b6pNGxCMcYMGCA0SHc0zvR0UyqWZM1Hh5pV340sMbDg5iaNXl7\n9GgAkpOT6dWrOSdONGTKlPU8/DC0bw+PPQZr17pPAeTs+RTZJzkV1rBH4XPnrZ/UtX9+1lr/DGlX\nS94Aetrh/FlSSjVRSsUqpU4qpZKVUqYs+kQqpY4qpf5RSv2slHrsLsd6CJiH5T0IN1GhQgWjQ7in\nwoULs2zrVrb16kXLSpVoW7YsLStVYluvXizbujVtHR8PDw9mzpzJAw88QJ8+LUlObs706b9QoAA8\n+yw0awbuMCvY2fMpsk9yKqxh81ldSqmvgNla6w0pX6/EMuYnRmv9zh19v9NaN7dpAHeP61ngSSAO\nWA68oLWOTfd8JyzFTHdgO5YxoqFANa11Urp+DwDrgZla66/ucT6Z1SUMdb+Vm7XWxMbGMmTIEPbv\n30+HDh0IDh7NjBnV2bULnnsORo8G+fEVQjhaXpvVNRFYqJQaopT6EHg+pf3bLPo6bFil1vpbrfUw\nrfVKMo/9BEuhM1NrPV9rnQC8CVwDIu7oNw/47l5FjxDO4H7bVaRfA2jOnDls27aNyMgAgoMHsmgR\nHDkCgYEQGgoHDjgoaCGEsDObFz4plVlfYHDKA2CZ1nqDsvBN1/2Wrc+fE0qp/EAglqVPANCWS2Eb\ngEbp+jXGchWonVJqp1IqXikVcK9jt2rVCpPJlOHRqFEjVqxYkaHfunXrMJky3X0jMjKSWbNmZWiL\nj4/HZDKRlJSUoX348OGMGzcuQ1tiYiImk4mEhIQM7VOnTs10P/zatWuYTKZMq5+azWbCw8Mzxdap\nUyd5Hy7wPjw9PenatSuLFi2iRo0alCpVko4dYd8+mD0b1q8fTkDAOLp2haNHnfd9pMrr+ZD3Ie/D\n3d6H2WxO+9tYunRpTCYTUVFRmV5jK3ZbwFApVQxoDFzRWn+f0rYMaAeMBaYBY7XWXewSwL1jSwba\npd7qUkqVAU4CjbTW29L1Gwc01Vo3yvpI9zyH3OpyMQkJCdRww+3P//0XPv8coqPh3Dl4/XV47z14\n+GGjI8sdd82nK5Ocuo68dqsLAK31Oa11bGrRk8ITy22m/sACIMZe5xfC1gYOHGh0CIYoUAB69YLD\nhy1jfhYtAj8/GDjQUgjlVe6aT1cmORXWcOiWFSmzoXoDxYAFWuvtDjt5xjjuvOKTH8t4nvZ3DHie\nC/hqrV/IwTnqA3FNmzbF19eXsLAwwsLCbPMGhCESExNl1giwatUmtm1rQkyMB0pB//6WR5EiRkeW\nPZJP1yM5zfvMZjNms5lLly7xww8/gB2u+LjlXl13Fj4pbT8D27TWfVO+VkAiMEVrPSEH55BbXcLl\n7Nu3jzp16hAYGMigQWPYtq0Fn3wCBQvC4MEQGSn7gAkhcs8pbnUppQra8sSOPp9SqqBSqp5S6pGU\npiopX5dP+XoS8IZSqrNSqgYwA/AB5toyDiHystq1a7Np0yYeeOABOnZsya5dwSxd+guhofDuu5Zb\nYNOnyz5gQgjnlZ0xPj/aLQrHnK8BsBPLOj4a+AiIB0YCaK0XA+8Ao1L61QVCtNZ/5eakUVFRmEwm\nzGZzbg4jhNNo2rQpW7ZsYeXKlZw+fZo2bRqSlBTK6tUHCQ62XPWpUQPmz5d9wIQQ2ZM6w8ues7rQ\nWlv1AHZa29cWD0efzw7x1wd0XFycFq5h7NixRofgdG7duqXnzJmjy5cvrz09PfXcuXP13r1av/CC\n1qB1zZpaL1mi9e3bRkeameTT9UhOXUdcXJzGcpGivrbx3+d82aiRvJVSnW1deN2DlwPPJcR9Xbt2\nzegQnE7qGkAvvfQS06dPJygoiAoVLJue/vKLZdp7aKhl9efoaAgJgfusq+gwkk/XIzkV1rB6cHPK\nGjyO3In8vNa6vQPPZ1MyuFm4I53FNhmbNsHQobBlCzz1lKUAatrUoACFEHmCPQc3u+WsLkeQ6ezC\nnaX/vaKUQmv49ltLAbRzp+XKT3S0ZUsMIYRIJdPZ8zC54iNExitAJ06cYPPmn/Dw6MDw4R4kJMCL\nL8IHH0CtWgYHKoRwKk4xnV0Id3fn3jbi/tLf9lq2bBlhYZ0YP74hMTHrmTsX4uOhdm3o3NmyKaoj\nST5dj+RUWEMKHyGsFBERYXQIeVrfvn3T1gB67rmWfPFFMF9++QtTp8L69VC9OvTsCadOOSYeyafr\nkZwKa0jhY2eyjo/rGDFihNEh5HmpawCtWLGC06dP07hxQzZuDGXNmoNER/9vH7ABA8De/3iXfLoe\nyWne54h1fAwf46OUqgQ0A5Zorf82NBgbkjE+Qtzb7du3WbBgAcOGDePkyZMkJCRQooQ/kybBpEnk\n6X3AhBC541JjfJRS3kqp4qlfa62PAduB95RSjRwdjxDCGJ6ennTp0oWDBw+yePFi/P398fWFkSPh\n6FHo3h3GjYPKlWHCBJAlWoQQtuDQwkcp9TpwATirlDqjlJqnlGoD/Ka1fhd4xZHxCCGM5+XlxYsv\nvpihrXhxmDgRfv8dOnaEIUPA3x8++UT2ARNC5I6jr/i8DLwGvAjMBGoCK4EzSqklQG0HxyOE1WbN\nmmV0CG6nbFnLpqcHD0JwMPTubRkEPW9e7vcBk3y6HsmpsIajC58dWuslWuuVWuvhWuuGQCUgGjgF\n9HJwPHYng5tdR3y8TW8zi2z48cd5tGmzmN27k6lfH7p2tUyDX7oUkpNzdkzJp+uRnOZ9eXZwc8pY\nnUeBLcAenXISpdRY4H2t9U2bn9TJyOBmIWzntddeY8GCBQQGBjJmzBgefLAF770Ha9da9gEbPRqe\nfdZ59gETQuROXhzc7AdMA+KBi0qpdUqpEUAC8IlSysdO5xVCuKAvvvgibQ2gli1bMnhwMB988Aub\nNoGPD7RqZdn/y7LCvRBC3J29Cp+/gPlAI2AU8DcQCcwGXgf2K6WilVItlVIF7RSDEMKFpF8D6MyZ\nMzRs2JCpU0P5/PODrFkDf/8NTz9t2Qdsxw6joxVCOCt7FT67gPla6+1a64+01i9orUsAAUAP4Acs\nA52/BS4opbYrpSYqpZ6wUzxCCBeglKJt27bs3r2buXPnsn37djp0aE9IiGbHDliyBBIT4bHHoH17\n2L/f6IiFEM7GLoWP1vqs1vq/WbQf0Fp/rrXurLWuDFQAugI7gBDgS3vEI4QtmEwmo0MQKdKvAbR8\n+XKUUnh4QIcOsG8fVu0DJvl0PZJTYQ0jFjCsmzrGR2t9Qmv9FTBFa11Ha+3n6HjsTWZ1uY5evVxu\n0mGe5+XlRbVq1TK0eXpCly6WKfDTpsGGDZYp8G+9BSdP/q+f5NP1SE7zvjw7q+uuJ1NqMDAC+Flr\n3Sxd+9PAc1rrwQ4Lxs5kVpcQzuHaNcvCh2PHWv4/MhIGD7YskiiEcE55cVbX3TyI5dZWhssfWutN\nQLxSqrWD4xFCuLDbt2/z6qsvUrHiYn7/PZlBg2DmTKhSBYYPh0uXjI5QCOFoji588mutF2qtZ975\nhNZ6MdDCwfEIIVzYxYsXuX79Op06daJFi4Y0bryeo0ehRw8YP95SAMk+YEK4F0cXPsWUUvcaxyO7\n8AintWLFCqNDENlUrFgxVq9ezcaNG9PWAHrppWA6ddrBtGkr6NTpf/uAffqp7AOW18lnVFjD0YXP\nx8CGe9zSKuTIYITIDhmgnnc9/fTTaWsAnT59mscee4wRI3rTt+/BtH3AevWy3T5gwhjyGRXWcGjh\nkzJA6T1guVJqr1LqA6VUe6VUC6XUSKTwEU5s0aJFRocgciF1DaA9e/YwZ84clFIsWbKEKlVg/nzY\nu5e0fcDq1MndPmDCGPIZFdZw6KyutJNaZjyNA54BUnfXWQ+8pLW+4PCA7CB1VlfTpk3x9fUlLCyM\nsLAwo8MSQqS4fv06Wmu8vb0ztO/YgewDJoRBzGYzZrOZS5cu8YNlDxqbz+oyqvBRQHEsRU8l4E+t\n9TGHB2JHMp1diLzthx9g6FDYvBmeegqioy37gQkh7M9lprMrpR5QSk3DsnfXGeA3oBsyqFkI4WSa\nNoWNG5NZvfp/+4A9+6zsAyZEXufowc0TgCJAP2AQsA54FdirlJJ/SwmnFh4ebnQIwobul8/t27dT\nq1ZNrlxZzPbtySxZAsePyz5gzkw+o8Iaji58Cqfs0/WZ1nqi1rojUBr4ADArpSo5OB4hrNayZUuj\nQxA2dL98Fi5cGH9/fzp16sQTTzTE13c9e/datw+YMIZ8RoU1HF34nL+zQWt9RWs9GXgReN/B8Qhh\nNRmc7lrul8+aNWuyatUqNm3alLYG0LPPBlOr1i9p+4CtX2+ZAt+zJ5w65aDAxV3JZ1RYw9GFz0NK\nqcpZPaG13gZcd3A8QghxT02bNmXLli2sXLmSM2fO0LBhQ155JZTWrY9z+DB8+CEsWgR+fjBgACQl\nGR2xEOJeHF34TMWygGGruzwvE0aFEE5HKYXJZGL37t3MnTuXXbt2cfv2bXx8LMXOkSMwaBDMmGHZ\nBmPECLh82eiohRBZcfQChjuBYcDXsoChyGs2b95sdAjChnKST09PT7p06cLBgwepUqVKWruvr6XY\nOXoUuneHceOgcmXZB8zR5DMqrOHoKz5orb8EGgN/AkOBJcBa4Amgr6PjEcJa48ePNzoEYUO5yaeH\nR9a/OosXh4kT4fffoWNH2QfM0eQzKqzh8MIHQGu9Q2vdHCiFpeCporUOcZVVm4VrWrhwodEhCBuy\nZz6LFLnClCk3OXgQmjf/3z5g8+fLPmD2JJ9RYQ2HFz5KqceVUuOUUlOAUOAPV1u1Ob2oqChMJpNs\nnucCfHx8jA5B2JA98zlo0CACAgLYsWMx8+Ylp+0D1qWLZR+wZcvAgEXzXZ58RvM+s9mMyWQiKirK\nbudw6JYVSqm3gMlAElAS8ARuAZ8C72qt/3FYMHYmW1YI4b52797NkCFDWL16NYGBgYwZM4YWLVqw\nY4dlG4x16yyFUHQ0hITIPmBC3MlltqzAsilpCa11WSwDmYOAj7Cs4fOtUsrLwfEIIYTN1atXL9Ma\nQMHBwcAO1q6FjRvB2xuee86yNcaPPxodsRDuw9GFz2Gt9WUArfW/WutNWut3garAfmQBQ+HEBgwY\nYHQIwoYckc/UNYBWrFjB6dOneeyxxwgNDaVhw3/48UdYtcqyD1jTppYiKC7O7iG5NPmMCms4uvA5\no5SqfWdjShH0FpbtK4RwShUqVDA6BGFDjsqnUoq2bduyZ88eZs+eTeHChfH29kYpaNXKsunp4sWW\nqfANGkCHDrIPWE7JZ1RYw9FjfIpi2Zh0FvCd1vr3O56fprXu5bCA7EjG+AghsuPWLfjiCxg5Ev74\nA159FYYPtyyIKIS7caUxPouAokAMcFApdVIpZVZK9VFKLcCynk8apdSbDo5PCCEMkS8fhIfDwYPw\n8cewdq3sAyaEPTi68Dmkta4GFMGyiGEM4INlNeeXgXlKqdVKqaFKqeZATwfHJ4QQhjp0aB89etzk\n8GEYPRoWLrTsAzZwIJw7Z3R0QuR9ji58vldKTQBaAXFa64la67Za6+JAbeBdLDu4dwfWAwEOjk+I\nu0pISDA6BGFDzpjPGzduEBwcTEBAAKtWLWbAgGSOHLHsBzZ9umUbjJEjZR+wu3HGnArn4+i9upYB\nQ4DLwIN3PLdfaz1Ta/2q1roi4A8kOjI+Ie5l4MCBRocgbMgZ8/nAAw+wdu1aqlatSqdOnWjYsCG/\n/LKeUaMsG6G+8QaMGWMZ9/PRR/CPy6x8ZhvOmFPhfIzYssJba71Ra/1nakPKQOAMtNZHgDEOjUyI\ne5g2bZrRIQgbctZ83m0NoGPHfuGjjyz7gHXoAIMHW/YBmz5d9gFL5aw5Fc7FoYWPUioaMCulFiml\n0p/bUyn18Z39tdafOS46Ie5Npsq6FmfP551rADVs2JC3336bcuVgxgxISICgIIiMhBo1LDPC3H0f\nMGfPqXAOjr7i85DWujXwFdA2tVFr/QuwXinV0cHxZJtSarlS6rxSarHRsQghXFv6NYDmzp1Ls2bN\n0p7z84MFC2DPHqhXDzp3hrp1Yfly2QdMiHtxdOGTOifhG+Dx9E9orf8Py0wvZzcZeM3oIIQQ7sPT\n05MuXbrw/PPPZ3qudm34+mvYtg3KloX27eGxxyzT4aUAEiIzRxc+pZVShbXWyUBW2/Jdd3A82aa1\n/gG4anQcwvHGjRtndAjChlwtnw0bWjY//f57KFAAnn0WmjWDzZuNjsxxXC2nwj5yVPgopYYppV5V\nSjVQShXKxkvnAKuVUjXv8rxsWSGc1rVr14wOQdiQq+UzOTmZhQsX0rjxTTZvhv/7P8u09yZNLFtj\nxNt07Vvn5Go5FfaR0ys+I4B5wAZgv1LqkFJqvVKq2r1epLXegmX15t3AS0qpsUqpHkqpSKVULJZp\n7nahlGqilIpNWS06WSllyqJPpFLqqFLqH6XUz0qpx+wVj8h7Ro4caXQIwoZcLZ/btm3j5ZdfJiAg\ngCVLFvPcc8nExVn2ATtyBAIDITQUDhwwOlL7cbWcCvvIza2ut7TWRbXWFbTWVYFOwOH7vUhrPQ1o\nDvwGvA1MBz4EDgH9cxHP/RQEdmFZDTrTnW+lVCfgI2A48CiW4mytUqq4HWMSQgibaNSoETt37syw\nBjAT5rMAACAASURBVNB3360nNBT27YPZs2H7dsuYoPBwOHbM6IiFMEZOC58/7pxqrrU+r7W2ajKl\n1vpHrXULwBsog2W219ta65s5jMeac36rtR6mtV5J1uOLooCZWuv5WusE4E3gGhCRRV91l2MIIYRh\n7rYG0M6dvxAeDr/9BpMnw+rVUK0a9OoFp08bHbUQjpXjwscWJ9da39Jan7W2YLIXpVR+IBD4LrVN\nW7at3wA0uqPveiy3655TSiUqpTLMTrtTq1atMJlMGR6NGjVixYoVGfqtW7cOkynT3TciIyOZNWtW\nhrb4+HhMJhNJSUkZ2ocPH55pcF9iYiImkynTUu5Tp05lwIABGdquXbuGyWRi8x2jIc1mM+Hh4Zli\n69Spk1u9j6SkJJd4H+Aa+cjt+0hKSnKJ9wGZ85G6BtAbb7zBrl27aNiwIWazmQIFoFu3azRoYCIi\nYjNffmmZFj94MHz+ufO9j1TW5iMpKckp85Hd9wHO+XNlr/dhNpvT/jaWLl0ak8lEVFRUptfYitI5\nmO+olPqv1voZO8TjEEqpZKCd1jo25esywEmgkdZ6W7p+44CmWutGWR/pnueoD8TFxcVRv36mhalF\nHmQymYiNjTU6DGEj7pLP27dv8+WXX9KuXTuKFCmS4bmLFy1bX8TEgKcnvPMO9OsHhQsbFGwuuUtO\n3UF8fDyBgYEAgVprmw7NN2LLCiHypBEjRhgdgrAhd8mnp6cnnTt3zlT0ABQtCh98YBn8HBEB0dGW\nfcAmTcqb+4C5S05F7uS08KmglPKyaSTGSgJuA6XuaC8FnMnNgaOiojCZTJjN5twcRjgBuXLnWiSf\n/1OypOWqz6FD8MILMHAgVK0KM2fCTbuNvLQ9yWnel3rbyxlvdSVjWcRvM/BDymO71vpWNo8zXmvt\n8O1077zVldL2M7BNa9035WuFZXf4KVrrCTk4h9zqEuL/27v3MKvqeo/j76/chOGIxyAkClMGNe1w\nEYEwAY/oeDtu7WhN6CllCg3UajTIxAB9IIMiLS2hNC9pY9hJoCABRQe5KDqA4AXEoIMRWKRhCCoy\n3/PH2jPMMLc1M3vvtffan9fz7GfYa/322t/N95k932f9bpIT1q1bx3PPPUdJSQnt2rVj82aYMgXK\nyuDYY+GWW2DUqKA7TCQTsrWrqzNwLjANeAbYbWZLzWyKmZ1pZh1DXGNwK96/WcyswMz6mVn/5KHj\nks8/kXz+I2CMmX3ZzE4EZgGdgPszFaOISBSefPJJxo4dy8knn8ycOXPo3buShx+GdeuC6e9f+lKw\nH9jcudoGQ3JfSwuf7cB3gcXAuwRTuzsCZySPLwHeNrOVyUUKzzezuh3MkMnuslOBtUAFwTo+M4E1\nwC0A7j4H+BZwa7JdX+Acd/97a95UXV3xcejsCMltyudBN9xwQ501gJYsWULfvjBvHqxaBUcfHXSD\nDRkCS5ZkZwGknOa+THR14e7NfgBP1/j3YQRFxfXAPIKNSCtrPA4kH/sJCo07gM8BvYAdLXn/XHgA\npwBeUVHhEg/jxo2LOgRJIeWzfuXl5T506FAHfOTIkb569erqc0884f6Zz7iD+xlnuK9YEWGg9VBO\n46OiosIJblKc4in++5yW6exm9h/A8BqPmoOGa72hu8ey11hjfEQkV7k78+fP56abbuK1115j27Zt\n9OjRI3ku2Afs5pth/Xq44AKYOhX692/ioiLNkI1jfBrdmNTdN7j7T9292N17ACcCVwEPESx+qJWP\nRUSylJlx0UUXsX79epYtW1Zd9ATn4MILYe3aYPDza6/BgAFQXAybNkUYtEhILb3js93de7b4Tc2O\nAS4Dpsb9js/w4cPp0qULo0aNYtSoUVGHJSKSUh9+CA88EMz82r4drrgCJk+GY46JOjLJRWVlZZSV\nlbF7926WLVsGabjj05rp7EPc/flWvbnZ6+5e2JprZCt1dYlIPnnvPZg1C773Pdi9G66+GiZOhO6H\nro4mEkI2dnUB3G1mjXZ5hfC3Vr5eJGPq2wtHcpfymRrTpk1j9uzZtGmzn29+M1gFetIkePDBYBXo\nm26Ct9/OTCzKqYTR0sLnJqADsMHMSkKu2VOfD1r4OpGMu/baa6MOQVJI+Ww9d2fLli211gDq1KmS\niRNh61b4xjfgxz8OFkGcNg327ElvPMqphNGirq7qFwezt0YBFwAvADe5+5vNeP1kd7+lxQFkMY3x\nEZF88eKLL3LTTTexcOFCBg4cyG233cbZZ58NwM6dcNttQTdYly5B99fVV8Phcdr0SFIma8f41Hsh\ns6HAJnd/KyUXzHEa4yMi+WbZsmXceOONrFq1ipEjR3LbbbcxaNAgALZtg1tvhfvvhx49ggHQV14J\nbdtGGrJkqWwd41OLu69S0SMikr+GDx/OihUrmDt3Ljt27GDChINbMfbqBffcAy+/DJ/9LIwZAyed\nBI88ApWVEQYteSdlhY9I3M2dOzfqECSFlM/0qLkG0COPPFLn/AknBMXO2rVw/PHB5qcDBsDvf9/6\nbTCUUwlDhY9ISNpvLV6Uz/Rq06YN3RuZy96/f7AC9IoVcNRRkEjAaafB0qUtf0/lVMJI2RgfqU2D\nm0VEGufumBnu8OSTwdT355+HkSODWWBDhkQdoWRaTg1ulto0uFlEpGHvvvsuI0aMYMyYMZSUlNCu\nXTvcYf78YObXyy8HW2NMnQp9+0YdrWRaTgxuFhERCWvv3r2ccMIJtdYAcq/koovgxRfhoYeC4qd/\nf7jsMti8OeqIJS5U+IiISMZ169aNhx9+mLVr19KnTx+Ki4sZPHgwS5YsoU0buPxy2LgR7r4bli2D\nT30KrroK3ngj6sgl16nwEQlp9OjRUYcgKaR8Zod+/fqxYMECysvLad++PUVFRZx11lmsWbOGdu2C\nxQ43b4YZM+Cxx6BPHygthb/Vs+GRciphqPARCamoqCjqECSFlM/sUrUG0Lx589ixYwcbNmyoPtex\nI1x/fbAP2MSJ8MtfBvuA3Xwz/POfB6+hnEoYGtycJprVJSLSMgcOHACCKfH1+cc/gjtAd94JHTrA\nt78N110HBQWZjFLSQbO6cphmdYmIpNeOHcG095//PFgL6OabgxWhO3SIOjJpLc3qEhERSXJ39u3b\nR48ecNdd8NprcN55wW7wxx8P990HH34YdZSSrVT4iIS0fPnyqEOQFFI+c9djjz1G7969mT17Nvv3\n7+eTnwyKnQceWM6QIVBSAp/+NMyZo33ApC4VPiIhzZgxI+oQJIWUz9w1YMAAzjzzzFprAFVWVjJn\nzgzmzIGKimDwc3ExDBwICxe2fh8wiQ8VPiIh1bfhouQu5TN3HXvssTz00EN11gC66qqrADjllKDY\nWbYMOneGCy6A00+H8vKIA5esoMJHJKROnTpFHYKkkPKZ+w5dA+jCCy/krLPOYuvWrQAMGxYUP3/8\nI7z3HpxxBpxzDrzwQrRxS7RU+KRZaWkpiURCuwaLiKRJ1RpAc+fO5f333+fII4+sPmcG554bFDu/\n/W2w8vOgQXDJJfDKKxEGLfUqKysjkUhQWlqatvfQdPY00XR2EZHsc+AAPPwwTJ4M//d/8D//A1Om\nBGOCJHtoOrtIFhg/fnzUIUgKKZ/xEyanbdrAl78MmzYFU+GfeAJOOAHGjYO//jUDQUrkVPiIhNSr\nV6+oQ5AUUj7jJ0xOd+zYwd69e2nfPih2Xn8dvvc9+M1voHdvGD8edu3KQLASGRU+IiFdd911UYcg\nKaR8xk+YnI4bN47CwsLqNYA6dQqKnS1bYMIEmDUr6Pa65RZ4550MBC0Zp8JHRETyxsyZM+tdA6hL\nl6DY2bIFrroKbrstKIB++EPYty/qqCWVVPiIiEjeOO6446rXACosLKxeA2jJkiUAdOsWFDt/+hNc\neil85ztQWAh33w0ffBBx8JISKnxEQtq4cWPUIUgKKZ/x05yc9uvXj4ULF/L000/Tvn17ioqKOOec\nc/gwuclXz55Bt9fGjXDmmXDNNXDiifDgg8HMMMldKnxEQpowYULUIUgKKZ/x05KcjhgxghUrVjBv\n3jxOP/102rZtW+t8797wq1/B+vXQrx9ccQX07Qu/+522wchVWscnTbSOT/xs27ZNM4FiRPmMn0zk\n9PnnYeJEWLIETj0Vpk2Ds88OFkqU1NE6PiJZQH8k40X5jJ9M5HTQIFi8GJ56Ctq3D7bA+M//hBUr\n0v7WkiIqfNJMW1aIiMTD0qVL2bt3LxDs+7V8OfzhD/DPfwaboF5wAaxdG22MuU5bVuQwdXWJiMTH\nnj176NmzJwUFBUyePJmSkhLatWsHQGUlPPoofPe7sHkzfOELcOutwYrQ0jLq6hLJAtOnT486BEkh\n5TN+0pnTzp07s3bt2nrXADrsMCguDjY9vfdeWLUKTjoJSkqC/cAku6jwEQmp6ha3xIPyGT/pzmnN\nNYD69OlDcXExgwYNql4DqG3boNjZvBluvx0WLIA+feC662DnzrSGJs2grq40UVeXiEi8LVu2jBtv\nvJFVq1Zx/fXXM3PmzFrn330XfvITmDEjWPzw618Ptsc46qiIAs4h6uoSERHJMsOHD69eA6i4uLjO\n+YKCYOXnrVuhtBTuvDPYBmPaNNizJ4KABVDhIyIi0mJmRiKRYPDgwQ22OfJImDo12AbjyiuDgc/H\nHQd33AHvvZe5WCWgwkckpF27dkUdgqSQ8hk/2Z7T7t2DYmfzZkgk4FvfCsYA3XMP7N8fdXT5Q4WP\nSEglJSVRhyAppHzGT7bmdN++fdxxxx3Vg6979QqKnVdeCdb/GTMmmAVWVhZMjZf0UuEjEtKUKVOi\nDkFSSPmMn2zN6cqVK5kwYQKFhYXMmjWL/cnbO8cfHxQ7a9cGG6Bedhn07w+//732AUsnFT4iIWl2\nXrwon/GTrTkdOXIkGzdu5Mwzz2TcuHG11gCCg8XOypXwkY8E3WBDh8LSpREHHlMqfERERNKsvjWA\nBg8eXL0GEBwsdpYsCbq8Ro6Es86C556LMPAYUuHTDGb2X2a20cw2mdlXoo5HRERyS79+/ViwYAHl\n5eW0b9+eoqIi5s+fX33e7GCx89hjwcKHn/kMXHwxbNgQYeAxosInJDNrA8wEzgAGAt82s3+PNCjJ\nqHvvvTfqECSFlM/4yaWcVq0BtHDhQs4///w6582CYufFF+Ghh4Kip18/uPxyeP31CAKOERU+4Q0G\nXnL3ne6+B1gAFEUck2TQmjUpXTxUIqZ8xk+u5dTMOO+882jbtm2Dbdq0CYqdjRvh7ruhvDwYCH31\n1fCXv2Qw2BhR4RPex4DtNZ5vB3pGFItE4Kc//WnUIUgKKZ/xE+ectmsXFDubNwdbYPzv/0JhIVx/\nPfz971FHl1vyovAxs2FmNt/MtptZpZkl6mlzjZltNbN9ZvasmQ2KIlYREZEqixYt4vvf/371GkAd\nOwbFzpYtwXYY99wTrAI9aRLs3h1xsDkiLwofoABYB4wD6qyOYGbFBON3JgMDgBeBRWbWtUazvwIf\nr/G8Z/KYiIhIWrz00ktMmjSJwsJCZs+eXb0G0BFHwOTJwT5gY8fCD34Axx4L06cHm6NKw/Ki8HH3\nx919krvPA6yeJqXAbHd/0N03Al8D9gI1lwFdDZxsZj3MrDNwLrAo3bGLiEj+uuGGG6rXABo7dmyd\nNYA+8pGg6+tPf4IvfhFuvjnoArvrLnj//YiDz1J5Ufg0xszaEczSerLqmLs78AQwtMaxA8ANwNPA\nGuCH7v52U9c///zzSSQStR5Dhw5l7ty5tdotXryYRKJODxzXXHNNnZkKa9asIZFI1NmXZvLkyUyf\nPr3WsW3btpFIJNi4cWOt43feeSfjx4+vdWzv3r0kEgmWL19e63hZWRmjR4+uE1txcXFefY5EIhGL\nzwHxyEdrP0cikYjF54B45CMVnyORSMTic0DtfNRcA2jPnj111gBavHgxX/tagp/9DDZtgqIi+MY3\noGvXaygpuZcPP8yOz1FTzXyUlZVV/208+uijSSQSlJaW1nlNyrh7Xj2ASiBR43mP5LEhh7SbDqxq\nxfucAnhFRYVLPCxatCjqECSFlM/4yZeclpeX+9ChQ/2II47wt99+u942L7/sfskl7uB+4onujz7q\nfuBAhgNthYqKCicYmnKKp7gOyPs7PiJhFRVp9YI4UT7jJ19yWrUGUEVFBUceeWS9bU46CX77W3j+\neTjmGPj852HQIHj8ce0DpsIHdgEHgO6HHO8O7GztxUtLS6tvv4qIiKSCmVFYWNhku1NPDYqd8vJg\nRth558GIEfDMMxkIsgWqur3S2dVlnmeln5lVAhe7+/wax54FnnP3bySfG7AN+Im7/6CF73MKUFFR\nUZG1G+eJiEj+cA+KoIkTgx3hzz0Xpk6FgQOjjqyuNWvWMDAIbKC7p3Rlyry442NmBWbWz8z6Jw8d\nl3z+ieTzHwFjzOzLZnYiMAvoBNwfQbiSpQ4dGCm5TfmMH+W0tq9+9au11gAyC+74vPACzJkTTIU/\n9dSgG+zVVyMONoPyovABTgXWAhUEg6VmEszMugXA3ecA3wJuTbbrC5zj7q1eD1NdXfGhHMaL8hk/\nyulBlZWVdOrUqd41gA47LCh2XnoJ7rsvGAf06U/DlVcGxVCU1NWVw9TVJSIiUduyZQuTJk3i17/+\nNYWFhUydOpVLL72Uww47eN/j/ffhF78Iur3OPx9++csIA05SV5eIiIg0W801gPr06VO9BtATTzxR\n3aZDB7j22mARxBkzIgw2Q1T4iIiIxFy/fv1YsGAB5eXltG/fngceeKBOm4IC6Nq1nhfHjLq60qSq\nq2v48OF06dKFUaNGMWrUqKjDEhGRPOfu7N27l4KCgqhDqaOsrIyysjJ2797NsmXLIA1dXSp80kRj\nfOJn9OjR3HfffVGHISmifMaPchofGuMjkgXyZVXYfKF8xo9yKmGo8BEJSV2V8aJ8xo9yKmG0jTqA\nuCstLdUYHxERkRBqjvFJF43xSRON8REREWkZjfERyQLLly+POgRJIeUzfpRTCUOFj0hIM/JhZa88\nonzGj3IqYajwEQnpkUceiToESSHlM36UUwlDg5vTTIOb46NTp05RhyAppHzGj3Ka+zS4OYdpcLOI\niEjLaHCziIiISAqo8BEJafz48VGHICmkfMaPciphqPARCalXr15RhyAppHzGj3IqYWiMT5pojI+I\niEjLpHOMj2Z1pZlmdYmIiISjWV05THd8REREWkazukSywMaNG6MOQVJI+Ywf5VTCUOEjEtKECROi\nDkFSSPmMH+VUwlDhIxLSXXfdFXUIkkLKZ/wopxKGCh+RkDRVNl6Uz/hRTiUMFT4iIiKSNzSdPc00\nnV1ERCScTExn1x2fNLv99tuZP3++ip4YmD59etQhSAopn/GjnOa+UaNGMX/+fG6//fa0vYcKH5GQ\n9u7dG3UIkkLKZ/wopxKGFjBMEy1gKCIi0jJawFBEREQkBVT4iIiISN5Q4SMS0q5du6IOQVJI+Ywf\n5VTCUOEjElJJSUnUIUgKKZ/xo5xKGCp8REKaMmVK1CFICimf8aOcShgqfERC0uy8eFE+40c5lTBU\n+IiIiEje0JYVaaYtK0RERMLRlhUxoC0r4uPee++NOgRJIeUzfpTT3KctK0SyyJo1KV08VCKmfMaP\nciphaMuKNNGWFSIiIi2jLStEREREUkCFj4iIiOQNFT4iIiKSN1T4iISUSCSiDkFSSPmMH+VUwlDh\nIxLStddeG3UIkkLKZ/wopxKGCh+RkIqKiqIOQVJI+Ywf5VTCUOEjIiIieUOFj4iIiOQNFT7NYGa/\nM7O3zGxO1LFI5s2dOzfqECSFlM/4UU4lDBU+zXMH8KWog5BoTJ8+PeoQJIWUz/hRTiUMFT7N4O7L\ngD1RxyHR6NatW9QhSAopn/GjnEoYKnxEREQkb8S28DGzYWY238y2m1mlmdVZ2crMrjGzrWa2z8ye\nNbNBUcSaCWVlZVlzvea8Nkzbpto0dr6hc6n+/0q1dMTX0mumOp9NtVM+03vN5r4unb+juZpP0Hdu\nc89lMqexLXyAAmAdMA6oswW9mRUDM4HJwADgRWCRmXWt0Wacma01szVm1iEzYaeHfgmbdy7bv1hz\n9Q+lCp/65Wo+w7ZX4RPt9fSdW1vbjL1Thrn748DjAGZm9TQpBWa7+4PJNl8DLgBKgBnJa/wM+Nkh\nr7PkoymHA7z66qstCT/ldu/ezZo1a7Lies15bZi2TbVp7HxD5+o7vnr16pT+H7ZGqvPZmmumOp9N\ntVM+03vN5r4unb+jzT0e55zm23dujb+dhzcZdDOZe52bIbFjZpXAxe4+P/m8HbAXuKTqWPL4/UAX\nd/9cA9dZAvQluJv0FvB5d3+ugbaXAQ+n8nOIiIjkmcvd/depvGBs7/g0oSvQBnjzkONvAic09CJ3\nP7sZ77EIuBz4M/BeM+MTERHJZ4cDnyT4W5pS+Vr4pJ27/wNIaZUqIiKSR1am46JxHtzcmF3AAaD7\nIce7AzszH46IiIhkQl4WPu6+H6gARlYdSw6AHkmaKkwRERGJXmy7usysACjk4Ays48ysH/CWu78B\n/Ai438wqgNUEs7w6AfdHEK6IiIhkQGxndZnZCOAp6q7h84C7lyTbjAMmEHRxrQOuc/cXMhqoiIiI\nZExsCx8RERGRQ+XlGJ9sYGb/ZWYbzWyTmX0l6nikdczsd2b2lpnNiToWaT0z+7iZPWVmL5vZOjO7\nNOqYpOXMrIuZPZ9chX+9mX016pik9cyso5n92cxmNOt1uuOTeWbWBngFGEGw2/saYIi7vx1pYNJi\nZjYc+DfgCnf/QtTxSOuY2dHAR919vZl1J5gM0cfd90UcmrRAcvJKB3d/z8w6Ai8DA/Wdm9vMbCrQ\nG3jD3SeEfZ3u+ERjMPCSu+909z3AAqAo4pikFdx9GUERKzGQ/N1cn/z3mwRLYBwVbVTSUh6oWki2\nY/JnmK2HJEuZWSHBgsN/bO5rVfhE42PA9hrPtwM9I4pFRBphZgOBw9x9e5ONJWslu7vWAduAH7j7\nW1HHJK3yQ+A7tKCAVeHTTGY2zMzmm9l2M6s0s0Q9ba4xs61mts/MnjWzQVHEKk1TPuMnlTk1s6OA\nB4Ax6Y5b6peqfLr7bnfvDxwLXG5m3TIRv9SWinwmX7PJ3V+vOtScGFT4NF8BwdT3cdSdKo+ZFQMz\ngcnAAOBFYJGZda3R7K/Ax2s875k8JpmXinxKdklJTs2sPfAY8L2GNiOWjEjp76i7/z3ZZli6ApZG\npSKfnwG+aGZbCO78fNXMbg4dgbvr0cIHUAkkDjn2LPDjGs8N+AswocaxNsAmoAfQGXgV+PeoP0++\nP1qazxrnzgAejfpz6JGanAJlwKSoP4Merc8n8FGgc/LfXYANwMlRf558f7T2Ozd5/gpgRnPeV3d8\nUsjM2gEDgSerjnmQmSeAoTWOHQBuAJ4mmNH1Q9fsgqwTNp/JtkuA3wDnmdk2MxuSyVglnLA5NbPP\nAp8HLjaztclp0CdnOl5pXDN+R48BnjGztUA5wR/WlzMZqzStOd+5rRHbLSsi0pXgbs6bhxx/k2D0\neTV3/wPwhwzFJS3TnHyenamgpFVC5dTdV6Dvx1wQNp/PE3SbSHYL/Z1bxd0faO6b6I6PiIiI5A0V\nPqm1CzhAsPdXTd2BnZkPR1pJ+Ywf5TRelM94yUg+VfikkLvvJ1jhdWTVseSKoSOBlVHFJS2jfMaP\nchovyme8ZCqf6sNuJjMrAAo5uG7AcWbWD3jL3d8AfgTcb2YVwGqgFOgE3B9BuNIE5TN+lNN4UT7j\nJSvyGfV0tlx7EOyvVUlwO67m45c12owD/gzsA1YBp0Ydtx7KZ748lNN4PZTPeD2yIZ/apFRERETy\nhsb4iIiISN5Q4SMiIiJ5Q4WPiIiI5A0VPiIiIpI3VPiIiIhI3lDhIyIiInlDhY+IiIjkDRU+IiIi\nkjdU+IiIiEjeUOEjIiIieUOFj4iIiOQNFT4iIiKSN1T4iEjWMLOEmS03s3Vm9r6ZVZrZO2Z2ZAPt\nu5vZCjPbk2xbaWb/MrMKMzs70/GLSPbT7uwiknXMzIB/AF2Sh8a7+48aaX8U8BrwB2Csu+9Lf5Qi\nkotU+IhI1jGzfsBTwAfAR4HX3f34Jl7zOjDA3f+VgRBFJEepq0tEstEw4GlgdvJ5bzMraqixmX0S\n2K6iR0SaosJHRLLRcGApcDewP3nsmkbajwDK0x2UiOQ+FT4iko2GAUvdfScwBzDgfDP7RAPth6PC\nR0RCUOEjIlnFzPoQjD98JXnox8mfhwFfa+BlpwEr0x2biOQ+FT4ikm2GE4zvAcDdXwBWEdz1+YqZ\nta3Z2Mw+BrytmVwiEoYKHxHJNsMJZnTVdEfyZzfg8/W0b3U3l5m1a+01RCT7qfARkWxTNbC5pt8B\nbyT/Pe6QcyOAZU1d1MwGmdnPkgseLjazOWY22cwON7NjgPsOad/ezBaa2evJhRG3J58vNLOnzOw1\nM3vUzD5lZp8zs6Vm9m6y7SvJ9+hR43qPmNnu5PmtZvb9Zv/PiEiraR0fEckaZvZxYJW71xnEbGYT\ngO8DDvR39w3J4+uBzzY0ld3MPgL8FPgPYIK7L6hxbiDwTWAAUObu0+p5fTHwa+ASd59b43g7oAw4\nMxnPNjN7Fjja3T/ZQCwXA//t7l9u8j9DRNJCd3xEJJvU181V5RdA1TiecQBm1hV4r5Gi5wTgOaAA\nGFiz6AFw9wpgNfCpRt739OTP5Ye8dj/BXaIjCcYedQJOaeQ6AMcmP4eIRESFj4hkkwYLH3d/G/gV\nwSDny82sc7L9M/W1N7NuwOPAO8AX3P29Bt5zIfAuQYFUn9MJVo7eVc+5zsmf/0Yws6wtjY83GtLI\n+4hIBqjwEZFsMozG75hUTW0vAK6k8YHNPwc+AVzVxIyvd4AV7n7g0BNmdgRBF1lDU+XPIOh6W5mM\nxRuKJ7n/WHt3/6CRWEQkzVT4iEhWSI7FOdzd/9xQG3d/FViSfDqW4G5MnTs+ZvZZ4CJgUXI6g27O\njgAAAoZJREFUfGP2AJMbOHcawffkinreoytQDDzl7r8lKIK2u/vWBq7VF9jQRCwikmYqfEQkW9Ra\nv6cRdxB0d50ItE12gR1qNMHdl/vqOVeLu+9z99UNnB7GwTs61ZKLLM4DFgMXmlkHYBCNd3OFmn0m\nIunVtukmIiIZcRaNd3MB4O5/NLPXgD40XEiMSP58upUxDSPYK+y6oKcKB9oTjAm6wd2fBTCz4UAH\nGi98TkMDm0Uip8JHRCJnZucAVwDrQr7kzuSjoUKjJ/BOfQOSzexs4NvAR4GOQCWwFbgwOVOrql17\ngrs4K919bBPxNDq+J6mjVpcWiZ66ukQkMmb2ezP7M8HMqo7A3Wa2wcyubeKl9wM7afiOzzvA3vpO\nuPsSdz8LuAHoDTzi7ufWLHqSBhHcxQnTPdUX+Ie7b67vpJkNBdaEuI6IpJnu+IhIZNz9wha+7l3g\nY400eQ44z8wKkm3rcxrBXZo/NnC+anxPmO0wOgH1Fj1JpcmHiERMd3xEJI5uJ/h+G1PfyeTU8ksJ\nxuo838A1qsb3rArxfmuBrg2819eBxe6+PcR1RCTNVPiISOy4+9PARGCamY02s+rvuuQ09FnA6wTj\nd+pbv6cdwR2hNY0sfFjTXUCBmV1e4xqHm9lk4KPufk+rPpCIpIz26hKR2DKzMwi6mI4B3gT+CfwN\n+AmwDRjs7s/UaH8McE+yfW/gX8ALwF019+lq4L16ATMI7vy8D3wAzHL3Ran9VCLSGip8REREJG+o\nq0tERETyhgofERERyRsqfERERCRvqPARERGRvKHCR0RERPKGCh8RERHJGyp8REREJG+o8BEREZG8\nocJHRERE8oYKHxEREckbKnxEREQkb/w/eVOww8l8UxgAAAAASUVORK5CYII=\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj4AAAGPCAYAAABcTNrhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3Xt8zuX/wPHXdTNsaM5kYYsKk9N0kJIhJO4pqVaOUyjH\nVXw7OkWhcpbDt5w6TF9KbVRYJZH0bUvCVD8ZSZRDmkOMvX9/fGzfzQ7uzX3afb+fj8f9qPv6XPfn\n877vy71duz7v67qMiKCUUkop5Q9sng5AKaWUUspdtOOjlFJKKb+hHR+llFJK+Q3t+CillFLKb2jH\nRymllFJ+Qzs+SimllPIb2vFRSimllN/Qjo9SSiml/IZ2fJRSSinlN7Tjo5RSSim/oR0fpZRSSvkN\n7fgUgjEm2BjzX2NMsjFmmzHmYU/HpJRSSinHGd2k1HHGGAOUFpF/jDGBwA4gQkSOeTg0pZRSSjmg\npKcDKE7E6iX+c+Fp4IX/Gg+Fo5RSSqlC0ltdhXThdtdWYB/wsogc9XRMSimllHKM33R8jDG3GWPi\njTG/GWMyjDH2POoMNsbsMcacNsZ8bYy54eI6InJcRJoCYcBDxpiq7ohfKaWUUpfPbzo+QFlgK/AY\nkCuxyRhzP/AqMAZoBnwPrDHGVMnrZCLy54U6t7kqYKWUUko5l18mNxtjMoBuIhKfrexrYIuIDL/w\n3AC/AjNFZMqFsmrAKRE5YYwJBjYCD4jIjjyuURnoCKTyv7wgpZRSSl1aGSAUWCMiR5x5Yk1uBowx\nAUAE8GJmmYiIMSYRaJmtah1ggdUnwgAz8ur0XNAReNs1ESullFJ+4SHgHWeeUDs+lipACeDQReWH\ngOsyn4jIf7FugzkiFeCtt96iQYMGTgjx8sTGxjJt2jSvOF9hXutI3UvVKeh4fsfyKnf2Z3g5XBFL\nUc/p7Pa8VD1tT9ees7Cvc+V31FnlnqA/cy/vO5qSkkLPnj3hwu9SZ9KOj+v8A9CgQQOaN2/u6VgI\nDg52ahyXc77CvNaRupeqU9Dx/I7lVe7sz/ByuCKWop7T2e15qXranq49Z2Ff58rvqLPKPUF/5l7+\nd/QCp6eKaMfHchg4D1S/qLw6cPByThwbG0twcDDR0dFER0dfzqkui7OvfTnnK8xrHal7qToFHc/v\nWF7lBw9e1j8Fp3LFv6WintPZ7Xmpetqerj1nYV/nyu9oYct9uU395WduXFwccXFxHD9+/JJxFJUm\nN/+vLK/k5n1Yyc0vF+EazYGkpKQkr/kLRF2ekJAQfvvtN0+HoZxE29P3aJv6juTkZCIiIsDaHSHZ\nmef2mxEfY0xZoB7/W2n5amNME+CoiPwKTAUWG2OSgG+AWCAIWOyBcJUXuvAlVD5C29P3aJsqR/hN\nxwdoAXyOtYaPYK3ZA7AEiBGR/1xYs2c81i2urUDHC+v1KOXRW5XK+bQ9fY+2qXKEX97qcofMW12t\nW7f2ihwfpZTv2bdvH4cPH/Z0GEoVSZUqVahdu3aOsuw5Phs2bAAX3OrSjo+LaI6PUsqV9u3bR4MG\nDTh16pSnQ1GqSIKCgkhJScnV+QHN8VHKK/Tr149FixZ5OgzlJMW9PQ8fPsypU6e8Zq0wpQojc52e\nw4cP59nxcSXt+CjloA4dOng6BOVEvtKe3rJWmFLFhT9tUqrUZdEcLd+i7amUf9IRHxfzlgUMlVJK\nKW/njgUMtePjYtOmTdNhaKWUUsoBmYME2ZKbnU5vdSnloI0bN3o6BOVE2p5K+Sft+CjloClTpng6\nBOVE2p5K+Sft+CjloGXLlnk6BOVE2p7F19ixY7HZ9NeXKhrN8XExTW72HUFBQZ4OQTmRtmfxZYzB\n2ke6cObOnUtQUBB9+vRxQVTKGTS52QdocrNSyluISJE6DJ4+t7O89tprVK1aVTs+XkyTm5VSSl2W\ntLQ0xgwbRvuwMLrVqkX7sDDGDBtGWlqaV59bKVfRjo9SDho5cqSnQ1BO5A/tmZaWRveWLWk5Zw7r\nUlP58LffWJeaSss5c+jesuVldVBcee7sNm7cyA033EBgYCDXXHMNCxYsyFVn0aJFtGvXjurVq1Om\nTBnCw8OZN29ejjphYWHs2LGD9evXY7PZsNlstG3bFoBjx47x5JNP0rhxY8qXL09wcDCdO3dm27Zt\nTnkPyrvorS6lHOTu/WSUa/lDe77y7LM8npJCp4yMrDIDdMrIQFJSePW55xg7Y4bXnTvT9u3b6dix\nI9WqVWP8+PGkp6czduxYqlWrlqPevHnzaNSoEVFRUZQsWZKEhAQee+wxRIRHH30UgBkzZjBkyBDK\nly/Pc889h4hQvXp1AH755Rfi4+Pp0aMHYWFhHDp0iPnz59OmTRt27txJjRo1Lut9KC8jIvpwwQNo\nDkhSUpIopZSzJSUlyaV+xrQLDZUMEMnjkQHSPjS0yNd35bkzdevWTYKCgmT//v1ZZbt27ZKSJUuK\nzWbLKvvnn39yvbZTp05Sr169HGWNGjWSyMjIXHXPnj2bq2zv3r1SpkwZmTBhwuW8BZWPS/37zTwO\nNBcn/37WW10uFhsbi91uJy4uztOhKKX8iIhQNj2d/NKNDRCUnp75h5rXnDtTRkYGa9eu5e677yYk\nJCSr/LrrrqNjx4456pYuXTrr///++2+OHDlC69at+eWXXxy65RYQEJDjukePHiUoKIjrrruO5OTk\nIr8HVXhxcXHY7XZiY2Nddg291eVi3jCr6+BBqFYNdNkLpfyHMYaTAQEI5NlBEeBkQECRZmK58tyZ\n/vzzT06fPk29evVyHbvuuuv4+OOPs55v2rSJMWPG8PXXX3Pq1KkccR4/fpzy5csXeC0RYfr06cyd\nO5c9e/Zw/vz5rNdXqVKlyO9BFZ7O6lKXLSMD7rwTbr0Vvv/e09EUb7t27fJ0CMqJ/KE9W3Xtypp8\n/uL5xGbjVrvdK89dGLt376Z9+/YcPXqUadOm8dFHH5GYmJg1YpCRLQcpPxMnTuSJJ56gTZs2vP32\n26xdu5bExEQaNmzo0OtV8aIjPj7OZoPp02HQoHNERJRk+HAYNw7KlfN0ZMXPqFGjiI+P93QYykn8\noT2fnDiR7p99hlxIQjZYozGf2GxMa9CA9yZM8MpzA1StWpXAwEB+/vnnXMeyd1oTEhI4e/YsCQkJ\nOW6Jffrpp7lel98I1HvvvUfbtm1zzRj766+/qFq1alHfgvJSOuLjB6pW3cnp09fw0EMJzJ0LDRrA\nypVWFqJy3OzZsz0dgnIif2jP8uXL897mzWwZMoQOoaFEhYTQITSULUOG8N7mzZe8BeSpcwPYbDY6\nduzIBx98wP79+7PKU1JSWLt2bdbzkiWtv9+zj8wcP36cxYsX5zpn2bJl+euvv3KVlyhRIlc+0vLl\ny/ntt98u6z0o76QjPn6gQoUKNGoUztKldrp168nJkzO4555KdOkCs2ZBaKinIywe/GH6sz/xl/Ys\nX768Na18xgynr67synMDjBs3jk8++YRbb72Vxx57jPT0dGbPnk2jRo2y1tjp0KEDAQEBdOnShYED\nB5KWlsbrr79O9erVOXjwYI7zRUREMG/ePCZOnEi9evWoVq0akZGRdOnShRdeeIGYmBhuueUWfvjh\nB95++23q1q3r1PejvISzp4npwzuns2dkZMiSJUukQoUKUqNGDXnqqXgJCREJDBSZNEnkzBlPR6iU\nKgxHprP7gi+//FJuuOEGKVOmjNSrV08WLFggY8eOzTGdfdWqVdK0aVMJCgqSq6++Wl555RVZtGiR\n2Gw22bt3b1a9Q4cOSdeuXSU4OFhsNlvW1PYzZ87IyJEjJSQkRMqWLSutW7eWLVu2SGRkpLRt29bt\n79kfeHI6uxG93+ESxpjmQFJSUpLHZ3Vld+DAAQYOHMiqVau4//6eVKo0gwULKlG/PsybZyVBK6W8\nX+asF2/7GaOUIy717zfbrK4IEXHqmgKa4+Ni3raOT82aNYmPj2fJkiWsWbOK+PjGfPFFGuXKwW23\nQf/+cPiwp6P0TpMnT/Z0CMqJtD2V8j66jo8P8IZ1fC5mjKF37960a9eOdevW0apVeTZtgn//G556\nCj78EF5+Gfr2BS/fbNmtsq8Pooo/bU+lvI+u46NcKiQkhL59+wJQogQMGgQ//gidOkFMDLRpAzt3\nejRErzJu3DhPh6CcSNtTKf+kHR+VQ/Xq8NZbkJgIv/8OTZrAM8+A/nGslFLKF2jHR+WpXTvYtg2e\nfx6mToXwcPjoI09HpZRSSl0e7fiofP3668+kpsawceNRrrkG7roL7r0Xsq0l5lcOa9a3T9H2VMo/\nacdH5Wv37t2sXLmSrl3DGTw4nrg42LTJWvl5+nQ4d87TEbpXTEyMp0NQTqTtqZR/0o6PylenTp3Y\nvn07ERERdOsWxerVvfjqq6P07g2PPw433gjffOPpKN1n7Nixng5BOZG2p1L+STs+qkAhISEkJCSw\nePFiEhISuOWWcDp2jOfrr629vm6+GQYPhjy2v/E53rYsgbo82p5K+Sft+LiYty1gWBTGGPr06cOO\nHTuIiIggKiqK2bN7s2VLBlOnwtKl1u2vZct041OllFJF544FDLXj42LTpk0jPj6e6OhoT4dy2bKP\n/oSEhFCqlI0RIyAlBVq1guho6NgR/u//PB2pUkqp4ig6Opr4+HimTZvmsmtox0cVSuboz0svvZRV\ndtVVsGIFrFoFP/8MjRrB+PFw5owHA3WBN954w9MhKCfS9lTZLV68GJvNxr59+7LK2rRpQ2RkpAej\n8py+ffsSFhZWpNe2adOGtm3bOjki59GOj3Kau+6CHTsgNhZeeAEaN4bPP/d0VM6TnOzUffKUh2l7\nquyMMZiL9ugxxmCz+eevybw+j8K81pv5Z4sqlwkKgpdegq1boVo1aNsWeveGP/7wdGSXb86cOZ4O\nQTmRtqe6lHXr1rFmzRpPh6GcTDs+yulSU1PZvTueL76AhQth9WqoX9/aBDUjw9PRKaWUY0qWLEnJ\nkrqXt6/Rjo9yusWLFxMVFUWfPr2IijrKjz9CVBQMGAC33grff69Tv5RSBTtx4gQjRowgLCyMMmXK\nUL16dTp06MDWrVtz1NuyZQudO3emUqVKlCtXjiZNmjBz5sys4z/88AP9+vWjbt26BAYGcuWVV9K/\nf3+OHj16yRguzlX54osvsNlsLF++nIkTJ1KrVi0CAwNp3749u3fvzvX6OXPmULduXYKCgrj55pvZ\nuHGjw/kvNpuNYcOGsWLFCsLDwwkKCuKWW25h+/btAMyfP59rrrmGwMBAIiMjc+QmZVq+fDktWrQg\nKCiIqlWr0qtXLw4cOJCr3gcffECjRo0IDAykcePGfPDBB3nGJCJMnz49q26NGjUYNGgQfxWz9Uy0\nK6ucbsyYMYSFhTF8+HASExOZP38+M2dGkvH3IlYkdKZZ01BqXbGQBx/8iWemjKF8+fKeDlkp5WUG\nDhzI+++/z9ChQ2nQoAFHjhxh48aNpKSk0LRpU8C6FdW1a1dq1qzJiBEjqFGjBikpKaxevZphw4Zl\n1dmzZw8xMTHUqFGDHTt2MH/+fHbu3MnmzZsLjCG/XJVJkyZRokQJRo4cyfHjx5k8eTI9e/bMcb65\nc+cydOhQbr/9dh5//HFSU1Pp1q0bFStWpFatWg59Bhs2bCA+Pp7BgwcD8OKLL9KlSxdGjRrF3Llz\nGTx4MMeOHWPy5MnExMSQmJiY9drFixcTExPDTTfdxKRJkzh06BDTp0/nq6++4rvvvuOKK64AYO3a\ntdx77700atSISZMmceTIEfr168dVV12VK54BAwawdOlSYmJiGD58OHv27GHWrFls3bqVTZs2UaJE\nCYfel8eJiD5c8ACaA5KUlCT+av/+/XLXXXcJIFcGB8t/jJF/CJAJPCNlOCXVSJWmtR+Tv//+29Oh\nOqRr166eDkE5UXFvz6SkJCnMz5iTJ0WSklz7OHnSee+vQoUKMnTo0HyPnz9/XsLCwuTqq68u8GfI\nP//8k6ts2bJlYrPZZOPGjVllixcvFpvNJnv37s0qa9OmjURGRmY9X79+vRhjJDw8XM6dO5dVPnPm\nTLHZbLJjxw4RETl79qxUqVJFbr75Zjl//nxWvaVLl4oxJsc582OMkcDAQNm3b19W2YIFC8QYIzVr\n1pST2T7sZ555Jkfs6enpUr16dWnSpImcOXMmq97q1avFGCNjx47NKmvatKmEhIRIWlpaVlliYqIY\nYyQsLCyr7MsvvxRjjCxbtixHnGvXrhVjjMTFxeX7ueXlUv9+M48DzcXJv591xEe5TOa6P3d36MCn\niYkMA0qTzrO8yAMsYzBzWLNvDjc2+561nzfBwT+CPGbIkCGeDkE5kb+1565dEBHh2mskJYGzFsSu\nUKECW7Zs4ffff+fKK6/Mdfy7774jNTWVGTNmFDhqXLp06az/P3PmDCdOnOCmm25CREhOTqZVq1aF\nji0mJibH6MZtt92GiPDLL7/QsGFDvv32W44cOcLkyZNzzAp78MEHGTFihMPXad++fY7RoZtuugmA\ne++9l6CgoFzlv/zyC7Vr1+bbb7/ljz/+YPz48ZQqVSqrXufOnalfvz6rV69mzJgxHDx4kO+//55n\nnnmGcuXKZdVr164dDRs25NSpU1llK1asoEKFCrRr144jR45klTdr1oxy5crx+eef88ADDzj83jxJ\nOz7KpYwxnPi//2MXMBD4ELADdfmFj7mT/9CD3qmzaNjQWvtn6FDw1lzCDh06eDoE5UT+1p7161sd\nE1dfw1mmTJlC3759qVWrFhEREXTu3JnevXtnrS2ze/dujDGEh4cXeJ5jx44xduxY3n33Xf7INr3U\nGMPx48eLFNvFt6oqVqyYdS2AvXv3Yoyhbt26OeqVKFGC0NDQIl8nODgYINdtqODgYEQk1/Wvvfba\nXOesX78+mzZtyqoHUK9evVz1rrvuOr777rus5z///DN//fUX1apVy1XXGJPjs/V2XvorRvkKEaFs\nejohQAJwNtsxA9zPcpZU+4Gr79nJE08Yli6F+fOtDVCVUs4TFOS80Rh36NGjB61bt2blypWsXbuW\nV155hcmTJ7Ny5Uo6duxYqPN8/fXXjBo1iiZNmlCuXDkyMjLo2LEjGUWcZppfLos4ec+e/K7jrutn\nl5GRQfXq1XnnnXfyvE7VqlVddm1n01ldyqWMMZwMCECwOjqlLzouwNnS/zB7tmHLFjDmfxufFvGP\nMaWUj6hevTqDBg3i/fffZ8+ePVSuXJmJEycCULduXUQka5ZTXv766y8+++wznn76aUaPHk1UVBTt\n2rUr8orEjqpTpw4iwv9dtH/P+fPnSU1Ndem1s1//xx9/zHXsxx9/pE6dOln1wBrNyatednXr1uXI\nkSPccssttG3bNtfj+uuvd8E7cQ3t+CiXa9W1K2vyWf30E5uNW+12AG64Ab75BqZNszY+rV/fuzY+\nzW+KpyqetD29V0ZGBn///XeOsipVqlCzZk3OXNgLp3nz5oSFhTF9+vR8b1lljoxcPLIzbdo0l64u\n3KJFCypXrsy///3vHNd+6623sm5HuVKLFi2oVq0a8+bNIz09Pav8448/JiUlhS5dugBQo0YNmjZt\nypIlS0hLS8uqt27dOnbu3JnjnPfddx/nzp1j/Pjxua53/vz5It829AS91eVisbGxBAcHEx0d7RMb\nlRbFkxMn0v2zz5CUFDplZGCwRno+sdmY1qAB702YAMChQ4cICAhg+PBK3HsvDB9ubXy6aBG89hpc\ndLvc7eLi4ujWrZtng1BOo+3pvdLS0rjqqqu49957s25PrVu3jm+//ZapU6cC1mjy3LlzsdvtNG3a\nlH79+nHllVeya9cudu7cyccff0z58uVp3bo1U6ZM4ezZs4SEhLB27VpSU1NdelsoICCAsWPHMmzY\nMCIjI7nvvvtITU1l0aJF1KtXz+VbOpQsWTJrinvr1q2Jjo7m4MGDzJw5k6uvvjpHgvVLL71Ely5d\naNWqFTExMRw5coTZs2fTqFEjTpw4kVWvdevWDBw4kEmTJrF161Y6dOhAQEAAP/30EytWrGDmzJnc\nc889lx17XFwccXFxru1IOXuamD50Onte/v77bxkzbJi0Dw0Ve0iItA8NlTHDhuWYhvrAAw9IjRo1\nJD4+Pqts1SqROnVEypQReeEFkTxmpirllwo7nb04OXv2rPzrX/+SZs2aSXBwsJQvX16aNWsm8+fP\nz1X3q6++ko4dO2bVa9q0qbz22mtZxw8cOCDdu3eXSpUqScWKFeWBBx6QgwcPis1mk/Hjx2fVy286\ne9u2bbOer1+/Xmw2m7z33ns5YkhNTRWbzSZLlizJUT579mwJCwuTwMBAufHGG2XTpk3SokUL6dy5\n8yU/A5vNJsOGDcvzOlOnTs1Rnl9cy5cvl4iICAkMDJQqVapI79695cCBA7mutXLlSgkPD5fAwEBp\n1KiRfPDBB9K3b1+5+uqrc9V9/fXX5YYbbpCyZctKcHCwNGnSRJ5++mk5ePBgVp2LP7e8eHI6u8c7\nCL760I5P/jIyMvIsz77uT8+ePeXIkSMiInLihMi//iVSsqRI/foi69e7M1qlvJMvd3x8VUZGhlSu\nXFkGDBjg6VA8zpMdH83xUW6X3zBv5ro/ixcvZtWqVYSHh5OQkEDZsjBpEiQnQ6VK0KYN9O0Lhw+7\nNWyllHJYZi5SdkuWLOHo0aNERkZ6ICKVSTs+yqsYY+jTpw/bt28nIiICu91Or169OHr0KNdfD19+\nCQsWQHw8XHedtQmqbnyqlPI2X3/9Nc2bN+ell15iwYIFDBw4kEceeYTGjRtz7733ejo8v6YdH+WV\nMkd/lixZwqpVq3j++ecBsNngkUesVWjvugv697dGgC6agOAS/fr1c/1FlNtoeypXCg0NpXbt2sya\nNYthw4aRkJBA3759SUxM1B3fPUw/feW1jDH07t2b9u3bU6ZMmRzHqlWzprz37QuPPgpNm8LIkfDc\ncxAY6Jp4/G2lX1+n7alcqU6dOrpkgpfSER/l9WrWrEmlSpXyPNa2LXz/PTz7LLzyCjRqBJ984po4\n/HU5Al+l7amUf9KOjyr2ypSBMWPghx8gNBTuvBPuvx9+/93TkSmllPI22vFRxd7Zs9YOYNdeC4mJ\n8OabsH69tfLznDlw/rxn41NKKeU9tOOjijUR4YEHHsia+WUM9OxpJT8/8AAMGQItW0K2TYaLbOPG\njZd/EuU1tD2V8k/a8VHFXrdu3XKs+wNQsaK1y/umTXD6NLRoAY8/Dtm2oym0KVOmOCli5Q20PZXy\nT9rxUcVa5syvvNb9AbjlFmvhw5degnnzoGFDKOpEi2XLljkxcuVp2p5K+Sft+CifcPG6P9lHfwIC\nYNQoa62fJk3g7rvBbod9+wp3jaCgIBdErjxF21Mp/6QdH+Uz8hr9mT17dtbx0FBISIAVKyApyRr9\nefVVOHfOczErpfK2ePFibDYb+xz4CyU0NJSYmBiXxuOOaxSFzWZj/PjxWc8L87n5K+34KJ+TOfqz\ndOlSevTokeOYMdC9O6SkWKs+jxpl5f9s2eKhYJVSeTLG5LuvX151Xc0d13CGwnxu/ko7PoVgjLnK\nGPO5MWaHMWarMUY3XPFSxhh69epF9erV8zx+xRUwY4bV4SlRwpr59dhj8Ndf+Z9z5MiRLopWeYK2\np3fr3bs3p0+fpnbt2p4OpVjRz+3StONTOOeA4SISDnQEphtjXLRBgnKHzNGeadOs9X8aNIBly0Ak\nd139QeJbtD29mzGGUqVKeTqMYkc/t0vTjk8hiMhBEdl24f8PAYeBvPdSUMVGyZIwfLi19k+rVhAd\nDZ06we7dOesNHTrUMwEql9D29G755apMmDCBWrVqUbZsWdq1a8fOfHYoPn78OCNGjKB27dqUKVOG\na665hilTpiAX/VXzyiuv0KpVK6pUqUJQUBAtWrTgvffeK3Lcy5Yto0WLFlxxxRUEBwfTuHFjZs6c\nmSu22NhYwsLCKFOmDLVq1aJPnz5Zs1HT09MZPXo0LVq0oEKFCpQrV47WrVuzfv36S14/r88tNDQU\nu93Opk2buOmmmwgMDKRu3bq8+eabuV6/bds2br/9doKCgqhVqxYTJ05k0aJFPpU3pJuUFpExJgKw\nichvno5FFU1sbCyRkZHY7XYAQkKsxOfVq2HwYGvfr+efhyefBP0DSin3yitX5fnnn2fixIl06dKF\nO++8k+TkZDp06EB6enqOeqdPn6Z169b8/vvvDBo0iFq1avHVV1/x9NNPc/DgQaZOnZpVd+bMmURF\nRdGzZ0/Onj3LsmXLuO+++1i1ahV33nlnoWJet24dDz74IHfccUfWOlEpKSl89dVXDBs2DICTJ09y\n66238uOPP9K/f3+aNWvG4cOHiY+PZ//+/VSqVIm///6bhQsXEh0dzYABA0hLS+ONN96gU6dOfPPN\nNzRu3LhQn5sxhp9//pkePXrQv39/+vbty8KFC+nXrx8tWrSgQYMGABw4cIDIyEhKlCjBs88+S1BQ\nEK+//jqlSpXyrbwhEfGLB3AbEA/8BmQA9jzqDAb2AKeBr4Eb8jlXJWA7cFMB12sOSFJSkijv888/\n/8hdd90lgPTs2VOOHDmS4/iJEyIjR4qUKCHSsKHIhg0eClSpfCQlJYkv/4xZvHix2Gw22bt3r4iI\n/Pnnn1K6dGmx2+056j377LNijJF+/fpllb3wwgtSvnx52b17d466Tz/9tAQEBMj+/fuzyv75558c\ndc6dOyfXX3+9tG/fPkd5aGhojmvkZcSIEVKhQoUC64wePVpsNpt8+OGH+dbJyMiQ9PT0HGXHjx+X\nGjVqyMMPP5yj3Bgj48aNy3p+8eeWGbvNZpNNmzZllf35559SpkwZGTlyZFbZ0KFDpUSJErJt27as\nsmPHjknlypVznfNyXerfb+ZxoLk4uT/gT7e6ygJbgcewPswcjDH3A68CY4BmwPfAGmNMlYvqlQJW\nAi+KiM4FKqZKly5NQkICixcvJiEhIce6PwBly8KUKdbih1dcAa1bQ/fuuzhyxINBK6fatWuXp0Nw\nu99//53k5OR8H/ndNspu586deb72dxfvCpyYmEh6enquW5QjRozIVXfFihXcdtttBAcHc+TIkaxH\nu3btOHdn3RUEAAAgAElEQVTuHBs2bMiqW7p06az//+uvvzh27Bi33XYbycnJhY6xQoUKnDx5kjVr\n1uRb5/3336dJkyZZI815McZQsqR1Q0ZEOHbsGGfPnqVFixZFigugYcOG3HLLLVnPq1SpwnXXXccv\nv/ySVbZmzRpatmzJ9ddfn+M9PfTQQ0W6prfym46PiHwiIqNF5EMgrzG7WGC+iCwVkV3AIOAUcPHC\nDUuAT0XkHddGrFzNGEOfPn3YsWNHnqs+AzRubG17MXcuxMePon59WLIk7+RnVbyMGjXK0yG43fz5\n84mIiMj3cfHyD3np0aNHnq+dP3++S2Pfu3cvAPXq1ctRXqVKFSpWrJij7Oeff+aTTz6hatWqOR53\n3HEHxhj++OOPrLqrVq2iZcuWBAYGUqlSJapVq8bcuXM5fvx4oWN87LHHuPbaa+ncuTO1atWif//+\nuTpBu3fvplGjRpc815IlS2jSpAllypShcuXKVKtWjdWrVxcpLsg7mb9ixYocO3Ys6/nevXtzfb6Q\n+zMv7vym41MQY0wAEAF8mlkmIgIkAi2z1WsF9AC6GWO+M8YkG2PCCzp3586dsdvtOR4tW7bkg4v2\nTVi7dm2efwEMHjyYN954I0dZcnIydrudw4cP5ygfM2YMkydPzlG2b98+7HZ7rr9uZ82alWs676lT\np7Db7bk2b4yLi6Nfv365Yrv//vt94n08/vjj9O/fP8foz7Rp07Leh80GgwbB5s2zqVhxMH37vkG7\ndvDjj971PnylPdz1PmbPnu0T76MwBg4cSFJSUr6P5cuXX/Icy5cvz/O1AwcOLFJMrpCRkcEdd9zB\np59+SmJiYo7HunXr6N69OwBffvklUVFRBAUFMXfuXD7++GMSExN58MEHcyVBO6Jq1aps3bqV+Ph4\noqKiWL9+PXfeeSd9+/Yt1Hneeust+vXrxzXXXMPChQtZs2YNiYmJtG3bloyMjELHBVCiRIk8y4vy\nPp0tLi4u63djjRo1sNvtxMbGuu6Czr53VhweXJTjA1x5oeymi+pNBjYX8Rqa41MM7d+/X6Kjo3Pk\nAFxszRqRunVFSpUSGTNG5PRp98WnVCZ/y/GJi4sTm80ma9euzVHvzz//zJXjEx4eLq1atbrkNUaM\nGCFly5bNlU/z4IMPis1my1HmSI5PXgYNGiQ2my0r36hRo0bSrFmzAl/TrVs3qVevXq7yVq1aSVhY\nWI4yR3N8unbtmut8bdq0kcjIyKzn1157rdx666256g0dOlRzfJTyVSEhIbzzzjuEhITkW6dDB/jh\nBxg5El580bod9tlnbgxSKT/Uvn17SpYsyaxZs3KUT5s2LVfd++67j82bN7N27dpcx44fP541alKi\nRAmMMZzLtm9NamoqH374YZFizH6bPFNmvsyZM2cA6N69O99//32B18hrdGbLli1s3ry5SHE5qmPH\njmzevJlt27ZllR09epR33vGtzA6dzm45DJwHLl7mtzpw0P3hKG8XGAgTJsBDD1m3wdq1g549rb2/\nqlXzdHRK+Z4qVarw5JNPMmnSJLp06ULnzp357rvvsnJ5shs5ciTx8fF06dKFvn37EhERwcmTJ9m2\nbRvvv/8+qampVKpUibvuuoupU6fSsWNHHnzwQQ4dOsRrr73GNddck+OXv6Mefvhhjh49Stu2bbnq\nqqtITU1l9uzZNGvWLGvK+MiRI1mxYgU9evSgX79+REREcOTIERISEpg/fz7XX389Xbp04f3336db\nt27cdddd/PLLL8yfP5/w8HBOnDjhlM8zL6NGjeKtt96iffv2DB06lLJly/L6669Tp04djh075jNT\n2nXEBxCRdCAJaJdZZqwWbgd8dTnnjo2NxW63ExcXd3lBKo/LK6+iQQNYvx4WLoSPPoL69eH116GI\nt+GVGxU1T0Z5zsSJExk3bhxbt25l1KhR7Nmzh7Vr11K2bNkcv5QDAwPZsGEDo0aN4osvvmDEiBFM\nnjyZ3bt3M378eIKDgwGIjIxk4cKFHDp0iNjYWN59912mTJlCt27dcl3bkT2wevXqRWBgIHPnzmXw\n4MG8+eabREdH89FHH2XVKVu2LBs3buTRRx/l448/Zvjw4cybN48GDRpw1VVXAdC3b19eeukltm3b\nxvDhw1m3bh1vv/02ERERea7Rc6m4CqqTvfyqq65i/fr1NGzYkJdeeonp06fTq1evrBylMmXKFHgd\nZ8jM99EcH+fk9ZQFmgBNsfJ5Rlx4XuvC8fuwZnH1BuoD84EjQNUiXk9zfHzM6NGjRUTkjTfeyLXu\nj4jIH3+I9OkjAiKtWols3+7mAFWhZLZnceXrOT7KewwfPlyCgoIkIyPDaefUHB/3aAF8hzWyI1hr\n9iQD4wBE5D/Ak8D4C/UaAx1F5E+PRKu8zrhx4zh06BBPPPFErnV/AKpWhcWLrXyfP/+Epk3hmWfg\n1CnPxKsKNm7cOE+HoJTX+eeff3I8P3LkCG+99Ra33Xab3uoqbkTkCxGxiUiJix4x2eq8JiKhIhIo\nIi1F5FtPxqy8T/Xq1dm+fXu+6/4AREbCtm3WdhevvmptfVHAemZKKeU1WrZsSWxsLAsWLGD8+PFE\nRESQlpbG888/7+nQnMZvOj6eojk+vickJKTAVZ8BSpeG0aOt2V9hYdamp9HRcFBT5ZVSXuyuu+7i\n448/5vHHH+fll18mNDSUTz75hFatWrnl+prjU4wfaI6Pz/nzzz9zle3fv7/APb9ERDIyRJYuFalS\nRSQ4WGTuXJHz590RsSpIXu1ZnGiOjyrONMdHqWIgJubi3Utyjv588cUXpKWl5apjDPTqBbt2wb33\nwqOPQqtW1u0w5Tl5tadSyvdpx0cpB40dOzbP8sw9v3bv3k2dOnXyfX3lytZU9w0b4PhxaN4cRo2C\nkyddFLAqUH7tqZTybdrxcTHN8fEdzZs3L/B4QECAQ+e57TbYuhXGj4dZsyA8HFavdkaEqjAu1Z5K\nKfdzR46PdnxcbNq0acTHxxMdHe3pUJQXKVXKmuq+fTtcey106QI9esCBA56OTCmlPCc6Opr4+Pg8\ntyJxFt2yQik3+eGHH7L27clUt6411X3ZMoiNtVZ+fvFFKw8on82UlcohJSXF0yEoVWge/Xfr7Gxp\nfeisLl/1+uuvF/m1ycnJBc78EhE5elRk4EARELnhBpHvvivy5ZQDLqc9vcHevXslKCgoc+aLPvRR\n7B5BQUH57vjuylldOuKjlIOSk5Pp379/kV7btGlTlixZwvDhw0lMTGT+/PnY7fYcdSpWhHnzoHdv\nGDgQWrSA4cNh3DgoV84Z70Bldznt6Q1q165NSkoKhw8f9nQoXmPSpEk89dRTng5DOahKlSrUrl3b\n7dc1Yo1OKCczxjQHkpKSkjSJUmU5cOAAAwYMYPXq1fTs2ZMZM2ZQqVKlXPXS02HqVKvTU6UKzJkD\nXbt6IGCllPKA5ORkIiIiACJEJNmZ59bkZhfTWV0qu5o1a5KQkMCSJUtYtWpVnqs+AwQEwL/+BTt2\nWLO+7Ha45x7Yv98DQSullJu4Y1aXjvi4iI74qEvJHP357LPP2LNnD9WrV8+znggsX27d9jpxAiZM\ngCFDNPlZKeW7dMRHKR+UOfqTnJycb6cHrJWf77sPUlKs/J/YWLjpJkhKcmOwSinlI7Tjo5SDLk5G\ndgZjDPXr13eoboUKVq7P5s1WDtCNN1qdoDx2yVAOcEV7Ks/SNlWO0I6PUg4aMmSIp0MArNGeb7+F\nyZNhwQJo2BA+/NDTURU/3tKeynm0TZUjtOOjlIM6dOjgkeueOXMmV1nJksKTT8LOndCkCXTrZj1+\n/dUDARZTnmpP5TrapsoR2vFxMZ3VpS7Hpk2bqFu3LgkJCaSlpTFm2DDah4XRrVYt2oeFsfDVYbzz\nThorVsA330CDBjBtGpw75+nIlVKq8HRWVzGms7qUMxw4cICBAweyatUqrgwOZsbff3OvCAZrSdM1\nNhtTGzTgvc2bycgoz7PPwmuvQdOm1m2wFi08/Q6UUqrwdFaXUl7ggw8+cPs1a9asSXx8PFHt25N2\n/DjDRFh14ZgBOmVkEJuSwqvPPUdwMMyeDV9/DRkZVi7Q8OHw999uD7tY8ER7KtfSNlWO0I6PUg7y\n1O1KYwwn/u//2AVEAHagF3D0wvFOGRlsio/Pqn/jjVby85Qp8PrrVvLzypXWekDqf/T2s+/RNlWO\n0I6PUg569913PXJdEaFsejohQAKwBFiF1Qk6gzXyE5SeTvbb1iVLwhNPWMnPzZpZqz536wb79nni\nHXgnT7Wnch1tU+UI7fgo5eWMMZwMCECwOjm9ge3Aq0BprFyfkwEBGGNyvbZOHYiPhxUrrFGghg2t\nPcA0+Vkp5a+046NUMdCqa1fW2P73dQ0B7rnw/5/YbNxawMJtxkD37tbKz/36wZNP/u92mFJK+Rud\n1eUimbO6WrduTXBwMNHR0URHR3s6LFVMpaWl0b1lS2JTUuiUkZE1q+sTm41pF2Z1lS9f3qFzffMN\nDBwI27ZZe3698AJccYVLw1dKKYfExcURFxfH8ePH2bBhA7hgVpd2fFxEp7P7nn79+rFo0SKPXT8t\nLY1Xn3uOTfHxBKWncyoggFZ2O09MmOBwpyfTuXMwcyY8/zxUrAizZlk5QHncLfNZnm5P5Xzapr5D\np7Mr5QU8vSps+fLlGTtjBuv27OGDX39l3Z49jJ0xI89Oz3//+1/69OnD0aNH8ziTlfz8+OP+nfzs\n6fZUzqdtqhzh8IiPMaYHEOjacHI4LSLL3Xg9p9IRH+VJ8fHx9OnThzJlyjB//vwCN28Usaa7Dx0K\nx4/D+PEwbJjVOVJKKU9w5YhPYX60vQL84syLX0IYUGw7Pkp5kt1uZ/v27QwcOJCoqCh69uzJjBkz\nqFSpUq66xlgjPu3bw3PPWcnPb72lKz8rpXxTYTo+R0Uk0mWRXMQY8527rqWULwoJCSEhIYGlS5cy\nYsQIEhMTWbBgAV27ds2z/hVXWHk/PXvCgAHWys+a/KyU8jWa46OUgzZu3OjpEArNGEOfPn3Yvn07\nERER2O12+vfvT0G3uDOnur/8cs6Vn31NcWxPVTBtU+WIwnR8prosCu+4nlIFmjJliqdDKLLM0Z8l\nS5YQHh6e52KH2eWX/Pzrr24K2A2Kc3uqvGmbKkfodHYX0eRm33Pq1CmCgoI8HYbbXZz8/MIL1v8X\n9+Rnf21PX6Zt6juK3XR2Y0y0MWaGMeZmV5xfKU/w1x+omcnPKSkQE2PtAXbTTZCU5OnILo+/tqcv\n0zZVjnB6x8cY8wjQBXgU+PBCWU1jzMfGmBPGmO3GmP7Ovq63io2NxW63667BqtjLTH7++ms4f97K\nBRoxAtLSPB2ZUspXxMXFYbfbiY2Nddk1nH6ryxizHrgDq/NTR0SmG2M2AK2As1j7K9YH3hKRR516\ncS+it7pUcbRt2zZSU1MLXPcHrJWfZ8yA0aOhUqX/rfyslFLOUNxudZ0TkXQRWXmh09MIuBXIACJF\n5AagHtDSGNPGBddXyiVGjhzp6RBcbunSpURFRdGrV698V30GK7/niSes5OcmTeDuu61HcUp+9of2\n9DfapsoRruj4XLy6c+baP1+LyNcAInIIeAR4zAXXV8olateu7ekQXO7ll19myZIlrFq1ivDwcBIS\nEgqsX6cOJCTA8uWwZYs19X3GDOtWmLfzh/b0N9qmyhGu6PjsNca0z/a8PdZG0luyVxKR/wKVXXB9\npVxi6NChng7B5Ywx9O7dO8e6P5ca/TEG7r3XSn7u0wdiY63k52SnDk47nz+0p7/RNlWOcEXH5xVg\nmTHmGWPMi0DmMrGf5FG3GPxdqJT/yb7uj6OjP8HBMHs2fPUVpKfDDTdYawGdOOGmoJVSygFO7/hc\nSEIaDjx14QHwnogkGktwturnnH19pZRzXDz6s3nzZoded/PN1srPkybBvHnW7a/4eBcHq5RSDnLJ\nOj4i8jbWJqPdgHYict+FQyuAo8aYicaYK4E/XXF9pVxh165dng7BIzJHf8aPH+/wawICYORI2LED\nwsMhKgq6d4fffnNhoIXkr+3py7RNlSNctleXiBwRkXgR+TxbcQnAAI8DbwHTXHV9pZxt1KhRng7B\nY4wxlCzCUs1hYfDRR7BsGWzaBA0aWFPfvSH52Z/b01dpmypHuHuT0hhgHPBv4GkR2erm6ytVZLNn\nz/Z0CMWSMXD//bBrFzz0EAwbBi1bwlYPf/u1PX2PtqlyhFs7PiJyVETGicgwEfnGnddW6nLpVNn8\npaamFjjzC6BCBZg71xr5OXUKWrSAJ5+EkyfdFORFtD19j7apcoTDHR9jTFlXBuLp6ymlim7QoEE0\natTokjO/AG65xZrqPmECzJljJT+vXu2GIJVSisKN+Hzpsii843pKqSJauHAhzZs3d2jdH4BSpeCp\np2D7drjuOujSBXr0gAMH3BSwUspvFabjY1wWhXdcT6kCTZ482dMheK2aNWvmWvcn3oE57HXrwpo1\n8PbbsGGDlfz82muQkeH6mLU9fY+2qXJEYaZpBBpjerssktzKuPFaLhMbG0twcDDR0dFER0d7Ohx1\nGU6dOuXpELxa5ro/7dq1Y+DAgURFRdGzZ09mzJhBpUqVCngdPPggdOpkjQINHgxvvgnz50Pjxq6L\nV9vT92ibFn9xcXHExcVx/Phxl13D4d3ZjTHvAfn/9HK+oyLS3Y3XcyrdnV35MxFh6dKlDB8+nCFD\nhjBhwgSHX/vllzBwIPz8s7UR6ujREBTkwmCVUl7HlbuzOzziU5w7IUop9zLG0KdPH9q3b0/FihUL\n9drbbrOmuk+ZYiVA/+c/1mywjh1dFKxSyq+4ex0fpZQfCQkJIagIwzWlSsFzz8EPP1iLIHbqZN0O\nO3TIBUEqpfyKdnyUctDhw4c9HYLfueYaSEyEpUth3TqoXx/+/W/nJD9re/oebVPlCO34KOWgmJgY\nT4fgc06fPn3JOsZAr17Wys933w0DBkDr1tY+YJdD29P3aJsqR2jHRykHjR071tMh+JSMjAw6duzo\n0Lo/AJUrw8KF8Pnn8Oef0KyZdTvMgb5TnrQ9fY+2qXKExzs+xphQY0xfXalZeTudnedcxhgefvjh\nrHV/HFn1GaBNG9i2DZ55Bl5+2Zry/umnhb++tqfv0TZVjnB7x8cYE2iMqZL5XERSgW+A54wxLd0d\nj1LKMzLX/dm+fTsREREOr/oMULo0jB0L338PISHQvj307m2NBCmlVEHc2vExxjwMHAMOGWMOGmOW\nGGO6AD+JyNPAQ+6MRynleSEhISQkJLB48WISEhIcXvUZrGTnzz+HN96AVaus54sWgYPLkyml/JC7\nR3weBHoB9wDzgQbAh8BBY8xyoJGb41HKYW+88YanQ/BZmev+7Nixg4iICKKioli8eLGDr4WYGCv5\nuXNn6/8jI+HHHwt+nban79E2VY5wd8fnWxFZLiIfisgYEbkRCAUmAgeAIW6ORymHJSc7dfFQlYfM\n0Z933nmH7t0Lt2ZqtWrWVhdr18L+/Vbuz7hxcOZM3vW1PX2PtqlyhMNbVhTqpFauTjNgE7BNLlzE\nGDMJeF5E0p1+US+jW1Yo5TmnT1urPk+ZYm2EOn8+3H67p6NSSjnKlVtWuGrEpy4wG0gG/jLGrDXG\njAV2AXOMMbrzjlLKZQIDYeJEa+uLypWtmWD9+4MDedNKKR/nqo7Pn8BSoCUwHjgJDAYWAg8DO40x\nE40xHXQau1LKVcLDrU1P582D996zkp/fekuTn5XyZ67q+GwFlorINyLyqojcLSJVgXBgILABK9H5\nE+CYMeYbY8wrxpibXRSPUspHiAi9evVyeOaXzWbt9r5rF7Rta60C3bEj7N7t4kCVUl7JJR0fETkk\nIp/lUZ4iIv8Wkd4iEgbUBvoC3wIdgbddEY8zGWPeN8YcNcb8x9OxKPey2+2eDkEBJ06c4NixY0RF\nRTm87g9AjRqwbBl89BH89BNce62dF1+Es2ddHLByG/2OKkd4YgHDxpk5PiKyX0TeAWaKyPUiUtfd\n8RTBdKwp+crPDBmikw69Qfny5Yu87g/AnXda+3zdc88QRo+G5s3hq69cGLByG/2OKke4ewHDp7BW\naf7ookPVL8z48noisgE44ek4lPt16NDB0yGoC/Ja96cwoz9ly8Ly5R349lsICoJWreDRR+Gvv1wc\nuHIp/Y4qR7h7xKci1q2tuOyFIvIFkGyMucvN8SilirG8Vn3+9ttvHX5906aweTPMnGklPTdoAP/5\njyY/K+XL3N3xCRCRZSIy/+IDIvIf4A5XXdgYc5sxJt4Y85sxJsMYk+tmsDFmsDFmjzHmtDHma2PM\nDa6KRynlHNlHf+68807q1atXqNeXKAFDh0JKCrRsCfffD127wt69LgpYKeVR7u74VDbGFJTH48o0\nw7JYs80eA3L9PWeMuR94FRiDtfji98Ca7BuqKv/2wQcfeDoEVYCQkBAWLlxIhQoVHKq/cuXKHM+v\nugrefx8++MBa/6dhQ3j1VTh3zhXRKlfQ76hyhLs7PjOAxAJuaZVz1YVF5BMRGS0iHwImjyqxwHwR\nWSoiu4BBwCkgJo+6Jp9zKB8WFxd36UrKq6WlpTFm2DDah4Ux5KGHaB8Wxphhw0hLS8uqExVljf48\n/DCMHAk33giFuHumPEi/o8oRbu34XFh2+jngfWPMD8aYF4wx3Y0xdxhjxuHCjk9BjDEBQATwabZY\nBUjEWoQxe911wLvAncaYfcaYmwo6d+fOnbHb7TkeLVu2zPWXydq1a/Ocijl48OBcG+8lJydjt9s5\nfPhwjvIxY8YwefLkHGX79u3Dbreza9euHOWzZs1i5MiROcpOnTqF3W5n48aNOcrj4uLo169frtju\nv/9+v3of7777rk+8D/CN9ijs+3jqqadoXq8eLefMYV1qKr+dPs0bqamsmTWLTs2b5+j8LF48i1Kl\nRrJlC2RkwE03weDBp+jc2fPvw1fawxXv49133/WJ9wG+0R6Ovo+4uLis3401atTAbrcTGxub6zXO\n4pK9ui55UWsfq8lAW/43crIOeEBEjrnh+hlANxGJv/D8SuA3oKWIbMlWbzLQWkRa5n2mAq+he3Up\n5UVGDx3KoTlzeEmEShcd+9hmY8uQIYydMSPX686dg+nTYcwYqFQJ5swBXS5GKdcqjnt1Xcp3WCs3\nXwncDFwtIh3d0elRSvmnT1eu5F0RwoGEi451yshgUz5rAZUsCU8+aa39c/311q2w7t3ht99cHrJS\nygXcvY5PKWPMbKy9uw4CPwH9cW1SsyMOA+eB6heVV8eKs8hiY2Ox2+1671kpDxIRqgA7sO5p27FW\nIc1c9ccAQenpFDQCHhoKq1fDu+/Cpk3W1Pc5c+D8eRcHr5Qfybzt5cpbXe4e8XkZuAIYAfwLWAv0\nBH4wxrR2cyxZRCQdSALaZZYZY8yF55e1puu0adOIj48nOjr68oJUHpfXfWpVPBhjOBkQQE2s0Z7F\nWIl64UA81jTPkwEBWF/7gs4D991n7fv14IMwZIi1+OG2bS5+A8oh+h0t/qKjo4mPj2fatGkuu4a7\nOz7lL+zTtUBEXhGR+4AawAtAnDEm1FUXNsaUNcY0McY0vVB09YXntS48nwo8YozpbYypD8wDgrB+\nRiqlq8IWc626dmWNzYYB+gDTsEZ/orD+wokoRPtWqGDt+L5xI6SlWdte/OtfcOqUS0JXDtLvqHKE\nW5ObjTGviMiT+Ry7CRggIv1ddO3bgc/JvYbPEhGJuVDnMWAU1i2urcBQESnSRFZNblbKu6SlpdG9\nZUtiU1LolJGBATKAUcYwLyCA7T/+SGhoaKHPe/YsvPwyvPAC1KwJc+dau78rpYrOl5KbKxljwvI6\ncGE21T+uurCIfCEiNhEpcdEjJlud10QkVEQCRaRlUTs92WmOj1LeoXz58ry3eTNbhgyhQ2goUSEh\ndAwNpdzQoew/dKhInR6AUqXg2Wfhhx8gLAw6dbJugx065Nz4lfIH7sjxcfeITzNgBdZIysUblWKM\neU1EHnNbQC6kIz5KeTcRuWROT+HPae35FRtrrf8zZQrExIDNU/NnlSqmfGbER0S+A0YDK71pAUOl\nHHHxwlyqeNu0aZPTz2kM9OplJT/b7fDIIxAZaa0ErVxPv6PKEW7/O0RE3gZaAX8AzwLLgTVY6/kM\nd3c8SjlqypQpng5BOZGj7SkibNmy5dIVs6lSBRYvhk8/hQMHoEkTawHEf1x2M1+BfkeVYzwyACsi\n34pIO6wkYl3AUBULy5Yt83QIyokcbc8vvviCm2++mV69enH06NFLvyCbtm2t3J9//QteesnqAK1f\nX4RglUP0O6oc4faOjzHmJmPMZGPMTKAH8KuIpLo7DnfR5GbfERQU5OkQlBM52p633347S5YsYdWq\nVYSHh5OQcPG6zwUrU8aa8fXdd1C1qnXrKyYGjhwpStSqIPodLf58Mbn5UWA61krJ1YASwDngNeBp\nETnttmBcTJOblfItBw4cYMCAAaxevZqePXsyY8YMKlW6eNevgmVkwOuvw6hR1mywqVPhoYes3CCl\n1P/4THIz1qakVUUkBCuRORJ4FbgH+MQYU8bN8SillENq1qxJQkLCZY3+2GwwYICV7BwZaSVCd+wI\nu3e7KGilVC7u7vjsFpG/AUTkzIW1dZ4GrgF2As+7OR6lHDZy5EhPh6CcqCjtaYyhd+/e7Nixg4iI\nCPr27cvx48cLfZ4rr7T2/Fq9Gn76CRo1gkmTID290KdS2eh3VDnC3R2fg8aYRhcXXugEPYq1fYVS\nXql27dqeDkE50eW0Z+boT1JSEsHBwUU+T+fO1q7vgwdbiyBGRMDXXxf5dH5Pv6PKEe7O8amAtTHp\nG8CnIvJ/Fx2fLSJD3BaQC2Xm+LRu3Zrg4GCio6N1o1KlVL6++85a9yc5GR59FF58ES6jT6VUsRQX\nF0dcXBzHjx9nw4YN4IIcH3d3fNYAYcBVQGngILAB2AzcCLwrIgnZ6g8SkXluC9CJNLlZKVVY58/D\n7CCs0LYAACAASURBVNnW6E9wMMyaBXffrcnPyv/4UnLzzyJyLXAF1iKG07B2QB8NPAgsMcZ8ZIx5\n1hjTDvCJ7SuUUv7p5MmThapfogQMHw47d1q3vbp3h27d4NdfXRSgUn7I3R2fz40xLwOdgSQReUVE\nokSkCtAIeBo4CgwA1gHhbo5PqXzt2rXL0yEoJ3J1e8bHx1OvXj3i4+ML/drateHDD2HFCvjvf6Fh\nQ5gxwxoRUvnT76hyhLv36noPeAb4G6h40bGdIjJfRHqKSB2gHrDPnfEpVZBRo0Z5OgTlRK5uzxYt\nWhAREUFUVFSRVn02xhrxSUmB3r2tjU9btoStW10UsA/Q76hyhCe2rAgUkfUi8kdmwYV8mBxE5Bfg\nJbdGplQBZs+e7ekQlBO5uj3zWvenKKM/wcEwZw5s2gSnT0OLFjByJBTyLppf0O+ocoS7k5snAk2B\nE0C0iGRcKL8B6CkiPrNJqc7qUkplyr7q80MPPcTMmTMLveozWOv8vPoqjBsHNWrA3LnQqZMLAlbK\nQ3xxVtdcEXnUGBMF2ERkZbZjXYAgEfmP2wJyIZ3VpZTKTkR48803GT58ODVq1GD79u2UKFGiSOfa\nvRsGDYLERHjgAZg+HapXd3LASnmQL83qytyWLwG4KfsBEVmFNdNLKaV8TvZVn2fOnFnkTg9A3bqw\ndi0sXWp1furXt/YAy8hwYsBK+Sh3d3xqGGPKX7jFldfKFP+4OR6lHDZ58mRPh6CcyFPtWbNmTe64\n447LPo8x1l5fKSnWlPdHHoE2bazn/kq/o8oRRer4GGNGG2N6GmNaGGPKFeKli4CPjDEN8jmuW1Yo\nr3Xq1ClPh6CcyFfas0oVWLQIPv0Ufv8dmjSBsWPhzBlPR+Z+vtKmyrWKlONjjMkABEjDmpp+BkgF\nBovIT5d47RBgKvA7EAfsAUoCHYG9IjK00AF5Ic3xUUq52+nTMHEiTJ5s3Q6bPx9uv93TUSlVeN6a\n4/OoiFQQkdoicg1wP7D7Ui8SkdlAO+An4AlgLvAi8DPw+GXEo5RSxd6aNWuKtO4PQGAgTJhgrfVT\nubJ16+vhh6EIp1LKZxW14/OriCzIXiAiR0XEoXVFReRLEbkDCASuBCqJyBMikl7EeJRSyiecOHEi\na92fhISES78gD+Hh8OWXMG+etfpzgwawbBm4cRKvUl6ryB0fZ1xcRM6JyCFHO0zFUWxsLHa7nbi4\nOE+Hoi7T4cOHPR2CciJvbc/u3buzfft2IiIisNvtRR79sdlg4EAr2fn22yE6+v/bu/MwKcqr7+Pf\nwybbEwRREBQ1MSriPsGFCBINEzXSaIghBAUhKi4gmbjEGKMmTxZFIy64EVEUXxuNGmQG1ygREAFl\n3NhMTEw044MGJbjggsx5/6geMw6z9HRXd3VX/z7X1Rd0dfVdpznX9Bzuuhc49lh47bUcBF0gCjWn\nkr5kMkkikaCioiJ3F3H3Vj+AJzN5Xyk9gIMAX7FihUs8DB8+POoQJESFns/a2lqfNWuWb7vttt67\nd29/8MEHs2qvstJ9553dO3VynzrVffPmkAItIIWeU0nfihUrnGAs8UEe8u/nKLasEClKl112WdQh\nSIgKPZ9mxrhx41i1atXne36deeaZGbd33HHBru9nnAEXXggDBwYboMZJoedUCkOmhU8/M+sYaiQi\nBU6z8+KlWPJZf8+vQw45pOU3NKNrV7j6ali+PFgH6NBDYcoUeP/9kIKNWLHkVKKVaeHzZeBtM3vI\nzC40s0Fm1q61jZjZ1AyvLyJSMupWfT7llFNCaa+sLCh+pk4NVnzee2/IYP9UkaKUza2ursDRwK+B\nRcBGM3vSzC4zsyPNrFMabRycxfVFRCRD7drBuefCqlWw774wYgR897vw5ptRRyaSW5kWPjXAz4HH\ngA8Jtp/oBAxNHX8c2GBmS8zscjM71sy+1Eg7ul0mRWPmzJlRhyAhUj4Du+4K8+cH090XLQqmvt90\nU3Hu+6WcSjoyLXz+5u6/dvejgW0Jem7OI9h89D8EhVAH4FDg/NTxd8ys2syuMbMTzKwfsEvWn0Ak\nT6qrQ108VCIWt3wuWrQo43V/zGDUKFi7NvjzrLPg8MNh5cqQg8yxuOVUciPTwufz/wu4e627P+fu\nV7v7CHffDtgfmAz8AXiboBBqCxyQOn4fwVYVO2QTvEg+3XDDDVGHICGKWz7nzJmT1bo/AN27w4wZ\nsHAhbNgABx4IP/tZsBVGMYhbTiU3Mt2ra7m7pz0+x8z2AIbUe/RLveTu3rbVARSBur26hgwZQrdu\n3Rg9ejSjR4+OOiwRiSl3Z/bs2UyZMoWOHTsyY8YMhg8fnnF7n3wCl18Ov/kN9OsXrAJ91FEhBizS\niGQySTKZZOPGjSxcuBBysFdXpoVPjbv3zfiiZrsAPwB+FffCR5uUikg+1dTUMHHiRObPn89JJ53E\ntddeS48ePTJub+3aYAXohQth3Di46qpgR3iRXCrETUp3NLOBmV7U3f/p7r8luN0lIiIh6du3L5WV\nlcyaNYvKykoGDBhAVVVVxu3ttRcsWBBMe3/wwWDw8+zZ2vdLilc209lvMrOuWV7/7SzfL5I3iUQi\n6hAkRHHOZ8NVn//6179m1V6bNvDDHwa9P8OGwdixUF4Of/tbSAGHJM45lfBkWvhcBGwDvGxmE9Jc\ns6cxn2b4PpG8mzRpUtQhSIhKIZ91vT9TpkwJpb1eveDuu+Hhh+HVV2GffYJxQJs3h9J81kohp5K9\njAofd7/c3fcFEsDuwFIzm2lmvVrZ1IJMri8ShfLy8qhDkBCVSj7NjDZtwt2W8eijg6nukybBxRcH\nK0EvWxbqJTJSKjmV7GT10+DuL7v7Re6+P3Ar0Kq6391/kc31RUQkGl26wJVXBhuddugAhx0GkyfD\ne+9FHZlI80L7b4C7P+PumS0eISIikVi9enXG6/5AsNbPsmUwbRrcfnuw79fcuSEGKBKycPs/RWJs\nrr7NY0X5DNb+mTBhAgMGDMh41WeAtm2DXd5Xr4YDDoATToDvfAdqakIMNg3KqaRDhY9ImpLJZNQh\nSIiUz2D8z/33309ZWVnWqz5DsNBhZSXcey8880ww9f2GG2DLlhCDboZyKulQ4SOSpnvuuSfqECRE\nymegsXV/5s2bl3F7ZnDiibBmDfzgB8EA6MMPh5dfDjHoJiinkg4VPiIiJa7huj8jRozIuvdn222D\nbS4WLYKNG+Ggg+Cii4pn3y+JLxU+IiICbN37M2vWrKzbPPxweP55uOQS+N3vYN994Yknso9VJFMq\nfERE5HN1vT9r1qzhnHPOCaXNbbaBn/8cXnoJdtoJvvnNYN+v9etDaV6kVVT45FhFRQWJREKD7mJg\n/PjxUYcgIVI+m7fjjjvSrl27UNvcc89g36+ZM4NB0HvtBXfeGd6+X8pp8UsmkyQSCSoqKnJ2DRU+\nOTZt2jTmzZvH6NGjow5FsqRVYeNF+YyGGUyYEAx+Li8Pen6GDQu2wMiWclr8Ro8ezbx585g2bVrO\nrqHCRyRNKl7jRfnMzvvvv5/V++vv+/W3vwVjf7Ld90s5lXSo8BERkVb59NNPGTRoUNYzv2Drfb++\n9rXC2PdL4kuFj4iItEr79u0577zzPl/3J5tVn+GL+361b699vyS3VPiIpGnx4sVRhyAhUj4z13Dd\nnzBWfYZg36+lS+Hqq/+779eDD6b/fuVU0qHCRyRNU6dOjToECZHymb2wV30GaNcOfvQjWLUq2Pfr\n+ONh5Mj09v1STiUdKnxE0jRnzpyoQ5AQKZ/haGzV5/vvvz/rdnfZJZjyfs898PTTQe/PjTdCbW3T\n71FOJR0qfETS1Llz56hDkBApn+Gq6/257777GD58eChtmsH3vhdMfR81Cs4+O1gJeuXKxs9XTiUd\nKnxERCQUZsbIkSPp0KFDqO127w4zZsBTT8G77wZjgS6+GD7+ONTLSIlQ4SMiIkVhyBB48UX42c9g\n6lTYb79gJWiR1lDhI5Km888/P+oQJETKZ3HaZhu47LKgAOrVC448MlgJ+p13lFNJjwofkTT169cv\n6hAkRMpn/m3evJnjjjsu65lfAP37B7e+ZsyABx4Inr/9dr/Q9v2S+FLhI5KmyZMnRx2ChEj5zL8P\nPvgAgBEjRoSy7k+bNnDaacHg5298A+68czJHHw1//3sY0UpcqfAREZG86N69+1br/mS76jPAjjsG\n094rK4MiaJ99gpWgP/sshKAldlT4iIhI3uRq1WeA446D1ath4kS48EIYOBCeey6EoCVWVPi0gpkd\nZ2ZrzewVM/th1PFIfq1duzbqECREyme0Glv1eWVTC/Skae3atXTtCtOmBVtfABxyCFRUQOoum4gK\nn3SZWVvgd8BQoAz4iZl1jzQoyasLLrgg6hAkRMpn9Or3/owcOZLdd989q/bq53TgQFi+HC6/HG65\nBQYMgPnzs41Y4kCFT/oOBla6+zp3/wCYD5RHHJPk0fTp06MOQUKkfBaOvn37Mn36dDp27JhVOw1z\n2r49nH9+sO9X//7BrbBRo2DduqwuI0VOhU/6+gD1t8mrAfpGFItEQNOf40X5jJ+mcrrbbvDww3DX\nXcGCh/37w623Nr/vl8RXSRQ+ZjbYzOaZWY2Z1ZpZopFzzjaz18zsIzNbamYDo4hVRETCZwZjxgSz\nvo4/PpgGP3QoaKhX6SmJwgfoArwAnAVstbyVmY0iGL9zKXAg8CLwqJn1rHfam8BO9Z73TR0TEZEc\n2rJlC7/4xS9Cmfm13XZw++3wxBPwf/8H++8Pv/wlfPJJCIFKUSiJwsfdH3H3S9z9QcAaOaUCuMXd\n73T3tcAZwCZgQr1zlgMDzGxHM+sKHA08muvYpXBcccUVUYcgIVI+i8eaNWu45pprGDBgQLOrPrcm\np0ceCS+9BOedB//7v8HGp4sXhxGtFLqSKHyaY2btCWZpPVF3zN0d+BNwWL1jW4BzgT8D1cBV7r6h\npfaPPfZYEonEFx6HHXYYc+fO/cJ5jz32GInEVnfgOPvss5k5c+YXjlVXV5NIJFi/fv0Xjl966aVb\n/eC//vrrJBKJrabuXn/99Vvta7Np0yYSiQSLG/z0J5NJxo8fv1Vso0aNKqnPsWnTplh8DohHPrL9\nHJs2bYrF54B45KO5z7FgwQK+//3vU1ZW9vmqz//617+2+hybNm1q1edYtOgxXn45QXU1fOlLMHgw\nnHEGnHqq8pHPz5FMJj//3di7d28SiQQVFRVbvScs5iW2sYmZ1QLHu/u81PMdCQYqH+buy+qddwUw\nxN0Pa7ylFq9zELBixYoVHHTQQSFELiJS2tyd2bNnM2XKFDp27MiMGTMYPnx4KG1v2QI33ww//Sl0\n6QLXXw8jRwZjgyT/qqurKSsrAyhz9+ow2y75Hh8RESkOZsbYsWO3WvV548aNWbfdti2cfXaw8vMh\nh8CJJwaDoN94I4TApaCo8IH1wBagV4PjvQCt9iAiUmD69OlDZWUld9xxB6+88grt27cPre2ddoK5\nc4Md3597DvbeO+j92bIltEtIxEq+8HH3zcAK4Ki6Y2ZmqedLsm2/oqKCRCJBMpnMtimJWMN74lLc\nlM/iVtf7s2zZMjp37gyEm9MTTgh6f8aOhSlTYNCgYDC05FbdeB+N8cmSmXUBdieY0VUN/BhYALzr\n7m+Y2feAWQSzuZYTzPL6LrCXu/87w2tqjE/MJBKJZmeUSHFRPuMnVzldsgROPz1Y8+e88+CSSyBV\na0mOaIxP9r4GPE/Qs+MEa/ZUA78AcPd7gfOAX6bO2w/4VqZFj8TTZZddFnUIEiLlM35yldNBg6C6\nGi69NNgAdd994fHHc3IpyYOSKHzc/Sl3b+PubRs8JtQ750Z339XdO7n7Ye7+XJQxS+FRz128KJ/x\n0zCntbW1PPXUU6G03aED/Pznwe2unXeG8vLgNpjumBafkih8oqQxPiIi0aisrGTo0KGcfPLJoaz6\nDLDnnsF+XzNnQlUV7LUXzJ4NJTBqJC80xqeIaYyPiEi0crnuD8Bbb0FFBSSTMGxYsA7Ql78cWvMl\nTWN8RApAwxVQpbgpn/HTMKdNrfsTVu9Pr15w993w0EPwl7/APvvA1KmweXMozUuOqPARSVN1daj/\n6ZCIKZ/x01RO66/7U1VVxT777ENlZWVo1z3mGFi5Mtju4qc/hYED4dlnQ2teQqZbXTlSd6tryJAh\ndOvWjdGjRzN69OiowxIRKWlvvvkmp59+OitXrmTt2rV07Ngx1PZXrIDTToMXX4Rzzgk2QO3aNdRL\nxFoymSSZTLJx40YWLlwIObjVpcInRzTGR0SkMLk7b731Fr17985J+599BtdcE6z307Mn3HQTfPvb\nOblUbGmMj4iISEjMLGdFD0C7dsFCh6tWBVteHHccjBoF67QJUkFQ4SMiIpIDu+0GDz8Md90FTz4J\n/fvDrbdCbW3UkZU2FT4iaUokElGHICFSPuMnrJy6eyg7vgOYwZgxwXYXI0YE43++8Y3guURDhY9I\nmiZNmhR1CBIi5TN+wsrpbbfdxl577RXqvl/bbQezZsGf/gQ1NbD//sHA508/De0SkiYNbs4RzeoS\nESlONTU1TJw4kfnz5zNmzBiuu+46evToEVr7H30UFD1XXgl77AEzZsDXvx5a80VNs7qKmGZ1iYgU\nL3fnzjvvZMqUKXTq1Ilbbrkl9NujL70U3PpavhzOPBN++1vo1i3USxQtzeoSERHJIzNj3Lhxn6/6\nPGLEiFBXfQbYbz9YsgSuuy7Y72vvveGPfwyteWmCCh+RNM2dOzfqECREymf85CKnffv2pbKyklmz\nZlFZWUl5eTlh3ilp2xYmT4bVq6GsDL7zHTjhhGAckOSGCh+RNCWTyahDkBApn/GTq5zW9f6sXr2a\n6dOnY2ahX2PnneHBB+EPf4ClS4Op7zfcAFu2hH6pkqcxPjmiMT4iIpKJDRvgwguDQc+HHgq//32w\nAWop0RifIlZRUUEikdD/LkVEJC3du8Mtt8DChfCf/8CBB8LFF8PHH0cdWe4lk0kSiQQVFRU5u4Z6\nfHJEPT4iIpKtTz4JZnv95jew665BL9DQoVFHlXvq8RERESlwd911V+gzv7bZBi67DF54AXbYIVj1\n+dRTIcRLlBwVPiJpGj9+fNQhSIiUz/iJOqft27enqqqKAQMGUFlZGWrbe+8d3Pq6+eZgAHT//jBn\nDuimTeup8BFJU3l5edQhSIiUz/iJOqejRo1i5cqVlJWVkUgkQu/9adMGJk6ENWtg8GAYPTrY+f2f\n/wztEiVBhY9ImrTlSLwon/FTCDltuO5PLnp/+vSB++6DuXPhxRdhwAC45hpNfU+XCh8REZEQNVz1\nOVezlEaMCBY+HD8efvzjYOr7iy+GfpnY0ayuHNEmpSIiUrfnV/fu3UPf66u+pUuDfb/WrIHzzoNL\nLoHOnXN2uZzRJqVFTNPZ42fx4sUcfvjhUYchIVE+46fUc/rpp3DVVfDLX0LfvsFA6GHDoo4qM5rO\nLlIApk6dGnUIEiLlM35KPacdOsBFFwW7vvfrB+XlMHYsrF8fdWSFRYWPSJrmzJkTdQgSIuUzfpTT\nwB57wJNPwsyZUFUVTH2/6y5Nfa+jwkckTZ2L8Ya5NEn5jJ9izWllZSXz5s0LtU0zmDAhGPNz1FFw\n8snwrW/B3/8e6mWKkgofERGRCD3wwAOMGDEi9HV/AHr1ChY6rKqCV14JNju98kr47LNQL1NUVPiI\niIhE6Lbbbsvpuj8A3/42rFoVLIB44YUwcCCsWBH6ZYqCCh+RNJ1//vlRhyAhUj7jp1hzWrfuz+rV\nq3O26jNA164wbVow9d0dDj4Yzj0XPvww1MsUPBU+Imnq169f1CFIiJTP+Cn2nPbp02erVZ8feuih\n0K8zcCA8+2yw4/uNNwYrPz/ySOiXKVgqfETSNHny5KhDkBApn/ETh5w2XPV5w4YNOblO+/bwk5/A\nypXw1a/CMcfAmDHw9ts5uVxB0QKGOaKVm0VEJBt1v5/NLMfXgdmzg20vjj0W7rwzp5drllZuLmJa\nuVlERIrJv/8dbHTau3fUkWjlZpGCsHbt2qhDkBApn/GjnGZn++0Lo+jJNRU+Imm64IILog5BQqR8\nxk+p5fTZZ58NfeZXKVDhI5Km6dOnRx2ChEj5jJ9SymltbS2nnHJKztb9iTMVPiJpKvapsvJFymf8\nlFJO27Rpw2OPPZbTdX/iSoWPiIhIEerbt+9W6/6EvedXHKnwERERKVIN1/3J1Z5fcaLCRyRNV1xx\nRdQhSIiUz/gp5ZzW9f7ccccdVFVVadxPM9pFHYBIsdi0aVPUIUiIlM/4KfWcmhljx47lmGOOoWfP\nnlGHU7C0gGGOaAFDERGRzGgBQxEREZEQqPAREREpIaU+8FmFj0ia1q9fH3UIEiLlM36U05Z9+OGH\nHHDAASU980uFT45VVFSQSCRIJpNRhyJZmjBhQtQhSIiUz/hRTlvWuXNnfvWrX1FVVVWQ6/4kk0kS\niQQVFRU5u4YGN+eIBjfHT3V1tXIZI8pn/Cin6aupqWHixInMnz+fk046iWuvvZYePXpEHdbnNLhZ\npADoCzVelM/4UU7T19iqz6Wy9o8KHxERkRLUcNXnUaNGsW7duqjDyjktYCgiIlLC6np/1q5dS+/e\nvaMOJ+fU4yOSppkzZ0YdgoRI+Ywf5TRzZkb//v2jDiMvVPiIpKm6OtTxdRIx5TN+lFNJh2Z15Yhm\ndYmIiGRGs7pEREREQqDCR0REREqGCh8REREpGSp8RNKUSCSiDkFCpHzGj3Iq6VDhI5KmSZMmRR2C\nhEj5jB/lVNKhwkckTeXl5VGHICFSPuNHOZV0qPARERGRkqHCR0REREqGCp9WMLMHzOxdM7s36lgk\n/+bOnRt1CBIi5TN+lFNJhwqf1rkGODnqICQaV1xxRdQhSIiUz/hRTiUdKnxawd0XAh9EHYdEY/vt\nt486BAmR8hk/yqmkQ4WPiIiIlIzYFj5mNtjM5plZjZnVmtlWK1uZ2dlm9pqZfWRmS81sYBSx5kMy\nmSyY9lrz3nTObemc5l5v6rWw/73Clov4Mm0z7Hy2dJ7ymds2W/u+XP6MFms+Qd+5rX0tnzmNbeED\ndAFeAM4CttqC3sxGAb8DLgUOBF4EHjWznvXOOcvMnjezajPbJj9h54Z+CFv3WqF/sRbrL0oVPo0r\n1nyme74Kn2jb03fuF7XL25XyzN0fAR4BMDNr5JQK4BZ3vzN1zhnAt4EJwNRUGzcCNzZ4n6UeLekI\nsGbNmkzCD93GjRuprq4uiPZa8950zm3pnOZeb+q1xo4vX7481H/DbISdz2zaDDufLZ2nfOa2zda+\nL5c/o609Huecltp3br3fnR1bDLqVzH2rzpDYMbNa4Hh3n5d63h7YBIysO5Y6Pgvo5u4nNNHO48B+\nBL1J7wInuvuyJs79AfD/wvwcIiIiJWaMu98dZoOx7fFpQU+gLfBWg+NvAXs29SZ3H9aKazwKjAH+\nAXzcyvhERERKWUdgV4LfpaEq1cIn59z9HSDUKlVERKSELMlFo3Ee3Nyc9cAWoFeD472AdfkPR0RE\nRPKhJAsfd98MrACOqjuWGgB9FDmqMEVERCR6sb3VZWZdgN357wysL5vZ/sC77v4GcDUwy8xWAMsJ\nZnl1BmZFEK6IiIjkQWxndZnZEcACtl7D5w53n5A65yzgAoJbXC8Ak939ubwGKiIiInkT28JHRERE\npKGSHONTCMzsODNba2avmNkPo45HsmNmD5jZu2Z2b9SxSPbMbCczW2Bmq8zsBTP7btQxSebMrJuZ\nPZtahf8lMzs16pgke2bWycz+YWZTW/U+9fjkn5m1BVYDRxDs9l4NHOLuGyINTDJmZkOA/wHGufv3\noo5HsmNmvYEd3P0lM+tFMBniq+7+UcShSQZSk1e2cfePzawTsAoo03ducTOzXwFfAd5w9wvSfZ96\nfKJxMLDS3de5+wfAfKA84pgkC+6+kKCIlRhI/Wy+lPr7WwRLYPSINirJlAfqFpLtlPozna2HpECZ\n2e4ECw4/3Nr3qvCJRh+gpt7zGqBvRLGISDPMrAxo4+41LZ4sBSt1u+sF4HXgSnd/N+qYJCtXAT8l\ngwJWhU8rmdlgM5tnZjVmVmtmiUbOOdvMXjOzj8xsqZkNjCJWaZnyGT9h5tTMegB3AKflOm5pXFj5\ndPeN7n4AsBswxsy2z0f88kVh5DP1nlfc/dW6Q62JQYVP63UhmPp+FltPlcfMRgG/Ay4FDgReBB41\ns571TnsT2Kne876pY5J/YeRTCksoOTWzDsAfgd80tRmx5EWoP6Pu/u/UOYNzFbA0K4x8Hgp838z+\nTtDzc6qZXZx2BO6uR4YPoBZINDi2FLi23nMD/gVcUO9YW+AVYEegK7AG6B715yn1R6b5rPfaUOAP\nUX8OPcLJKZAELon6M+iRfT6BHYCuqb93A14GBkT9eUr9ke13bur1ccDU1lxXPT4hMrP2QBnwRN0x\nDzLzJ+Cwese2AOcCfyaY0XWVa3ZBwUk3n6lzHwfuAY4xs9fN7JB8xirpSTenZvZ14ETgeDN7PjUN\nekC+45XmteJndBdgkZk9DzxF8It1VT5jlZa15js3G7HdsiIiPQl6c95qcPwtgtHnn3P3KqAqT3FJ\nZlqTz2H5CkqyklZO3f1p9P1YDNLN57MEt02ksKX9nVvH3e9o7UXU4yMiIiIlQ4VPuNYDWwj2/qqv\nF7Au/+FIlpTP+FFO40X5jJe85FOFT4jcfTPBCq9H1R1LrRh6FLAkqrgkM8pn/Cin8aJ8xku+8ql7\n2K1kZl2A3fnvugFfNrP9gXfd/Q3gamCWma0AlgMVQGdgVgThSguUz/hRTuNF+YyXgshn1NPZiu1B\nsL9WLUF3XP3HbfXOOQv4B/AR8Azwtajj1kP5LJWHchqvh/IZr0ch5FOblIqIiEjJ0BgfERERKRkq\nfERERKRkqPARERGRkqHCR0REREqGCh8REREpGSp8REREpGSo8BEREZGSocJHRERESoYKHxERre7/\nswAABZ5JREFUESkZKnxERESkZKjwERERkZKhwkdERERKhgofESkYZpYws8Vm9oKZfWJmtWb2nplt\n28T5vczsaTP7IHVurZm9b2YrzGxYvuMXkcKn3dlFpOCYmQHvAN1Sh85396ubOb8H8BegCjjT3T/K\nfZQiUoxU+IhIwTGz/YEFwKfADsCr7r5HC+95FTjQ3d/PQ4giUqR0q0tECtFg4M/ALannXzGz8qZO\nNrNdgRoVPSLSEhU+IlKIhgBPAjcBm1PHzm7m/COAp3IdlIgUPxU+IlKIBgNPuvs64F7AgGPNbOcm\nzh+CCh8RSYMKHxEpKGb2VYLxh6tTh65N/dkGOKOJtw0CluQ6NhEpfip8RKTQDCEY3wOAuz8HPEPQ\n6/NDM2tX/2Qz6wNs0EwuEUmHCh8RKTRDCGZ01XdN6s/tgRMbOT/r21xm1j7bNkSk8KnwEZFCUzew\nub4HgDdSfz+rwWtHAAtbatTMBprZjakFDx8zs3vN7FIz62hmuwC3Nzi/g5k9ZGavphZGrEk9f8jM\nFpjZX8zsD2bW38xOMLMnzezD1LmrU9fYsV57c8xsY+r118zs8lb/y4hI1rSOj4gUDDPbCXjG3bca\nxGxmFwCXAw4c4O4vp46/BHy9qansZrYdcAOwL3CBu8+v91oZ8CPgQCDp7r9u5P2jgLuBke4+t97x\n9kASODIVz+tmthTo7e67NhHL8cB33H1si/8YIpIT6vERkULS2G2uOr8H6sbxnAVgZj2Bj5spevYE\nlgFdgLL6RQ+Au68AlgP9m7nu4ak/Fzd472aCXqJtCcYedQYOaqYdgN1Sn0NEIqLCR0QKSZOFj7tv\nAGYTDHIeY2ZdU+cvaux8M9seeAR4D/ieu3/cxDUfAj4kKJAaczjBytHrG3mta+rP/yGYWdaO5scb\nHdLMdUQkD1T4iEghGUzzPSZ1U9u7AKfQ/MDmGcDOwOktzPh6D3ja3bc0fMHMvkRwi6ypqfJDCW69\nLUnF4k3Fk9p/rIO7f9pMLCKSYyp8RKQgpMbidHT3fzR1jruvAR5PPT2ToDdmqx4fM/s6MAJ4NDUd\nvjkfAJc28doggu/Jpxu5Rk9gFLDA3e8jKIJq3P21JtraD3i5hVhEJMdU+IhIofjC+j3NuIbgdtde\nQLvULbCGxhP0vtzeyGtf4O4fufvyJl4ezH97dD6XWmTxQeAxYLiZbQMMpPnbXGnNPhOR3GrX8iki\nInnxTZq/zQWAuz9sZn8BvkrThcQRqT//nGVMgwn2Cpsc3KnCgQ4EY4LOdfelAGY2BNiG5gufQWhg\ns0jkVPiISOTM7FvAOOCFNN9yferRVKHRF3ivsQHJZjYM+AmwA9AJqAVeA4anZmrVndeBoBdnibuf\n2UI8zY7vSemk1aVFoqdbXSISGTOrNLN/EMys6gTcZGYvm9mkFt46C1hH0z0+7wGbGnvB3R93928C\n5wJfAea4+9H1i56UgQS9OOncntoPeMfd/9rYi2Z2GFCdRjsikmPq8RGRyLj78Azf9yHQp5lTlgHH\nmFmX1LmNGUTQS/NwE6/Xje9JZzuMzkCjRU9KReohIhFTj4+IxNE0gu+30xp7MTW1/LsEY3WebaKN\nuvE9z6RxveeBnk1c6xzgMXevSaMdEckxFT4iEjvu/mfgZ8CvzWy8mX3+XZeahn4z8CrB+J3G1u9p\nT9AjVN3Mwof1TQe6mNmYem10NLNLgR3c/dasPpCIhEZ7dYlIbJnZUIJbTLsAbwH/Ad4GrgNeBw52\n90X1zt8FuDV1/leA94HngOn19+lq4lr9gKkEPT+fAJ8CN7v7o+F+KhHJhgofERERKRm61SUiIiIl\nQ4WPiIiIlAwVPiIiIlIyVPiIiIhIyVDhIyIiIiVDhY+IiIiUDBU+IiIiUjJU+IiIiEjJUOEjIiIi\nJUOFj4iIiJQMFT4iIiJSMv4/teAO3g6/yRQAAAAASUVORK5CYII=\n", "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -396,7 +422,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 80, "metadata": { "collapsed": false }, @@ -404,18 +430,18 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 26, + "execution_count": 80, "metadata": {}, "output_type": "execute_result" }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAGNCAYAAADpZIAdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3Xl4VdX1//H3CiIQQFDDoCjz6Fyi/sQKzoCIt7QOOLUa\nbJ1ALbZQq34L+ogKTlUBpRUFK8W5FKeviHUofgE10TqBIggordSICBoUJOv3x8083puce09y8nk9\nz31MztnnnHWzuMlyn733MXdHREREpLHLCDsAERERkSCoqBEREZFIUFEjIiIikaCiRkRERCJBRY2I\niIhEgooaERERiQQVNSIiIhIJKmpEREQkElTUiIiISCSoqBEREZFIiFxRY2aDzWyhmW0ws0IziyVw\nzDFmlmtm35nZR2Z2XjpiFRERkeBErqgBWgNvA5cCtT7Yysy6A08DLwIHA3cC95nZiakLUURERIJm\nUX6gpZkVAqPcfWENbaYCJ7n7QWW2zQfaufuINIQpIiIiAYhiT02yjgAWV9j2PDAohFhERESkjlTU\nQGdgY4VtG4HdzKxFCPGIiIhIHewSdgCNkZntCQwD1gLfhRuNiIhIo9IS6A487+5fBnliFTXwOdCp\nwrZOwBZ3/76aY4YB81IalYiISLSdA/w1yBOqqIGlwEkVtg0t2l6dtQAPPfQQAwYMSFFYiRs/fjx3\n3HFHgzhfMscm0ra2NjXtr25fVduD/hnWRypiqes5g85nbe2Uz9SeM9njUvkZDWp7GPQ7t36f0RUr\nVnDuuedC0d/SIEWuqDGz1kBvwIo29TSzg4FN7v6pmd0E7O3uxWvR3AuMLZoFdT9wPHAaUNPMp+8A\nBgwYwMCBA1PxNpLSrl27QOOoz/mSOTaRtrW1qWl/dfuq2h70z7A+UhFLXc8ZdD5ra6d8pvacyR6X\nys9oUNvDoN+59f+MFgl8+EYUBwofCrwF5BJfp+Y2IA+4rmh/Z2Df4sbuvhY4GTiB+Po244EL3L3i\njKgG66yzzmow50vm2ETa1tampv3V7atq++eff15rLOkSdD7rc86g81lbO+UztedM9rhUfkaT3R7l\nnDbF37mpEul1alLFzAYCubm5uQ3m/xykfrp06cKGDRvCDkMConxGj3IaHXl5eWRnZwNku3tekOeO\nYk+NSNKKPmASEcpn9CinkggVNSKkt3tUUk/5jB7lVBIRuYHCInWhX5jRUlM+169fT35+fhqjkSD0\n69ePvLxA71RICmVlZdG1a9e0X1dFjYg0GevXr2fAgAEUFBSEHYpIpGVmZrJixYq0FzYqakSAnJwc\nHnjggbDDkIBUl8/8/HwKCgoazBpTIlFUvA5Nfn6+ihqRMAwdOjTsECRAteWzoawxJSLB0kBhETSm\nJmqUT5GmSUWNiIiIRIKKGhEREYkEFTUiwJIlS8IOQQKkfIo0TSpqRIBp06aFHYIESPksb/LkyWRk\n6Ne9RJ/+lYsADz/8cNghSICUz/LMDDNL+rh77rmHuXPnpiAikdRQUSNCfKEoiQ7lMxgzZ85UUSON\niooaEZEauHujPLdIU6SiRkSkgq1btzLp8ss5oUcPRu27Lyf06MGkyy9n69atDfrcEB8kfdhhh9Gq\nVSv69OnDn/70p0ptHnjgAY4//ng6depEy5Yt2X///bn33nvLtenRowfvv/8+L7/8MhkZGWRkZHDc\ncccB8NVXX/Hb3/6Wgw46iLZt29KuXTtGjBjBO++8E8h7EKkrrSgsAkyYMIFbbrkl7DAkIPXJ59at\nWzl10CCuXLGCyYWFGODA8zNmcOo//sETS5fStm3bBndugPfee49hw4bRsWNHrr/+enbs2MHkyZPp\n2LFjuXb33nsvBxxwAD/5yU/YZZddeOqpp7j00ktxdy655BIA7rzzTsaNG0fbtm259tprcXc6deoE\nwJo1a1i4cCGnn346PXr0YOPGjcyaNYtjjjmGDz74gM6dO9f5PYjUi7vrleQLGAh4bm6uSzTcdddd\nYYcgAaoun7m5uV7bZ/cPl13mz2VkuEOl17MZGT7p8svrHFcqz+3uPmrUKM/MzPTPPvusZNvKlSt9\nl1128YyMjJJt3333XaVjhw8f7r179y637YADDvBjjz22Utvt27dX2rZu3Tpv2bKl33DDDfV5CxIB\ntX3OivcDAz3gv8+6/SQCXHbZZWGHIAGqTz5fe+ophhUWVrlveGEhry1c2CDPXVhYyKJFi/jpT39K\nly5dSrb369ePYcOGlWvbokWLkq+3bNnCl19+yZAhQ1izZk1Ct8GaN29e7rqbNm0iMzOTfv36kZeX\nV+f3IFJfKmpERIq4O6137KC6yc8GZO7YUacBvqk8N8AXX3zBtm3b6N27d6V9/fr1K/f9a6+9xgkn\nnECbNm1o3749HTp04JprrgHg66+/rvVa7s4dd9xB3759adGiBVlZWXTs2JF33303oeNFUkVFjYhI\nETPj2+bNqa6scODb5s3rtOZLKs+djNWrV3PCCSewadMm7rjjDp599lkWL17M+PHjgXjPS22mTJnC\nb37zG4455hjmzZvHokWLWLx4Mfvtt19Cx4ukigYKiwArV66kf//+YYchAalPPn98yik8P2MGw6v4\n4/y/GRkcFYvVOa5UnrtDhw60atWKVatWVdq3cuXKkq+feuoptm/fzlNPPVXuNtWLL75Y6bjqCqwn\nnniC4447rtLMqs2bN9OhQ4e6vgWRelNPjQgwceLEsEOQANUnn7+dMoXbBwzguYyMkl4VB57LyOCO\nAQP4zQ03NMhzZ2RkMGzYMBYsWMBnn31Wsn3FihUsWrSo5Ptddon/v2zZHpWvv/6aOXPmVDpn69at\n2bx5c6XtzZo1q3Sb7LHHHmPDhg11jl8kCCpqRIDp06eHHYIEqD75bNu2LU8sXcryceMY2r07P+nS\nhaHdu7N83Lh6T7lO5bkBrrvuOtydo446imnTpjFlyhSOO+44DjjggJI2Q4cOpXnz5owcOZKZM2cy\ndepUDj300JLp2mVlZ2fzzjvvMGXKFB555BFeeuklAEaOHMnLL7/MmDFjuO+++7jiiiu45JJL6NWr\nV73iF6m3oKdTNYUXmtIt0iglMqW7osLCwpTFk4pz//Of//TDDjvMW7Zs6b179/Y//elPPnny5HJT\nup9++mk/5JBDPDMz03v27Om33nqrP/DAA56RkeHr1q0rabdx40Y/5ZRTvF27dp6RkVEyvfv777/3\nCRMmeJcuXbx169Y+ZMgQX758uR977LF+3HHHBf6epHEJc0q3eR1H2jdlZjYQyM3NzWXgwIFhhyMi\nCcrLyyM7Oxt9dkVSp7bPWfF+INvdA10DQLefREREJBJU1IgAU6dODTsECZDyKdI0qagRAQoKCsIO\nQQKkfIo0TSpqRIjPGpHoUD5FmiYVNSIiIhIJKmpEREQkElTUiAD5+flhhyABUj5FmiYVNSLAmDFj\nwg5BAqR8ijRNKmpEgMmTJ4cdggRI+RRpmlTUiIBWl40Y5VOkaVJRIyIiIpGgokZEJALmzJlDRkYG\n69evDzuUGq1bt46MjAwefPDBhNvefvvtKY/rlVdeISMjg1dffTXl1wpaVbGff/759OjRI8SowqGi\nRgSYPXt22CFIgJpiPs0MMws7jIRUjPO5555rEAsmNpafX1Uqxm5mZGQ0vT/xTe8di1QhLy/QB8VK\nyJTPhqtbt25s27aNn//85yXbnn32Wa6//voQo4Kjjz6abdu2MWTIkFDjCMp9993HypUrww4j7XYJ\nOwCRhmDGjBlhhyABUj4bnp07d1JYWEjz5s3Zddddy+1z95CiKq9iXI1Zs2bNaNasWdhhpJ16akRE\nImzmzJkccMABtGzZki5dujBu3Di+/vrrSu1mzJhBr169yMzM5IgjjmDJkiUcc8wxHHfccSVtduzY\nwR/+8AcOPfRQ2rdvT5s2bRgyZAgvv/xyuXOVHQtz55130rt3b1q2bMmKFSsqjanJyclh5syZAGRk\nZJCRkVHlH+M///nPJec5/PDDefPNN8vtP//882nbti2ffvopI0eOpG3btuyzzz4l53733Xc5/vjj\nadOmDd27d2f+/Pnljq9uTM3y5csZMWIEe+yxB23atOHggw/mrrvuqvFn/sMPP3DdddfRt29fWrVq\nRVZWFoMHD+bFF18s1+7DDz/kjDPOoGPHjmRmZtK/f3+uvfbakv3r16/n0ksvpX///mRmZpKVlcUZ\nZ5zBunXrarx+8c+j7Jiasjmp7WcJ8Nhjj7H//vvTqlUrDjroIBYsWNAoxumop0ZEJKImT57M9ddf\nz9ChQ7n00kv58MMPmTlzJm+++SavvfZaSfFwzz33cNlll3H00Udz5ZVXsnbtWkaNGsXuu+/Ovvvu\nW3K+LVu2cP/993PWWWdx4YUXsnXrVmbPns3w4cN5/fXXOeigg8pd//777+f777/noosuokWLFuyx\nxx7s3LmzXJuLL76Yf//73yxevJh58+ZV2Wszb948vvnmGy6++GLMjKlTp3LqqaeyZs2akvdgZhQW\nFnLSSSdx9NFHc8sttzBv3jwuu+wyWrduzTXXXMO5557Lqaeeyr333st5553HkUceSbdu3UquU3Fc\nygsvvMApp5zC3nvvza9//Ws6d+7MihUreOaZZ7j88sur/blPmjSJm2++mQsvvJDDDjuMLVu28Oab\nb5KXl8fxxx8PwDvvvMPgwYNp0aIFF110Ed26dWP16tU8/fTT3HDDDQC88cYbLFu2jLPOOot99tmH\ntWvXMnPmTI499lg++OADWrZsWW0M1Y2xSuRn+cwzz3DmmWdy8MEHc/PNN/PVV19xwQUX0KVLl4Y/\n7sjd9UryBQwEPDc310Wk8cjNzfWofnbnzJnjGRkZvm7dOnd3/+KLL7xFixZ+0kknlWs3Y8YMz8jI\n8Dlz5ri7+/bt2z0rK8uPOOII37lzZ0m7Bx980M3Mjz322JJthYWFvmPHjnLn+/rrr71z587+y1/+\nsmTb2rVr3cy8ffv2/uWXX5ZrX7xv7ty5JdvGjRvnGRkZld5TcdsOHTr4119/XbJ94cKFnpGR4c88\n80zJtvPPP98zMjJ86tSpJds2b97smZmZ3qxZM3/sscdKtn/44YduZn7dddeVbHv55Zc9IyPDX3nl\nFXd337lzp/fo0cN79uzpW7ZsqRRbTQ455BA/5ZRTamwzZMgQb9eunX/22WfVtvnuu+8qbVu+fLmb\nmT/00EPVxu4e/3n06NGj5PtkfpYHHnigd+3a1QsKCkq2vfrqq25m5c5Zndo+Z8X7gYEe8N9n9dSI\nALFYjIULF4YdhgQkqHwWFECqx1r27w+ZmcGfd/HixezYsYNf//rX5bb/6le/4uqrr+aZZ57hvPPO\n44033uDLL79k6tSp5WbLnH322ZWONTN22SX+Z8Pd2bx5Mzt37uTQQw+tcnD2aaedxh577FHv93Lm\nmWey2267lXw/ePBg3J01a9ZUanvBBReUfN2uXTv69evH6tWrOe2000q29+3bl/bt21d5fLG33nqL\ntWvXcuedd9K2bduk4m3fvj3vv/8+H3/8Mb179660Pz8/n3/+85+MHz+eLl26VHueFi1alHz9ww8/\nsGXLFnr27En79u3Jy8vjnHPOSSouqP1n+Z///If33nuPa6+9llatWpVrd+CBB7J169akr5lOKmpE\ngHHjxoUdggQoqHyuXAnZ2YGcqlq5uZCKBZCLx1307du33PbmzZvTs2fPkv3r16/HzOjVq1e5ds2a\nNaN79+6Vzjt37lxuv/12Vq5cyY4dO0q29+zZs1Lbqo6vi7K3wCBeNAB89dVX5ba3bNmSPffcs9y2\ndu3asc8++1Q6Z7t27SodX9bq1asxM/bff/+k473++usZNWoUffv25YADDmD48OH8/Oc/58ADDwQo\nKSBqO/d3333HjTfeyJw5c9iwYUPJrTkzq3JcVCJq+1kW/7uo+O8BoHfv3rz11lt1um66qKgRAYYO\nHRp2CBKgoPLZv3+86Eil/v1Te/4gPfTQQ+Tk5PCzn/2MiRMn0rFjR5o1a8aNN95YZa9H2f/Tr4/q\nZvEU/5GvrV2ixwdl8ODBrF69mr///e8sWrSI2bNnc8cddzBr1qykHrY6btw45s6dy/jx4zniiCNo\n164dZsbo0aMpLCysU2zp/lmkm4oaEZFqZGamphclHYoHwH744Yflekx27NjBJ598woknnljSzt35\n+OOPOfroo0va7dy5k7Vr13LwwQeXbHviiSfo1asXjz/+eLlr/eEPf6hXrA1x8GmvXr1wd957771y\nM8AS1b59e8477zzOO+88CgoKGDx4MJMnT2bMmDElvVrvvfdejed44oknOP/885k2bVrJtu+//57N\nmzcnHU+iiv/dfPzxx5X2VbWtodGUbhGRCDrhhBNo3rx5penH9913H1u2bGHkyJEAHHrooey55578\n+c9/Lvd//w899FCl2zNV/V/+8uXLWbp0ab1ibd26NRCfXdVQDBw4kB49evDHP/4x6Vs9mzZtKvd9\nZmYmvXv35vvvvwcgKyuLIUOGcP/99/Ppp59We55mzZpV6pG56667Ks0gC9Jee+3FAQccwIMPPkhB\nQUHJ9ldeeYV33303ZdcNinpqRIAFCxYwatSosMOQgCif8T+cv//977n++usZPnw4sViMlStXcs89\n93D44YeXDDJt3rw5kydP5vLLL+fYY4/ljDPOYO3atTzwwAP07t27XC/KyJEjefLJJxk1ahQnn3wy\na9asYdasWey///588803dY41Ozsbd+eyyy5j2LBhNGvWjNGjR9f7Z5CssrdgzIx77rmHWCzGIYcc\nQk5ODnvttRcrV67kgw8+4Lnnnqv2PPvttx/HHHMM2dnZ7LHHHrzxxhs8/vjj5aaB33XXXQwePJiB\nAwdy4YUX0qNHDz755BOeffbZknErI0eO5C9/+Qu77bYb++23H0uXLuXFF18kKyurxtjr68Ybb2TU\nqFEceeSR5OTksGnTJmbMmMGBBx5Yrzyng3pqRKDSQlzSuCmfcZMmTWL69Ol8+umnXHnllTz++ONc\nfPHFPP/88+V6XcaOHctdd93Fp59+yoQJE3j11VdZuHAh7dq1K7cWyvnnn89NN93EO++8wxVXXMEL\nL7zAvHnzyM7OrvLZQ9XdVqq4/Wc/+xmXX345zz//PL/4xS84++yzaz1PVdsTvV6ixw8dOpSXXnqJ\nfv36cfvtt/Ob3/yGf/zjH8RisSqvU+yKK65g3bp13HzzzVxxxRX885//5MYbb+TWW28taXPQQQex\nbNkyjj76aO69916uuOIK/va3v5Urxu+66y5+8Ytf8Ne//pXf/va3bNy4kcWLF9OmTZuE3nuiOam4\nfeTIkcyfP58dO3Zw1VVX8eSTT3L//ffTt2/fGtfGaQgsKoOD0snMBgK5ubm5DGysN9xFmqC8vDyy\ns7PRZ7d27k6HDh049dRTmTVrVtjhSAPwox/9iI4dO/L888/X2K62z1nxfiDb3QN9UFske2rMbKyZ\nfWJm28xsmZkdVkv7c8zsbTP71sz+bWazzaz+iyuIiDQCxWM9ypo7dy6bNm3i2GOPDSEiCdMPP/xQ\nadzOyy+/zL/+9a8G/+8hcmNqzGw0cBtwIfA6MB543sz6unt+Fe1/DMwFrgCeBroAs4A/AadVbC8i\nEjXLli1j/PjxnH766ey5557k5uZy//33c9BBB5VbtE6ahg0bNnDCCSdw7rnnsvfee7NixQpmzZrF\n3nvvzUUXXRR2eDWKXFFDvIiZ5e4PApjZxcDJwBhgWhXtjwA+cffix/quM7NZwMR0BCsiErbu3bvT\ntWtX7r77bjZt2sQee+xRMn6meAVhaTp23313Dj30UGbPns0XX3xB69atOeWUU7jpppvYfffdww6v\nRpH612pmzYFs4Mbibe7uZrYYGFTNYUuBKWZ2krs/Z2adgNOBZ1IesDQYOTk5PPDAA2GHIQFRPpPT\nrVs3FixYEHYY0kDstttujXawfdTG1GQBzYCNFbZvBDpXdYC7/x9wLvCImW0H/gN8BWjd/CZEKwpH\ni/Ip0jRFrahJmpntB9wJTCb+9O1hQA/i42qkiTjrrLPCDkECpHyKNE1RK2rygZ1ApwrbOwGfV3PM\nVcBr7n67u7/n7i8AlwJjim5FVWvEiBHEYrFyr0GDBlXqxl20aFGV6xqMHTuW2bNnl9uWl5dHLBYj\nP7/8mOZJkyYxderUctvWr19fsqBWWXfffTcTJkwot62goIBYLMaSJUvKbZ8/fz45OTmVYhs9erTe\nh95HJN+HiKTP/PnzS/42du7cmVgsxvjx41N2vcitU2Nmy4Dl7n5F0fcGrAfucvdbqmj/OLDd3c8u\ns20QsATo4u6ViiGtUyPSOGmdGpHU0zo1wbod+JWZ/cLM+gP3ApnAHAAzu8nM5pZp/xRwqpldbGY9\niqZ430m8MKqud0cipmJPgzRuyqdI0xSp2U8A7v6omWUB1xO/7fQ2MMzdvyhq0hnYt0z7uWbWBhgL\n3ApsBl4kfltKmohp06Zx1FFHhR2GBKS2fK5YsSKN0Yg0LWF+viJ3+ykddPspegoKCsjMzAw7DAlI\ndflcv349AwYMKPf0YREJXmZmJitWrKBr166V9qXy9lPkempE6kIFTbRUl8+uXbuyYsWKSgONRSRY\nWVlZVRY0qaaiRkSalK5du4byy1ZEUi+KA4VFRESkCVJRIwKV1l+Rxk35jB7lVBKhokYEdDsiYpTP\n6FFOJRGa/VQHmv0kIiJSN1p8T0RERKQWKmpEREQkElTUiIAeehgxymf0KKeSCBU1IsDEiRPDDkEC\npHxGj3IqiVBRIwJMnz497BAkQMpn9CinkggVNSJoumjUKJ/Ro5xKIlTUiIiISCSoqBEREZFIUFEj\nAkydOjXsECRAymf0KKeSCBU1IkBBQUHYIUiAlM/oUU4lEXpMQh3oMQkiIiJ1o8ckiIiIiNRCRY2I\niIhEgooaESA/Pz/sECRAymf0KKeSCBU1IsCYMWPCDkECpHxGj3IqiVBRIwJMnjw57BAkQMpn9Cin\nkggVNSKgWWwRo3xGj3IqiVBRIyIiIpEQWlFjZt3N7Hwzax1WDCIiIhIdaStqzKyVmWUVf+/ua4HX\ngWvNbFC64hCpyuzZs8MOQQKkfEaPciqJSEtRY2a/BL4CNprZ52Y218xGAh+5+++Bc9IRh0h18vIC\nXdRSQqZ8Ro9yKolIy2MSzOwfwD3AdmAgcBKQTbzQeQno4O7HpDyQgOgxCSIiInWTysck7BLkyWrw\nprs/VvT134FJZrYvcBrQHbguTXGIiIhIRAVa1BSNjfkR8Brwjpd2AxWaWXN331Hc1t0/Be4I8voi\nIiLSdAXdU9MLmA448I2ZLQf+D1gJzDCzX7u7nh8vIiIigQt6oPAXwIPAIOB64FtgLHA/8EvgAzOb\nYmZDNZVbGpJYLBZ2CBIg5TN6lFNJRNA9NW8DO9z9deLTtW8DMLMBwFHAYOBs4PfAD2b2NvAq8Li7\nLws4FpGEjRs3LuwQJEDKZ/Qop5KItMx+qnRRs32AIZQWOpnu3ivtgdSRZj+JiIjUTRRmP2FmBwEf\nu3uBu38G/NXM8tz90nTFICIiItGVrsX3riJ+O+rZCrs6mdnN6YhBREREoi1dj0nYHTgfmF92o7u/\nAuSZ2clpikOkSgsWLAg7BAmQ8hk9yqkkIl1FTXN3f9jdZ1Xc4e6PAiemKQ6RKs2fP7/2RtJoKJ/R\no5xKItJV1OxpZjUNBN6epjhEqvTII4+EHYIESPmMHuVUEpGuouZOYHENt5napCkOERERiai0zH5y\n9zwzuxZ40sw+AhYQX9NmC/Fp3SpqREREpF7SNqXb3eeZ2QpgKnA1YEW7XgDOTFccIiIiEk3puv0E\nxHts3P1EoBNwBNDT3Ye5+1fpjEOkopycnLBDkAApn9GjnEoi0rVOzeCy37t7PvGneK9Nx/VFajN0\n6NCwQ5AAKZ/Ro5xKItLVU/P7KrbtbWYzzGxkmmIQqdZZZ50VdggSIOUzepRTSURKihoz+9jM3jCz\n28xsFKXjZ0q4+xp3HwvsbmbnpSIOERERaTpS1VNzIfAG8UX1ngCGmtmHZnafmZ1nZj2KG7r7X4CD\nUhSHiIiINBEpKWrc/R/ufqm7HwTsCbwPPA70BmYCH5vZZ2b2SNGznw5ORRwiiVqyZEnYIUiAlM/o\nUU4lESkfU+Pum4EN7n6Nux8DtAOOBG4ratIbmJDqOERqMm3atLBDkAApn9GjnEoi0rVOzX3FX7j7\nD8DyopdIg/Dwww+HHYIESPmMHuVUEpGW2U/u/kTFbWbWzcymmtlJ6YhBpCaZmZlhhyABUj6jRzmV\nRKRrnZrbiqZvn25mnQDcfR1wFVBoZn8M+HpjzewTM9tmZsvM7LBa2u9qZlPMbK2ZfWdma8zs/CBj\nEhERkdRK1+2nL4HJwCWAm9kq4BXgVWAN0DWoC5nZaOLjdS4EXgfGA8+bWd+iRf+q8hjQAcgBVgN7\nkebVlkVERKR+0vWHuyMwDMgCRgELiU/jvp94cRPk+JrxwCx3f9DdVwIXAwXAmKoam9lwYDAwwt1f\ncvf17r7c3ZcGGJM0cBMmaKx6lCif0aOcSiLSVdR8X1QwbHL3p9x9orsPAvYhPtX7wSAuYmbNgWzg\nxeJt7u7AYmBQNYedArwJ/K5omvmHZnaLmbUMIiZpHLp2DayzUBoA5TN6lFNJRLqKmm5m1q7iRnf/\nAvglwU3pzgKaARsrbN8IdK7mmJ7Ee2r2J96LdAVwGjAjoJikEbjsssvCDkECpHxGj3IqiUhXUTMP\n+JeZVXoimbsXAJ6mOKqSARQCZ7v7m+7+v8CVwHlm1iLEuERERCQJ6ZrS/RRwO/Csma0zsz+Z2Rgz\nO8HMfgn0CehS+cBOoFOF7Z2Az6s55j/EFwf8psy2FcSfV7VPTRcbMWIEsVis3GvQoEEsWLCgXLtF\nixYRi8UqHT927Fhmz55dblteXh6xWIz8/PJjmidNmsTUqVPLbVu/fj2xWIyVK1eW23733XdXuv9c\nUFBALBartCrn/PnzycnJqRTb6NGj9T70PvQ+9D70PvQ+6vU+5s+fX/K3sXPnzsRiMcaPH1/pmKBY\nfMhJepiG6e8dAAAgAElEQVRZNnADcALx20QA7wJnuPuHAV1jGbDc3a8o+t6A9cBd7n5LFe1/BdwB\ndCzqNcLMfkJ8rE8bd/++imMGArm5ubkMHDgwiLAlZCtXrqR///5hhyEBUT6jRzmNjry8PLKzswGy\n3T0vyHOnddqyu+e6+0nEp08fDvR394ODKmiK3A78ysx+YWb9gXuBTGAOgJndZGZzy7T/K/Ep5w+Y\n2QAzGwJMA2ZXVdBINE2cODHsECRAymf0KKeSiHStUwOAmbUGWhWtF/NmKq7h7o+aWRZwPfHbTm8D\nw4oGJUN8wPC+Zdp/a2YnAncTf7L4l8AjwP+kIj5pmKZPnx52CBIg5TN6lFNJRFqKGjPrQryn5Nj4\nt/Yl8QXvphWtLBwod59J/GngVe2rdAPQ3T8ivo6ONFGaLhotymf0KKeSiHTdfrqP+Oq+lwI3Av8i\nvhjee2Z2RppiEBERkQhL1+2nD9z9mrIbzCwT+Dlwj5ltdPdX0hSLiIiIRFDgPTVF07UvMLO+ZTZ7\n0Syk0g3uBe4+CziM+IJ3IqGpON1RGjflM3qUU0lEUkWNmf3BzM41s0PNrE01zX4C/AlYYWb/MbNH\ngebAo2a2a8XG7r4GWJtk3CKBKigoCDsECZDyGT3KqSQiqXVqzKyQ+Oq/W4EtwPfEC5KxRYNtMbM3\ngSHAoUX/HUL8uUutgS+AJUWvN4gviDcAONfdRwfyjtJA69SIiIjUTSrXqanLmJpL3P1Pxd+Y2R7A\n12X2X1m0iN2rRS/MrBmlRc7RwLXA7sQfT/Ai8Ks6RS8iIiJSJNmi5tOyBQ2Au2+q8P2rFQ9y953A\n8qLXLQBFa8l86+7bkoxBREREpJJkBwp/GtSF3T1fBY00FBWflSKNm/IZPcqpJCLZomZ7SqIQCdmY\nMWPCDkECpHxGj3IqiUjrs59EGqrJkyeHHYIESPmMHuVUEpFsUdPVzFqmJBKREGkWW7Qon9GjnEoi\nki1qegL/NbNnzewqMzvSzJKeQWVm05I9RkRERKQmdZnS3QYYTukDIL8zs+WUTuFemsAA4MPrcF0R\nERGRaiXbU7MB+B9gEfAtYEAr4Jii7S8AX5nZ/5nZzWY2wsx2q+I8uoUlDcrs2bPDDkECpHxGj3Iq\niUi2qFnt7lPcfTjQnniPy2+Bp4DNxIucXYEjgAlF2780szwz+6OZ/dTMugLdAnsHIgHIywt0UUsJ\nmfIZPcqpJCLZxyT8w92Pq2H/gZQ+GmEI0KnM7nIXcvdmyYXacOgxCSIiInXTkB6TUN1DLAFw93eB\nd4EZAEVP6i5b5HQtbprkdUVERERqlGxR0yWZxkUPufwIuA/AzLoBZwM3JHldERERkRolO6ZmLzM7\nrK4Xc/d17n4T8EldzyEiIiJSlbqsKHyPmdV4GyoB/63n8SKBisViYYcgAVI+o0c5lUQkW9RcDbQA\n3jWzMWbWqo7X1TOkpEEZN25c2CFIgJTP6FFOJRFJFTXufrO7HwjEgN7AMjObbWadajm0opeSbC+S\nUkOHDg07BAmQ8hk9yqkkoi4rCped5XS1mQ0CdiR5/HV1ua6IiIhIdepU1JTl7kuDCERERESkPhK+\n/WRmfc2sLgOLRRq8BQsWhB2CBEj5jB7lVBKRTJHyKLDBzG4tWjlYJDLmz58fdggSIOUzepRTSUTC\nRY27HwKcCewOLDGzt8zs12bWIWXRiaTJI488EnYIEiDlM3qUU0lEsrOfXnH3C4DOwG3ACGC9mS00\ns5+ZWfNUBCkiIiJSmzqNkXH3be7+kLsPBfoA/wdMAf5jZjPqs+qwiIiISF3Ue+Cvu39WtH7NAODk\nos3/a2YrzOwqM0vqeVEiIiIidRHobCZ3X+7uY4G9gP8BjgRWm9kiMzu7HisQi6RUTk5O2CFIgJTP\n6FFOJREpmaLt7tvd/XF3jwH7As8CE4DPi1YgHpyK64rUlVYrjRblM3qUU0mEuXv6LmZ2MPAL4Byg\nAHgQeNDd16QtiACY2UAgNzc3l4EDB4YdjoiISKORl5dHdnY2QLa75wV57rQupufu/3L33wBdgMuB\n/YD3zOxVM7vAzNqmMx4RERGJjlBWCHb3ne7+tLufAewN/BW4gPjsqfvCiElEREQat9Afe+Dum939\nXnc/EhgIvBJ2TNL0LFmyJOwQJEDKZ/Qop5KI0Iuastz9I3f/S9hxSNMzbdq0sEOQACmf0aOcSiIa\nVFEjEpaHH3447BAkQMpn9Cinkoi0FTVmdpCZZVbY1i9d1xepSWZmZu2NpNFQPqNHOZVEpKWoMbOr\ngNeJr1dTVmczuzkdMYiIiEi07ZKm6+wOnA+0K7vR3V8xs05mdrK7P5OmWERERCSC0nX7qbm7P+zu\nsyrucPdHgRPTFIdIlSZMmBB2CBIg5TN6lFNJRLqKmj3NrFcN+7enKQ6RKnXt2jXsECRAymf0KKeS\niHQVNXcCi83s5Gr2t0lTHCJVuuyyy8IOQQKkfEaPciqJSMuYGnfPM7NrgSfN7CNgAfA2sAU4ChU1\nIiIiUk/pGiiMu88zsxXAVOBqwIp2vQCcma44REREJJrS/UDLPHc/EegEHAH0dPdh7v5VOuMQqWjl\nypVhhyABUj6jRzmVRIT1QMt84H3gyzCuL1LRxIkTww5BAqR8Ro9yKolI2+2nKrQCrjSzQmCKu28L\nMRZp4qZPnx52CBIg5TN6lFNJRFqKGjPrAUwC9gT+BSx099eBq82sC3AD8Jt0xCJSFU0XjRblM3qU\nU0lEum4/zSE+jqY5kAMsM7NVZnY1sAfQIU1xiIiISESlq6jJc/eT3H24u3chPkj4KeAK4lO7NwZ5\nMTMba2afmNk2M1tmZocleNyPzWyHmeUFGY+IiIikXrqKmu/LfuPur7v7lUBnIMvdA1v/2sxGA7cR\nv931I+K3u543s6xajmsHzAUWBxWLNB5Tp04NOwQJkPIZPcqpJCJdRc3bZvazihs9Lujp3OOBWe7+\noLuvBC4GCoAxtRx3LzAPWBZwPNIIFBQUhB2CBEj5jB7lVBJh7p76i5jtCjwJvAlML5rSnYrrNCde\nwJzq7gvLbJ8DtHP3n1ZzXA5wEXAk8D/AT9x9YA3XGQjk5ubmMnBgtc1ERESkgry8PLKzswGy3T3Q\n4R7p6qmZDxxOvGDYaGbvm9lMMzvTzPYK8DpZQDMqj9HZSPxWVyVm1ge4ETjH3QsDjEVERETSKF1F\nzX/cvSOwO/AT4FngUOAvwGdm9kKa4ijHzDKI33Ka5O6rizeHEYuIiIjUT7qKmjfMbApwCPCsu09w\n98OJFzmnEC8sgpAP7CQ+fbysTsDnVbRvS7y4ml4062kH8d6kQ8xsu5kdU9PFRowYQSwWK/caNGgQ\nCxYsKNdu0aJFxGKxSsePHTuW2bNnl9uWl5dHLBYjP7/8HbpJkyZVGii3fv16YrFYpeXD7777biZM\nKD/2uqCggFgsxpIlS8ptnz9/Pjk5OZViGz16dJN6H/n5+ZF4HxCNfNT3feTn50fifUA08hHE+8jP\nz4/E+4Bo5CPR9zF//vySv42dO3cmFosxfvz4SscEJV1jai4CdiX+NO657v7vFF5rGbDc3a8o+t6A\n9cBd7n5LhbYGDKhwirHAscCpwNqqVjrWmJroicViLFy4sPaG0igon9GjnEZHKsfUpOsxCScSv+3U\nDLjczP4JvAq86u7vBHyt24E5ZpYLvE58NlQm8QUAMbObgL3d/TyPV3QflD3YzP4LfOfuKwKOSxqw\nyZMnhx2CBEj5jB7lVBKRlqLG3U8zszbEZxcdVfS6GWhlZpuB14AFwDx3/776MyV0rUeL1qS5nvht\np7eBYe7+RVGTzsC+9bmGRI963KJF+Ywe5VQSkZbbT1Ve2GwXIJv4zKN9gZ7AOuCnKei9CZRuP4mI\niNRNFKZ0V+LuP7j7cuAk4G/AXsBs4Gkz05PLREREJClpKWrM7FgzW2Bm08zsoLL73H07kOHuX7j7\njcBPgT+kIy6RYhVnEUjjpnxGj3IqiUhXT83vgTXASOAtM/vAzG43s4vM7CpgUHFDd88FtqQpLhEg\n3h0q0aF8Ro9yKolI15TuO8tMsT4COAsYCnQD1gJj3H2ZmU0H8oDe7n51ygOrI42pERERqZsoTOl+\nwMz+CDzs7suo/qGRhwK/ACamKS4RERGJiHRN6X7bzCYCPzOzz9z9s2qaHgm0d/dN6YhLREREoiNd\nPTUAzd394ZoaFD1QUgWNiIiIJC1ds59+BfzHzH5d9P0pZva0mU03sx7piEGkJlU9W0UaL+UzepRT\nSUS6Zj91BY4B/lY0pftJoDdQADxhZv3TFIdIlcaNGxd2CBIg5TN6lFNJRLpuP7UuHuFsZtOIF1Pn\nuHuumXUDrgEuTFMsIpUMHTo07BAkQMpn9Cinkoh09dS0MbOsoqdiDwc+L1qPBndfB9TreU8iIiIi\nSfXUmNkfiC+itxJY6e7fJHjoVGAx0BwYQLxnpiwtticiIiL1kmxPzWRgLvEC5QMzW2VmL5hZ35oO\ncvfVxKdr/x44xt1vAigaLHwV8F3SkYsEaMGCBWGHIAFSPqNHOZVE1OX20yXu3t7du7p7H2A0sLq2\ng9y9wN0XuvurFc8HvFKHOEQCM3/+/LBDkAApn9GjnEoiknpMgpmtc/duKYynUdBjEkREROomlY9J\nSLan5tMgLy4iIiISlGSLmu0piUJERESkntI1pVtEREQkpZItarqaWcuURCISopycnLBDkAApn9Gj\nnEoiki1qegL/NbNnzewqMzvSzJJelbhoVWGRBkOrlUaL8hk9yqkkItnZT4Vlvi0+8DtgOfBq0Wup\nu2+r5Twvu/sxyYXacGj2k4iISN2kcvZTsr0sG4B7gcHEF9NrA7Qi/rDKo4va7DCzPEqLnCXuXnHF\nYN3CEhERkUAlW9SsdvcpAGaWAQwEhhAvaI4Cdgd2BY4A/h8wASg0s3eJFzivALlAk1/rRkRERIKV\n7JiakttP7l7o7m+6++3u/hN33xM4GLgMeAz4L2BAM+CQou2PA58AHYMIXiQoS5YsCTsECZDyGT3K\nqSQi2aKmTU073f1dd5/h7qPdfS+gP3Ah8BDxhfus6CXSoEybprHrUaJ8Ro9yKolI9vZTl2Qau/tH\nwEfAfQBm1g04G7ghyeuKpNTDDz8cdggSIOUzepRTSUSyPTV7mdlhdb2Yu68rekL3J3U9h0gqZGZm\nhh2CBEj5jB7lVBJRlxWF7zGzGm9DJeC/9TxeREREpJxki5qrgRbAu2Y2xsxa1fG6eoaUiIiIBCqp\nosbdb3b3A4EY0BtYZmazzaxTktd9Kcn2Iik1YcKEsEOQACmf0aOcSiKSfsQBxGc5Ae8CV5vZIGBH\nksdfV5friqRK165dww5BAqR8Ro9yKolI6jEJEqfHJIiIiNRNKh+TUJeBwiIiIiINjooaERERiQQV\nNSLAypUrww5BAqR8Ro9yKolQUSMCTJw4MewQJEDKZ/Qop5IIFTUiwPTp08MOQQKkfEaPciqJUFEj\ngqaLRo3yGT3KqSRCRY2IiIhEgooaERERiQQVNSLA1KlTww5BAqR8Ro9yKolQUSMCFBQUhB2CBEj5\njB7lVBKhxyTUgR6TICIiUjd6TIKIiIhILVTUiIiISCSoqBEB8vPzww5BAqR8Ro9yKolQUSMCjBkz\nJuwQJEDKZ/Qop5IIFTUiwOTJk8MOQQKkfEaPciqJUFEjAprFFjHKZ/Qop5IIFTUiIiISCSpqRERE\nJBIiWdSY2Vgz+8TMtpnZMjM7rIa2PzWzRWb2XzP72sz+z8yGpjNeCd/s2bPDDkECpHxGj3IqiYhc\nUWNmo4HbgEnAj4B/Ac+bWVY1hwwBFgEnAQOBl4CnzOzgNIQrDUReXqCLWkrIlM/oUU4lEZF7TIKZ\nLQOWu/sVRd8b8Clwl7tPS/Ac7wEPu/sN1ezXYxJERETqIJWPSdglyJOFzcyaA9nAjcXb3N3NbDEw\nKMFzGNAW2JSSIEVERNLAHf77X/joo/irb18YPDjsqFIrUkUNkAU0AzZW2L4R6JfgOSYArYFHA4xL\nREQkJbZsgVWrSouXsq8tW+JtzOB3v1NR06SY2dnA/wAxd9ea3CIi0iB8/z2sXl114bKxzP/Gd+oU\n75E56CA4/fT41336QK9e0LJlePGnS9QGCucDO4FOFbZ3Aj6v6UAzOxP4E3C6u7+UyMVGjBhBLBYr\n9xo0aBALFiwo127RokXEYrFKx48dO7bSiP68vDxisVil55xMmjSJqVOnltu2fv16YrEYK1euLLf9\n7rvvZsKECeW2FRQUEIvFWLJkSbnt8+fPJycnp1Jso0ePblLvIxaLReJ9QDTyUd/3EYvFIvE+IBr5\nCOJ9xGKxSLwPqD4fZ5wxmlmzFrBoEUyfDpdfDoceuojWrWO0agX77w8//SnccAP8/e9j+f772Vx8\nMcybB6+/Di+/nMfhh8d48sl8Zs+GiRNh1Ch49NFJ3HlnOP+u5s+fX/K3sXPnzsRiMcaPH1/pmKA0\nlYHC64kPFL6lmmPOAu4DRrv70wlcQwOFI2bRokUMHaqZ/FGhfEZPVHLqHu9Zqep20ccfw/bt8Xa7\n7hrvXenXr7S3pV+/+H87dYrfTmqsNFA4ObcDc8wsF3gdGA9kAnMAzOwmYG93P6/o+7OL9l0OvGFm\nxb0829x9S3pDl7BE4ZellFI+o6ex5XTLlqpvFX30EWzdGm9jBt26xYuW446Diy4qLWK6doVmzcJ9\nD41R5Ioad3+0aE2a64nfdnobGObuXxQ16QzsW+aQXxEfXDyj6FVsLqDHwoqISJW++y4+zqWqXpeq\nxrkcfDCccUbTG+eSTpEragDcfSYws5p9ORW+PzYtQYmISKOzcyesX191j8u6dfHbSQBt25beIjr+\n+NLCpW9f2G23cN9DUxLJokYkWQsWLGDUqFFhhyEBUT6jJ5U5LR7n8tFHlXtdKo5z6d07XqiccUZp\nEdO3L3Ts2LjHuUSFihoR4iP09UcwOpTP6Akip19/XXXhUnacS0ZG+XEul1wS/7pvX9h3X41zaegi\nN/spHTT7SUSkYSoe51LV7aL//re0XefO5Xtayo5zadEivPibAs1+EhERKbJzZ3w8S3GxUrbnpew4\nl912Ky1YTjih/LRojXOJJhU1IiLS4LjD559XXbisXl06zqVFi/LjXMr2vHTooHEuTY2KGhERCc3m\nzdU/t+ibb+JtMjKge/d4D8sJJ8Cll5beLtJ6LlKWihoRICcnhwceeCDsMCQgymfDsm1b5XEuxYVM\nxXEu/frBwIFw5pmlt4t69oSLL1ZOpXYqakRofKuVSs2Uz/SrOM6l7Gv9+vLjXIrHtRx/fPzr4u/b\ntq3+/MqpJEKzn+pAs59EpCmqOM6lbI9LxXEuffqULj5XdnaR1nMRzX4SEZG02by5cm9LcfFSdpxL\nt27xXpYTT4SxY0tvF+27b3y/SLqpqBERaYKqGudS/Prii9J2e+0V72XJzo6PcynudenZU+u5SMOj\nokYEWLJkCUcddVTYYUhAlM+4H36oepzLqlXlx7m0a1c6rmXYsNLbRrWNc0kn5VQSoaJGBJg2bZp+\nYUZIU8pnVeNcyq7nsmNHvF3xei79+sFZZ5UWLv36QVZWwx/n0pRyKnWnokYEePjhh8MOQQIUxXx+\n9VXV67msWlV+nEuPHvFipWyPS/FzixrzOJco5lSCp6JGBMjMzAw7BAlQY83ntm3xp0JX1euSn1/a\nruw4l7PPLj/OZdddw4s/lRprTiW9VNSIiKRRdeNcPvoIPv20dJxL+/als4mGDy8d49KQxrmINDQq\nakREAuYO//lP1YXLmjXlx7kU3yI655zyD1xsDONcRBoaFTUiwIQJE7jlllvCDkMCkq58VjfO5aOP\n4Ntv420qjnMpfuBinz6Nf5xLOukzKolQUSMCdO3aNewQJEBB5rOgID7Oparipew4l733jhcqhx0W\n73UpLlyiPM4lnfQZlUToMQl1oMckiETLDz/A2rXVj3Mp1r595WX/i28XtWkTWvgijYoekyAiUk/u\n8O9/l1/yv+x6Lj/8EG/XqlV8PZficS7Ft4v69oU999Q4F5GGTEWNiETKV19V3eOyalXpOJdmzaB7\n93gPS/HMouIel3320TgXkcZKRY0IsHLlSvr37x92GJKg4nEuVfW6xMe5rAT6s/fe8YLl8MPh3HNL\nbxf16KFxLo2NPqOSCBU1IsDEiRNZuHBh2GFIGTt2lB/nUrZ4qTjOpfgW0fDh8a9nzJjIM88s1DiX\nCNFnVBKhokYEmD59etghNEllx7kUFy4ffli6nkvFcS59+sR7XMoO1K1qPZcjjpiugiZi9BmVRKio\nEUHTRVNt06bK41uqGudSvJ7LiBHlZxd16ZLcOBflM3qUU0mEihoRCUTZcS4VX19+WdquS5fScS5l\nZxf17AnNm4cXv4g0fipqRCRhFce5lO11KTvOZffdS2cTFfe69O0bv4Wk20IikioqakSAqVOn8rvf\n/S7sMBLm7liKFkxxhw0bqi5cqhrn0q8f/Pzn5W8X7blnSkJLWGPLp9ROOZVEqKgRAQoKCsIOoVZb\nt27l1muu4bWnnqL1jh1827w5Pz7lFH47ZQpt6/DY5orjXMoWL8U/jmbN4reFise5FBctffsmP84l\nnRpDPiU5yqkkQo9JqAM9JkHSbevWrZw6aBBXrljBsMJCDHDg+YwMbh8wgCeWLq2ysPn22/LjXMpO\ni644zqX4dlHZxwD06KFxLiISLD0mQaSJu/Waa7hyxQqGFxaWbDNgeGEhOz5YxdXj7mT4GddW6nn5\n7LPSc5Qd53LyyfH/Fr80zkVEokBFjUgjsGTh0/yycC/+QV8+qvD6xHvww4PNmf5gfJxLcS/LeeeV\n73kJe5yLiEiqqagRAfLz88nKygo7jJJxLsUL0MVfzvvr36crrQDYhR30ZA19+YiRPE0/PmRu1lc8\nkvcIXbpYgx3nkk4NJZ8SHOVUEqGiRgQYM2ZM2pZgrzjOpexr06bSdvvsE+9pOfJIY9unt3LbV8vp\nyyp6sobm/FDSzoHH2nRn3331+Ohi6cynpIdyKolQUSMCTJ48OdDz7dgBn3xSdeGyYUNpuz32iI9z\n6dcPTjml9HZR797QunVpu0nNv6D5jOfoV2ZMTbH/zcjgqFgs0Pgbu6DzKeFTTiURKmpEoE6z2AoL\na17PZefOeLvMzNJiZdCg0inRffokPs7lt1OmcOo//oEXDRYunv30vxkZ3DFgAE/ccEPS8UeZZiVG\nj3IqiVBRI1KLL7+susdl1SrYti3eZpdd4uu59OkDI0dWXs+lvuvktW3blieWLuW2a6/l9oULydyx\ng4LmzflxLMYTN9xQp3VqRESiRkWNCPFxLmXXcCn7+uqr0nb77hsvVI48Es4/v3RmUffuqV/PpW3b\ntky+8064886UrigsItJYqaiRJmP79vg4l6qKlw0bZgMXAPFbQn37Qv/+8XEuxT0uvXvHbyU1BCpo\najZ79mwuuOCCsMOQACmnkggVNRIphYXxBecqrp770UfxgqbsOJfiXpYf/xiWLs1jypQL6NMnPnhX\nGre8vDz9AYwY5VQSocck1IEekxAu99JxLqtWlV/TZdUq+O67eLvicS5ll/0vHuuy9971H+ciIiLJ\n02MSpEn65pvS3paKvS5VjXM56ijIySm9XdS9e7ywERGRpkG/8iVUxeNcqhqg++9/l7bLyirtZYnF\n4kVLv37Qq1fDGeciIiLhUlEjKVd2nEvFXpey41xaty5dz+Woo0pvF/Xtq3EuIiJSOxU1Egh3yM+v\neiG6iuNcevWKFyo/+Un59Vz22iu8cS6xWExLsEeI8hk9yqkkQkWNJKXsOJeKr82bS9t17Vra4zJm\nTPn1XBriOJdx48aFHYIESPmMHuVUEtEA/7xI2LZvjy/zX1XxUnGcS9++sN9+MGpU6e2ihrSeS6KG\nDh0adggSIOUzepRTSYSKmiaq4jiXiuu5FD83sXXr0mJl8ODSMS5az0VERBoaFTURVtU4l+IxLmXH\nuTRvHh/n0qdPvMeloYxzERERSYaKmgjYurXqW0WrVlUe51Lc43LBBaU9Lg11nEs6LViwgFGjRoUd\nhgRE+Ywe5VQSEckVhc1sLPBboDPwL+Ayd3+jhvbHALcB+wPrgSnuPreG9g1mReGBA+Gtt0q/z8oq\n7WXp06e010XrudRs0KBBLF26NOwwJCDKZ/Qop9GhFYWTYGajiRcoFwKvA+OB582sr7vnV9G+O/A0\nMBM4GzgBuM/M/u3uL6Qr7roaOxZatCgtYnbfPeyIGqcOHTqEHYIESPmMHuVUEhG5ooZ4ETPL3R8E\nMLOLgZOBMcC0KtpfAqxx94lF339oZkcVnafBFzV6vpuIiEhcRtgBBMnMmgPZwIvF2zx+f20xMKia\nw44o2l/W8zW0b3Dmz5/fYM6XzLGJtK2tTU37q9sX9M8raKmIr67nDDqftbVTPlN7zmSPS+VntLHm\nE/Q7N9l96cxppIoaIAtoBmyssH0j8fE1VelcTfvdzKxFsOGlhj5gye1r6L80G+sfQRU1VWus+Uy0\nvYqacM+n37nlRfH2Uzq0BFixYkXYcQDw9ddfk5cX3Fir+pwvmWMTaVtbm5r2V7evqu2vv/56oD/D\n+gg6n/U5Z9D5rK2d8pnacyZ7XCo/o8luj3JOm9rv3DJ/O1vWGnSSIjX7qej2UwFwqrsvLLN9DtDO\n3X9axTGvALnufmWZbecDd7h7lcNuzexsYF6w0YuIiDQp57j7X4M8YaR6atx9h5nlAscDCwHMzIq+\nv6uaw5YCJ1XYNrRoe3WeB84B1gLf1SNkERGRpqYl0J3439JARaqnBsDMzgDmABdTOqX7NKC/u39h\nZjcBe7v7eUXtuwPvEp/SfT/xAuiPwAh3rziAWERERBqoSPXUALj7o2aWBVwPdALeBoa5+xdFTToD\n+5Zpv9bMTgbuAC4HPgMuUEEjIiLSuESup0ZERESapqhN6RYREZEmSkWNiIiIRIKKmhQws5FmttLM\nPo2UGJcAAAmISURBVDQzPcigkTOzJ81sk5k9GnYsUn9mto+ZvWRm75vZ22Z2WtgxSd2ZWTsze8PM\n8szsHTP7ZdgxSf2ZWSszW2tmVT3eqPrjNKYmWGbWDPgAOBr4BsgD/p+7fxVqYFJnZjYEaAuc5+5n\nhB2P1I+ZdQY6uvs7ZtYJyAX6uPu2kEOTOihatqOFu39nZq2A94k//Vm/cxsxM7sB6AV8WubZjLVS\nT03wDgfec/fP3f0b4Bni695II+XurxIvUCUCij6b7xR9vRHIB/YINyqpK48rXi+sVdF/Lax4pP7M\nrDfQD3gu2WNV1ARvb2BDme83AF1CikVEamBm2UCGu2+otbE0WEW3oN4G1gO3uPumsGOSerkV+D11\nKE5V1JRhZoPNbKGZbTCzQjOLVdFmrJl9YmbbzGyZmR0WRqxSO+UzeoLMqZntAcwFfpXquKVqQeXT\n3b9290OAHsA5ZtYhHfFLeUHks+iYD9394+JNycSgoqa81sQX67sUqDTYyMxGA7cBk4AfAf8Cni9a\n7K/Yv4F9ynzfpWibpF8Q+ZSGJZCcmtmuwN+AG919eaqDlmoF+hktWmT1X8DgVAUsNQoin0cAZ5rZ\nGuI9Nr80s2sTjsDd9ariBRQCsQrblgF3lvneiK9APLHMtmbAh8BeQBtgBbB72O+nqb/qms8y+44B\nHgv7fegVTE6B+cAfwn4PetU/n0BHoE3R1+2IP/Zm/7DfT1N/1fd3btH+84BpyVxXPTUJKnoCeDbw\nYvE2j//UFwODymzbCfwGeJn4zKdbXaPwG5xE81nU9gXgEeAkM1tvZv8vnbFKYhLNqZn9GDgdGGVm\nbxVNBd4/3fFKzZL4jHYD/mlmbwGvEP+j+X46Y5XaJfM7tz4i9+ynFMoi3guzscL2jcRHaZdw96eB\np9MUl9RNMvk8MV1BSb0klFN3fw397msMEs3nG8RvZUjDlvDv3GLuPjfZi6inRkRERCJBRU3i8vn/\n7d1drBxlHcfx78+0pS+oREtRFItWVC4EEaux0JYI+JbUd+WCRCVGI/UlEiJeeNErEuIFoqKgEmli\nIgSM0ahFSkJRoIVKkdgEk9qkDaRJNSLaUgtt6N+LmQNL2d1z5LB7esbvJ5ns2Z1nZp49T7L55XmZ\ngadpnvzd6yRg7/iro2myPbvHNu0W27NbxtKehpopqqrDNHcePX/is/ZOlucDm2eqXnphbM/usU27\nxfbslnG1p+PKPZIsAt7Is+vi35DkTOCfVfUocDWwPsk2YCtwGbAQWD8D1dUkbM/usU27xfbslmOi\nPWd62dextNE8r+kITRdZ7/aTnjJrgd3AQWAL8I6Zrreb7fn/stmm3dpsz25tx0J7+kBLSZLUCc6p\nkSRJnWCokSRJnWCokSRJnWCokSRJnWCokSRJnWCokSRJnWCokSRJnWCokSRJnWCokSRJnWCokSRJ\nnWCokSRJnWCokSRJnWCokTQWST6U5J4kDyV5KsmRJPuSnDCg/ElJ7k3yRFv2SJL9SbYluXDc9Zd0\n7PMp3ZLGKkmAx4CXtx99vaquHlL+FcAO4DfApVV1cPS1lDQbGWokjVWSM4FNwCFgCbCzqt40yTE7\ngbOqav8YqihplnL4SdK4rQTuAn7Yvl+W5L2DCic5FdhjoJE0GUONpHFbBdwJXAccbj/70pDyq4Hf\nj7pSkmY/Q42kcVsJ3FlVe4FbgAAfTHLKgPKrMNRImgJDjaSxSXIazVy+h9uPvtO+vgT44oDDVgCb\nR103SbOfoUbSOK2imU8DQFU9AGyh6a35XJI5vYWTnAw87oonSVNhqJE0TqtoVj71uqZ9PRH4ZJ/y\n0x56SjJ3uueQdOwz1Egap4lJwr1+ATza/r32qH2rgT9MdtIky5P8oL1Z38YktyRZl2R+kqXAjUeV\nn5dkQ5Kd7U399rTvNyTZlGRHkluTnJ7ko0nuTHKgLftwe41X95zv5iT/bvfvSnLV//yfkTRt3qdG\n0lgkeS2wpaqeNyE4yRXAVUABb6uq7e3nfwbOGbScO8krge8DbwWuqKrf9uw7G/gacBZwU1Vd2ef4\ni4CfAR+vql/2fD4XuAl4T1ufR5LcB7yqqk4dUJePAB+rqk9P+s+QNBL21Egal35DTxN+DEzMm1kL\nkGQx8OSQQPNm4H5gEXB2b6ABqKptwFbg9CHXPbd9veeoYw/T9O6cQDPXZyHw9iHnAXh9+z0kzRBD\njaRxGRhqqupx4Kc0E4YvTnJ8W/7ufuWTnAj8DtgHfKqqnhxwzQ3AAZrw08+5NHc0/keffce3ry+l\nWYE1h+Hze9415DqSxsBQI2lcVjK8p2Niefci4LMMnyT8I+AU4AuTrIzaB9xbVU8fvSPJy2iGrQYt\nFz+PZjhsc1uXGlSf9nlW86rq0JC6SBoxQ42kkWvnvsyvqt2DylTVX4A72reX0vSiPK+nJsk5wIeB\n29sl4cM8AawbsG8FzW/gvX2usRi4CNhUVT+nCTh7qmrXgHOdAWyfpC6SRsxQI2kcnnN/miGuoRmC\negswpx2WOtolNL0mN/bZ9xxVdbCqtg7YvZJne2Ke0d4g8FfARmBNkuOA5QwfeprSKi1JozVn8iKS\nNG0XMHzoCYCqui3JDuA0BoeE1e3rXdOs00qaZ099pRk9ooB5NHNwLq+q+wCSrAKOY3ioWYGThKUZ\nZ6iRNFJJ3gd8Bnhoiod8r90GhYjXAPv6Te5NciHwDWAJsAA4AuwC1rQrmibKzaPpfdlcVZdOUp+h\n82laC7zrsTTzHH6SNBJJfp1kN80KpAXAdUm2J/nyJIeuB/YyuKdmH/Cffjuq6o6qugC4HFgG3FxV\n7+8NNK3lNL0vUxkyOgN4rKr+2m9nkncDD07hPJJGzJ4aSSNRVWte4HEHgJOHFLkf+ECSRW3ZflbQ\n9K7cNmD/xHyaqTyCYSHQN9C0Lms3STPMnhpJs823aX67Pt9vZ7u8+hM0c2P+OOAcE/Nptkzhen8C\nFg+41leBjVW1ZwrnkTRihhpJs0pV3QV8E7gyySVJnvkda5diXw/spJkv0+/+NHNpenIeHHLTvl7X\nAouSXNxzjvlJ1gFLquqGaX0hSS8an/0kaVZKch7NsM9S4G/Av4C/A98FHgHeWVV395RfCtzQll8G\n7AceAK7tfe7TgGu9DvgWTY/NU8Ah4Pqquv3F/VaSpsNQI0mSOsHhJ0mS1AmGGkmS1AmGGkmS1AmG\nGkmS1AmGGkmS1AmGGkmS1AmGGkmS1AmGGkmS1AmGGkmS1AmGGkmS1AmGGkmS1AmGGkmS1An/BafN\nC8QpqtORAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAGNCAYAAADpZIAdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3XucznX6x/HXNRIGkYhSGFlSUUv1SxtRFkmTtjadi7bD\n5hTFdrBLLYVKSw4plDbRltZSWlKpFMpMbaqhIofaVkYHasIw1++P75iMGdwzc9/3d+ae9/PxmEfN\n5/4ernsut7l8Pydzd0RERETKuqSwAxARERGJBhU1IiIikhBU1IiIiEhCUFEjIiIiCUFFjYiIiCQE\nFTUiIiKSEFTUiIiISEJQUSMiIiIJQUWNiIiIJAQVNSIiIpIQEq6oMbO2ZjbXzL4ysxwzS43gnPZm\nlmZm283sUzO7Nh6xioiISPQkXFEDVAU+AG4BDrqxlZk1Al4EXgVOBsYCU8zst7ELUURERKLNEnlD\nSzPLAbq7+9wDHDMKOM/dW+7VNhOo4e5d4xCmiIiIREEiPqkpqjOARfu0LQDahBCLiIiIFJOKGqgH\nbNqnbRNwmJlVCiEeERERKYZDwg6gLDKzI4DOwDpge7jRiIiIlCmVgUbAAnffEs0Lq6iB/wF192mr\nC2x19x37OaczMCOmUYmIiCS2K4FnonlBFTWwFDhvn7ZOue37sw7g6aefpnnz5jEKK3IDBgzg4Ycf\nLhXXK8q5kRx7sGMO9Pr+XiusPdo/w5KIRSzFvWa083mw45TP2F6zqOfF8jMarfYw6O/ckn1GMzIy\nuOqqqyD3d2k0JVxRY2ZVgSaA5TY1NrOTgW/dfaOZ3Q8c7e571qJ5FOidOwtqGnAucAlwoJlP2wGa\nN29Oq1atYvE2iqRGjRpRjaMk1yvKuZEce7BjDvT6/l4rrD3aP8OSiEUsxb1mtPN5sOOUz9hes6jn\nxfIzGq32MOjv3JJ/RnNFffhGIg4UPhV4H0gjWKfmISAduCf39XrAsXsOdvd1wPlAR4L1bQYA17v7\nvjOiSq3LL7+81FyvKOdGcuzBjjnQ6/t7rbD2//3vfweNJV6inc+SXDPa+TzYccpnbK9Z1PNi+Rkt\nansi57Q8/p0bKwm9Tk2smFkrIC0tLa3U/MtBSqZ+/fp89dVXYYchUaJ8Jh7lNHGkp6fTunVrgNbu\nnh7NayfikxqRIsv9gEmCUD4Tj3IqkVBRI0J8H49K7CmfiUc5lUgk3EBhkeLQX5iJ5UD53LBhA5mZ\nmXGMRqKhWbNmpKdHtadCYqh27do0aNAg7vdVUSMi5caGDRto3rw5WVlZYYciktCSk5PJyMiIe2Gj\nokYE6NmzJ0888UTYYUiU7C+fmZmZZGVllZo1pkQS0Z51aDIzM1XUiIShU6dOYYcgUXSwfJaWNaZE\nJLo0UFgEjalJNMqnSPmkokZEREQSgooaERERSQgqakSAJUuWhB2CRJHyKVI+qagRAUaPHh12CBJF\nymd+w4YNIylJf91L4tOfchFg1qxZYYcgUaR85mdmmFmRz5s0aRLTp0+PQUQisaGiRoRgoShJHMpn\ndEycOFFFjZQpKmpERA7A3cvktUXKIxU1IiL72LZtG0P79aNjSgrdjz2WjikpDO3Xj23btpXqa0Mw\nSPq0006jSpUq/OpXv+Kxxx4rcMwTTzzBueeeS926dalcuTInnngijz76aL5jUlJS+Pjjj1m8eDFJ\nSUkkJSVxzjnnAPDdd99x++2307JlS6pXr06NGjXo2rUrH374YVTeg0hxaUVhEWDQoEE88MADYYch\nUVKSfG7bto2L27RhYEYGw3JyMMCBBRMmcPFrrzF76VKqV69e6q4N8NFHH9G5c2eOPPJI7r33XrKz\nsxk2bBhHHnlkvuMeffRRTjrpJC688EIOOeQQ5s2bxy233IK788c//hGAsWPH0qdPH6pXr86QIUNw\nd+rWrQvA2rVrmTt3Lr///e9JSUlh06ZNTJ48mfbt2/PJJ59Qr169Yr8HkRJxd30V8QtoBXhaWppL\nYhg3blzYIUgU7S+faWlpfrDP7l/69vWXk5LcocDX/KQkH9qvX7HjiuW13d27d+/uycnJ/uWXX+a1\nrVq1yg855BBPSkrKa9u+fXuBc7t06eJNmjTJ13bSSSd5hw4dChy7c+fOAm3r16/3ypUr+/Dhw0vy\nFiQBHOxztud1oJVH+fezup9EgL59+4YdgkRRSfL59rx5dM7JKfS1Ljk5vD13bqm8dk5ODgsXLuSi\niy6ifv36ee3NmjWjc+fO+Y6tVKlS3v9v3bqVLVu20K5dO9auXRtRN1jFihXz3ffbb78lOTmZZs2a\nkZ6eXuz3IFJSKmpERHK5O1Wzs9nf5GcDkrOzizXAN5bXBti8eTM///wzTZo0KfBas2bN8n3/9ttv\n07FjR6pVq0bNmjWpU6cOd999NwA//PDDQe/l7jz88MM0bdqUSpUqUbt2bY488khWrlwZ0fkisaKi\nRkQkl5nxU8WK7K+scOCnihWLteZLLK9dFGvWrKFjx458++23PPzww8yfP59FixYxYMAAIHjycjAj\nRozgtttuo3379syYMYOFCxeyaNEiTjjhhIjOF4kVDRQWAVatWsXxxx8fdhgSJSXJ528uuIAFEybQ\npZBfzv9OSuKs1NRixxXLa9epU4cqVarw2WefFXht1apVef8/b948du7cybx58/J1U7366qsFzttf\ngTV79mzOOeecAjOrvv/+e+rUqVPctyBSYnpSIwIMHjw47BAkikqSz9tHjGBM8+a8nJSU91TFgZeT\nkni4eXNuGz68VF47KSmJzp07M2fOHL788su89oyMDBYuXJj3/SGHBP+W3fuJyg8//MCTTz5Z4JpV\nq1bl+++/L9BeoUKFAt1kzz33HF999VWx4xeJBhU1IsD48ePDDkGiqCT5rF69OrOXLmV5nz50atSI\nC+vXp1OjRizv06fEU65jeW2Ae+65B3fnrLPOYvTo0YwYMYJzzjmHk046Ke+YTp06UbFiRbp168bE\niRMZNWoUp556at507b21bt2aDz/8kBEjRvDss8/y+uuvA9CtWzcWL15Mr169mDJlCv379+ePf/wj\nxx13XIniFymxaE+nKg9faEq3SJkUyZTufeXk5MQsnlhc+6233vLTTjvNK1eu7E2aNPHHHnvMhw0b\nlm9K94svvuinnHKKJycne+PGjf3BBx/0J554wpOSknz9+vV5x23atMkvuOACr1GjhiclJeVN796x\nY4cPGjTI69ev71WrVvV27dr58uXLvUOHDn7OOedE/T1J2RLmlG7zYo60L8/MrBWQlpaWRqtWrcIO\nR0QilJ6eTuvWrdFnVyR2DvY52/M60Nrdo7oGgLqfREREJCGoqBEBRo0aFXYIEkXKp0j5pKJGBMjK\nygo7BIki5VOkfFJRI0Iwa0QSh/IpUj6pqBEREZGEoKJGREREEoKKGhEgMzMz7BAkipRPkfJJRY0I\n0KtXr7BDkChSPkXKJxU1IsCwYcPCDkGiSPkUKZ9U1IiAVpdNMMqnSPmkokZEREQSgooaEZEE8OST\nT5KUlMSGDRvCDuWA1q9fT1JSEk899VTEx44ZMybmcb3xxhskJSXx5ptvxvxe0VZY7Ndddx0pKSkh\nRhUOFTUiwNSpU8MOQaKoPObTzDCzsMOIyL5xvvzyy6ViwcSy8vMrzL6xmxlJSeXvV3z5e8cihUhP\nj+pGsRIy5bP0atiwIT///DNXX311Xtv8+fO59957Q4wKzj77bH7++WfatWsXahzRMmXKFFatWhV2\nGHF3SNgBiJQGEyZMCDsEiSLls/TZvXs3OTk5VKxYkUMPPTTfa+4eUlT57RtXWVahQgUqVKgQdhhx\npyc1IiIJbOLEiZx00klUrlyZ+vXr06dPH3744YcCx02YMIHjjjuO5ORkzjjjDJYsWUL79u0555xz\n8o7Jzs7mL3/5C6eeeio1a9akWrVqtGvXjsWLF+e71t5jYcaOHUuTJk2oXLkyGRkZBcbU9OzZk4kT\nJwKQlJREUlJSob+MH3/88bzrnH766axYsSLf69dddx3Vq1dn48aNdOvWjerVq3PMMcfkXXvlypWc\ne+65VKtWjUaNGjFz5sx85+9vTM3y5cvp2rUrtWrVolq1apx88smMGzfugD/zXbt2cc8999C0aVOq\nVKlC7dq1adu2La+++mq+41avXs2ll17KkUceSXJyMscffzxDhgzJe33Dhg3ccsstHH/88SQnJ1O7\ndm0uvfRS1q9ff8D77/l57D2mZu+cHOxnCfDcc89x4oknUqVKFVq2bMmcOXPKxDgdPakREUlQw4YN\n495776VTp07ccsstrF69mokTJ7JixQrefvvtvOJh0qRJ9O3bl7PPPpuBAweybt06unfvzuGHH86x\nxx6bd72tW7cybdo0Lr/8cm688Ua2bdvG1KlT6dKlC++++y4tW7bMd/9p06axY8cObrrpJipVqkSt\nWrXYvXt3vmNuvvlm/vvf/7Jo0SJmzJhR6FObGTNm8OOPP3LzzTdjZowaNYqLL76YtWvX5r0HMyMn\nJ4fzzjuPs88+mwceeIAZM2bQt29fqlatyt13381VV13FxRdfzKOPPsq1117LmWeeScOGDfPus++4\nlFdeeYULLriAo48+mltvvZV69eqRkZHBSy+9RL9+/fb7cx86dCgjR47kxhtv5LTTTmPr1q2sWLGC\n9PR0zj33XAA+/PBD2rZtS6VKlbjpppto2LAha9as4cUXX2T48OEAvPfeeyxbtozLL7+cY445hnXr\n1jFx4kQ6dOjAJ598QuXKlfcbw/7GWEXys3zppZe47LLLOPnkkxk5ciTfffcd119/PfXr1y/9447c\nXV9F/AJaAZ6WluYiUnakpaV5on52n3zySU9KSvL169e7u/vmzZu9UqVKft555+U7bsKECZ6UlORP\nPvmku7vv3LnTa9eu7WeccYbv3r0777innnrKzcw7dOiQ15aTk+PZ2dn5rvfDDz94vXr1/A9/+ENe\n27p169zMvGbNmr5ly5Z8x+95bfr06Xltffr08aSkpALvac+xderU8R9++CGvfe7cuZ6UlOQvvfRS\nXtt1113nSUlJPmrUqLy277//3pOTk71ChQr+3HPP5bWvXr3azczvueeevLbFixd7UlKSv/HGG+7u\nvnv3bk9JSfHGjRv71q1bC8R2IKeccopfcMEFBzymXbt2XqNGDf/yyy/3e8z27dsLtC1fvtzNzJ9+\n+un9xu4e/DxSUlLyvi/Kz7JFixbeoEEDz8rKymt788033czyXXN/DvY52/M60Mqj/PtZT2pEgNTU\nVObOnRt2GBIl0cpnVhbEeqzl8cdDcnL0r7to0SKys7O59dZb87XfcMMN3HXXXbz00ktce+21vPfe\ne2zZsoVRo0blmy1zxRVXFDjXzDjkkODXhrvz/fffs3v3bk499dRCB2dfcskl1KpVq8Tv5bLLLuOw\nww7L+75t27a4O2vXri1w7PXXX5/3/zVq1KBZs2asWbOGSy65JK+9adOm1KxZs9Dz93j//fdZt24d\nY8eOpXr16kWKt2bNmnz88cd8/vnnNGnSpMDrmZmZvPXWWwwYMID69evv9zqVKlXK+/9du3axdetW\nGjduTM2aNUlPT+fKK68sUlxw8J/l119/zUcffcSQIUOoUqVKvuNatGjBtm3binzPeFJRIwL06dMn\n7BAkiqKVz1WroHXrqFxqv9LSIBYLIO8Zd9G0adN87RUrVqRx48Z5r2/YsAEz47jjjst3XIUKFWjU\nqFGB606fPp0xY8awatUqsrOz89obN25c4NjCzi+OvbvAICgaAL777rt87ZUrV+aII47I11ajRg2O\nOeaYAtesUaNGgfP3tmbNGsyME088scjx3nvvvXTv3p2mTZty0kkn0aVLF66++mpatGgBkFdAHOza\n27dv57777uPJJ5/kq6++yuuaM7NCx0VF4mA/yz1/Lvb98wDQpEkT3n///WLdN15U1IgAnTp1CjsE\niaJo5fP444OiI5aOPz6214+mp59+mp49e/K73/2OwYMHc+SRR1KhQgXuu+++Qp967P0v/ZLY3yye\nPb/kD3ZcpOdHS9u2bVmzZg3/+te/WLhwIVOnTuXhhx9m8uTJRdpstU+fPkyfPp0BAwZwxhlnUKNG\nDcyMHj16kJOTU6zY4v2ziDcVNSIi+5GcHJunKPGwZwDs6tWr8z0xyc7O5osvvuC3v/1t3nHuzuef\nf87ZZ5+dd9zu3btZt24dJ598cl7b7NmzOe6443j++efz3esvf/lLiWItjYNPjzvuONydjz76KN8M\nsEjVrFmTa6+9lmuvvZasrCzatm3LsGHD6NWrV95TrY8++uiA15g9ezbXXXcdo0ePzmvbsWMH33//\nfZHjidSePzeff/55gdcKayttNKVbRCQBdezYkYoVKxaYfjxlyhS2bt1Kt27dADj11FM54ogjePzx\nx/P96//pp58u0D1T2L/yly9fztKlS0sUa9WqVYFgdlVp0apVK1JSUvjb3/5W5K6eb7/9Nt/3ycnJ\nNGnShB07dgBQu3Zt2rVrx7Rp09i4ceN+r1OhQoUCT2TGjRtXYAZZNB111FGcdNJJPPXUU2RlZeW1\nv/HGG6xcuTJm940WPakRAebMmUP37t3DDkOiRPkMfnHeeeed3HvvvXTp0oXU1FRWrVrFpEmTOP30\n0/MGmVasWJFhw4bRr18/OnTowKWXXsq6det44oknaNKkSb6nKN26deOFF16ge/funH/++axdu5bJ\nkydz4okn8uOPPxY71tatW+Pu9O3bl86dO1OhQgV69OhR4p9BUe3dBWNmTJo0idTUVE455RR69uzJ\nUUcdxapVq/jkk094+eWX93udE044gfbt29O6dWtq1arFe++9x/PPP59vGvi4ceNo27YtrVq14sYb\nbyQlJYUvvviC+fPn541b6datG3//+9857LDDOOGEE1i6dCmvvvoqtWvXPmDsJXXffffRvXt3zjzz\nTHr27Mm3337LhAkTaNGiRYnyHA96UiMCBRbikrJN+QwMHTqU8ePHs3HjRgYOHMjzzz/PzTffzIIF\nC/I9denduzfjxo1j48aNDBo0iDfffJO5c+dSo0aNfGuhXHfdddx///18+OGH9O/fn1deeYUZM2bQ\nunXrQvce2l+30r7tv/vd7+jXrx8LFizgmmuu4YorrjjodQprj/R+kZ7fqVMnXn/9dZo1a8aYMWO4\n7bbbeO2110hNTS30Pnv079+f9evXM3LkSPr3789bb73Ffffdx4MPPph3TMuWLVm2bBlnn302jz76\nKP379+ef//xnvmJ83LhxXHPNNTzzzDPcfvvtbNq0iUWLFlGtWrWI3nukOdm3vVu3bsycOZPs7Gzu\nuOMOXnjhBaZNm0bTpk0PuDZOaWCJMjgonsysFZCWlpZGq7La4S5SDqWnp9O6dWv02T04d6dOnTpc\nfPHFTJ48OexwpBT49a9/zZFHHsmCBQsOeNzBPmd7Xgdau3tUN2pLyCc1ZtbbzL4ws5/NbJmZnXaQ\n4680sw/M7Ccz+6+ZTTWzki+uICJSBuwZ67G36dOn8+2339KhQ4cQIpIw7dq1q8C4ncWLF/Of//yn\n1P95SLgxNWbWA3gIuBF4FxgALDCzpu6eWcjxvwGmA/2BF4H6wGTgMeCSfY8XEUk0y5YtY8CAAfz+\n97/niCOOIC0tjWnTptGyZct8i9ZJ+fDVV1/RsWNHrrrqKo4++mgyMjKYPHkyRx99NDfddFPY4R1Q\nwhU1BEXMZHd/CsDMbgbOB3oBows5/gzgC3ffs63vejObDAyOR7AiImFr1KgRDRo04JFHHuHbb7+l\nVq1aeeNn9qwgLOXH4YcfzqmnnsrUqVPZvHkzVatW5YILLuD+++/n8MMPDzu8A0qoP61mVhFoDdy3\np83d3cwWAW32c9pSYISZnefuL5tZXeD3wEsxD1hKjZ49e/LEE0+EHYZEifJZNA0bNmTOnDlhhyGl\nxGGHHVZmB9sn2pia2kAFYNM+7ZuAeoWd4O7vAFcBz5rZTuBr4DtA6+aXI1pROLEonyLlU6IVNUVm\nZicAY4FhBLtvdwZSCMbVSDlx+eWXhx2CRJHyKVI+JVpRkwnsBuru014X+N9+zrkDeNvdx7j7R+7+\nCnAL0Cu3K2q/unbtSmpqar6vNm3aFHiMu3DhwkLXNejduzdTp07N15aenk5qaiqZmfnHNA8dOpRR\no0bla9uwYUPeglp7e+SRRxg0aFC+tqysLFJTU1myZEm+9pkzZ9KzZ88CsfXo0UPvQ+8jId+HiMTP\nzJkz83431qtXj9TUVAYMGBCz+yXcOjVmtgxY7u79c783YAMwzt0fKOT454Gd7n7FXm1tgCVAfXcv\nUAxpnRqRsknr1IjEntapia4xwA1mdo2ZHQ88CiQDTwKY2f1mNn2v4+cBF5vZzWaWkjvFeyxBYbS/\npzuSYPZ90iBlm/IpUj4l1OwnAHf/h5nVBu4l6Hb6AOjs7ptzD6kHHLvX8dPNrBrQG3gQ+B54laBb\nSsqJ0aNHc9ZZZ4UdhkTJwfKZkZERx2hEypcwP18J1/0UD+p+SjxZWVkkJyeHHYZEyf7yuWHDBpo3\nb55v92ERib7k5GQyMjJo0KBBgddi2f2UcE9qRIpDBU1i2V8+GzRoQEZGRoGBxiISXbVr1y60oIk1\nFTUiUq40aNAglL9sRST2EnGgsIiIiJRDKmpEoMD6K1K2KZ+JRzmVSKioEQF1RyQY5TPxKKcSCc1+\nKgbNfhIRESkeLb4nIiIichAqakRERCQhqKgRAW16mGCUz8SjnEokVNSIAIMHDw47BIki5TPxKKcS\nCRU1IsD48ePDDkGiSPlMPMqpREJFjQiaLppolM/Eo5xKJFTUiIiISEJQUSMiIiIJQUWNCDBq1Kiw\nQ5AoUj4Tj3IqkVBRIwJkZWWFHYJEkfKZeJRTiYS2SSgGbZMgIiJSPNomQUREROQgVNSIiIhIQlBR\nIwJkZmaGHYJEkfKZeJRTiYSKGhGgV69eYYcgUaR8Jh7lVCKhokYEGDZsWNghSBQpn4lHOZVIqKgR\nAc1iSzDKZ+JRTiUSKmpEREQkIYRW1JhZIzO7zsyqhhWDiIiIJI64FTVmVsXMau/53t3XAe8CQ8ys\nTbziECnM1KlTww5Bokj5TDzKqUQiLkWNmf0B+A7YZGb/M7PpZtYN+NTd7wSujEccIvuTnh7VRS0l\nZMpn4lFOJRJx2SbBzF4DJgE7gVbAeUBrgkLndaCOu7ePeSBRom0SREREiieW2yQcEs2LHcAKd38u\n9///BQw1s2OBS4BGwD1xikNEREQSVFSLmtyxMb8G3gY+9F8eA+WYWUV3z95zrLtvBB6O5v1FRESk\n/Ir2k5rjgPGAAz+a2XLgHWAVMMHMbnV37R8vIiIiURftgcKbgaeANsC9wE9Ab2Aa8AfgEzMbYWad\nNJVbSpPU1NSwQ5AoUj4Tj3IqkYj2k5oPgGx3f5dguvZDAGbWHDgLaAtcAdwJ7DKzD4A3gefdfVmU\nYxGJWJ8+fcIOQaJI+Uw8ymnJfPcd7NoFdeqEHUlsRfVJjbtvcvfXCmnPcPfH3f0ad08BGgDXASuA\nzsCMaMYhUlSdOnUKOwSJIuUz8SinxbN5M9x1FzRsCH/9a9jRxF68Zj9hZi2Bz909y92/BJ4xs3R3\nvyVeMYiIiJQHX38NDz4Ijz4KZtC7NwwcGHZUsRevxffuIOiOmr/PS3XNbGQ8YhAREUl0GzdCnz6Q\nkgJTp8Jtt8H69TBqFNStG3Z0sRevbRIOJ+humrl3o7u/AaSb2flxikOkUHPmzAk7BIki5TPxKKcH\ntnYt3HADHHcczJwJQ4bAunVw771wxBFhRxc/8SpqKrr7LHefvO8L7v4P4LdxikOkUDNnzjz4QVJm\nKJ+JRzkt3OrVcO210LQpzJ0LI0YExcyQIVCzZtjRxV+8ipojzOy4A7y+M05xiBTq2WefDTsEiSLl\nM/Eop/mtXAmXXQbNm8OiRfDQQ/DFFzBoEFSvHnZ04YlXUTMWWHSAbqZqcYpDRESkzEpLg4sugpYt\nYdkymDgx6Hrq3x+Sk8OOLnxxmf3k7ulmNgR4wcw+BeYQrGmzlWD9GhU1IiIi+7F0KQwfDvPnQ5Mm\nwSDgq6+GihXDjqx0iduUbnefYWYZwCjgLsByX3oFuCxecYiIiJQVb7wRrC/z6qtBV9PTT0OPHnBI\n3H57ly3x6n4Cgic27v5boC5wBtDY3Tu7+3fxjENkXz179gw7BIki5TPxlKecusMrr0C7dtC+PWRm\nwnPPwUcfwZVXqqA5kHitU9N27+/dPZNgF+918bi/yMFotdLEonwmnvKQU3d48UVo0wY6dYLt24MZ\nTe+/D5dcAklxfQxRNsXrR3RnIW1Hm9kEM+sWpxhE9uvyyy8POwSJIuUz8SRyTnNyYPZsaNUKLrgg\nGCezYAEsXx58b3bwa0ggJkWNmX1uZu+Z2UNm1p1fxs/kcfe17t4bONzMro1FHCIiIqXVrl3wzDPQ\nokXwJKZWLXj9dXjzzeBJjYqZoovVk5obgfcIFtWbDXQys9VmNsXMrjWzlD0HuvvfgZYxikNERKRU\nyc6GJ54IBv5eeSU0agRvvx0MBm7fXsVMScSkqHH319z9FndvCRwBfAw8DzQBJgKfm9mXZvZs7t5P\nJ8ciDpFILVmyJOwQJIqUz8STCDndsSPYYPJXv4JeveCkk2DFCnjpJTjzzLCjSwwxH1Pj7t8DX7n7\n3e7eHqgBnAk8lHtIE2BQrOMQOZDRo0eHHYJEkfKZeMpyTrOyYOxYaNwYbrkFzjgDPvwQ/vlPaN06\n7OgSS7wmhk3Z8z/uvgtYnvslUirMmjUr7BAkipTPxFMWc/rjjzBpEjz4IGzZEnQ13XUXNGsWdmSJ\nK14rCs/et83MGgK3AIvd/eV4xCGyP8laXzyhKJ+Jpyzl9IcfYPx4GDMGtm2D666DO+4IntRIbMVr\nnZqHcqdv/97M6gK4+3rgDiDHzP4W5fv1NrMvzOxnM1tmZqcd5PhDzWyEma0zs+1mttbMrotmTCIi\nkti2bIG//AUaNgxWAb78cvj8c3jsMRU08RKv7qctwDDgj4Cb2WfAG8CbwFqgQbRuZGY9CMbr3Ai8\nCwwAFpiKSPJRAAAgAElEQVRZ09xF/wrzHFAH6AmsAY4izqsti4hI2bRpU/BUZuLEYM2Zm2+G22+H\no44KO7LyJ16/uI8EOgO1ge7AXIJp3NMIiptojq8ZAEx296fcfRVwM5AF9CrsYDPrArQFurr76+6+\nwd2Xu/vSKMYkpdygQRqrnkiUz8RTGnP61Vdw662QkhIUNH36wLp18NBDKmjCEq8nNTvc/fXc/5+X\n+4WZ1QHGAk9F4yZmVhFoDdy3p83d3cwWAW32c9oFwArgT2Z2NfATQdH1Z3ffHo24pPRr0CBqDwul\nFFA+E09pyun69TByJEybBsnJMHgw9OsXLJ4n4YpXUdPQzGq4+w97N7r7ZjP7AzAcGBiF+9QGKgCb\n9mnfBOxvvHljgic12wmeItUGJgG1gOujEJOUAX379g07BIki5TPxlIacfv453H8/PPUU1KwJw4ZB\n795w2GFhRyZ7xKv7aQbwHzMrsCOZu2cBHqc4CpME5ABXuPsKd/83QYF1rZlVCjEuEREpBTIy4Kqr\ngqnY8+fDqFFBN9Odd6qgKW3iUtS4+zxgDDDfzNab2WNm1svMOuY+qflVlG6VCewG6u7TXhf4337O\n+ZpgccAf92rLINiv6pgD3axr166kpqbm+2rTpg1z5szJd9zChQtJTU0tcH7v3r2ZOnVqvrb09HRS\nU1PJzMw/pnno0KGMGjUqX9uGDRtITU1l1apV+dofeeSRAv3PWVlZpKamFliVc+bMmfTs2bNAbD16\n9ND70PvQ+9D7KNfv46abhtKixShOPDHYj2ncOHjjjQ0sXpzKxo1l532EmY+ZM2fm/W6sV68eqamp\nDBgwoMA50WLu8XtIYmatCbqaOhJ0EwGsBC5199VRuscyYLm798/93oANwDh3f6CQ428AHgaOzH1q\nhJldSLCtQzV331HIOa2AtLS0NFq1ahWNsCVkq1at4vjjjw87DIkS5TPxxDOn770Hw4fD3LnBIOA7\n74Rrr4VDD43L7RNeeno6rYOllFu7e3o0rx3Xacvunubu5xFMnz4dON7dT45WQZNrDHCDmV1jZscD\njwLJwJMAZna/mU3f6/hnCKacP2Fmzc2sHTAamFpYQSOJafDgwWGHIFGkfCaeeOR0yRLo0gVOPx1W\nrYInn4TVq+GGG1TQlBXxGigMgJlVBarkrhezIhb3cPd/mFlt4F6CbqcPgM7uvjn3kHrAsXsd/5OZ\n/RZ4hGBn8S3As8CfYxGflE7jx48POwSJIuUz8cQqp+7w2mvBk5nFi4NNJmfNgksugQoVDnq6lDJx\nKWrMrD7Bk5IOwbe2hWDBu9G5KwtHlbtPJNgNvLDXCnQAuvunBOvoSDlVmqaLSskpn4kn2jl1h5df\nDoqZpUuhVSt44QW48EJI0tKrZVa8UjeFYHXfWwjWkPkPwWJ4H5nZpXGKQUREyrmcHJgzB047Dc4/\nP2ibPx9WrICLLlJBU9bFq/vpE3e/e+8GM0sGrgYmmdkmd38jTrGIiEg5s3s3PP88jBgBK1dC+/aw\naBGccw6YhR2dREvUa9Lc6drXm1nTvZo9dxbSLw3uWe4+GTgN6B/tOESKYt/pjlK2KZ+Jp7g53bUL\n/v73YKzMZZfB0UfDW2/B66/DueeqoEk0RSpqzOwvZnaVmZ1qZtX2c9iFwGNAhpl9bWb/ACoC/zCz\nAuPH3X0tsK6IcYtEVVZWVtghSBQpn4mnqDnduROmTAkWzLvmGmjaFJYvh3//G846K0ZBSuiKtE6N\nmeUQrP67DdgK7CAoSHrnDrbFzFYA7YBTc//bjmDfparAZmBJ7td7BAviNQeucvceUXlHcaB1akRE\nSqft22Hq1GDV340bg1lMd98Np5wSdmSyRyzXqSnOmJo/uvtje74xs1rA3ns6DcxdxO7N3C/MrAK/\nFDlnA0OAwwm2J3gVuKFY0YuIiAA//QSTJ8MDD8A33wRdTXfdBSeeGHZkEk9FLWo27l3QALj7t/t8\n/+a+J7n7bmB57tcDALlryfzk7j8XMQYREREAtm6FCRNgzBj4/nu4+upgBeBfRWvzHSlTijpQeGO0\nbuzumSpopLTYd68UKduUz8Szb06/+w7uuQcaNQp2y77kEvjsM5g2TQVNeVbUomZnTKIQCVmvXr3C\nDkGiSPlMPHtyunlz0K3UsCGMHBkMAl6zBiZNCgocKd/iuk2CSGk1bNiwsEOQKFI+E0/v3sO4/fag\neDGD3r1h4ECoWzfsyKQ0KWpR08DMKrv79phEIxISzWJLLMpn4ti4EUaPhscfb0XlynDbbdC/Pxxx\nRNiRSWlU1KKmMfCNmS3hl9lN77r7rqJcxMxGu7u20RURkUKtXRt0Lz35JFSvDkOGQJ8+ULNm2JFJ\naVac7qdqQBd+2QByu5kt55ciZ2kEA4BPL8Z9RUQkwa1eDffdBzNmBE9jRoyAm28OChuRgynqQOGv\ngD8DC4GfAAOqAO1z218BvjOzd8xspJl1NbPDCrlO5eKHLBJ9U6dODTsEiSLls+xZuTJYW6Z582BP\npocegi++gEGDgoJGOZVIFLWoWePuI9y9C1CT4InL7cA84HuCIudQ4AxgUG77FjNLN7O/mdlFZtYA\naBi1dyASBenpUV3UUkKmfJYd6enB7tgtW8KyZTBxYtD11L8/JCfvfZxyKgdX1G0SXnP3cw7wegt+\n2RqhHbD3uPR8N3L3CkULtfTQNgkiIiWzbBn89a8wfz40aRJM077qKqhYMezIJNZK0zYJ+9vEEgB3\nXwmsBCYA5O7UvXeR02DPoUW8r4iIJIA33giKmVdfhRNOCMbOXHopHKIFRiQKitr9VL8oB7v7p+4+\nxd2vcfdGQApwdxHvKSIiZZg7LFwI7dpB+/aQmQnPPx+Mo7niChU0Ej1FLWqOMrPTinszd1/v7vcD\nXxT3GiIiUja4w7x5cMYZ0LlzsIP23Lnw/vtw8cWQVNTfQCIHUZw/UpPM7IDdUBH4poTni0RVampq\n2CFIFCmf4crJgdmzoVUrSE2FQw+FBQtg+XK44IJgReCiUk4lEkUtau4CKgErzayXmVUp5n21h5SU\nKn369Ak7BIki5TMcu3YFY2RatAg2mKxVC15/Hd58Ezp1Kl4xs4dyKpEoUlHj7iPdvQWQCjQBlpnZ\nVDMr6u4brxfxeJGY6tSpU9ghSBQpn/GVnQ1PPBGsMXPVVcHGku+8EwwGbt++ZMXMHsqpRKJYw7P2\nmuV0l5m1AbKLeP49xbmviIiUHjt2BMXMyJGwfj107w6zZkEwW1ck/ko85tzdl0YjEBERKRuysmDK\nlGCjyf/+N5iSPW9e0O0kEqaIu5/MrKmZaay6JKQ5c+aEHYJEkfIZGz/+CA88ACkpMHAgnHsuZGQE\nT2diXdAopxKJohQp/wC+MrMHc1cOFkkYM2fODDsEiSLlM7p++CHYWLJhQ7j7brjwQvj0U5g+HZo1\ni08MyqlEIuLuJ3c/xczOBq4BlpjZWmA6MMPdN8cqQJF4ePbZZ8MOQaJI+YyOLVvgb3+DRx4J1pj5\nwx9g8GBo0ODg50abciqRKOrspzfc/XqgHvAQ0BXYYGZzzex3ZqZdO0REyrhNm+BPfwpmMY0ZA9df\nH+yYPX58OAWNSKSKNUbG3X9296fdvRPwK+AdYATwtZlNKMmqwyIiEo6vvoJbbw3GzEyaBH36wLp1\n8NBDcNRRYUcncnAlHvjr7l/mrl/THDg/t/nfZpZhZneYWZH2ixIRkfhavx5uuQUaNw7GyfzpT0Ex\nc//9UKdO2NGJRC6qs5ncfbm79waOAv4MnAmsMbOFZnZFCVYgFompnj17hh2CRJHyGZnPPw+6lpo0\ngeeeg2HDggJn6NBgNeDSRDmVSMRkira773T35909FTgWmA8MAv6XuwJx21jcV6S4tFppYlE+D+yT\nT4KVf5s1g/nzYdSo4MnMnXfCYYeFHV3hlFOJhLl7/G5mdjLB7KkrgSzgKeApd18btyCiwMxaAWlp\naWm0atUq7HBERCLywQfB1OzZs6F+/aCb6frroYqeoUscpaen0zpYdrq1u6dH89pxXUzP3f/j7rcB\n9YF+wAnAR2b2ppldb2bV4xmPiEh58O67wW7Zv/41pKXB5MmwZk0wEFgFjSSSUFYIdvfd7v6iu18K\nHA08A1xPMHtqShgxiYgkmiVLoHNn+L//+2WxvE8/hRtugEMPDTs6kegLfdsDd//e3R919zOBVsAb\nYcck5c+SJUvCDkGiqDzn0/2X3bHbtg32Zpo1Cz7+GK65Bg4p8Y5/4SjPOZXIhV7U7M3dP3X3v4cd\nh5Q/o0ePDjsEiaLymE93ePll+M1voGNH2LYNXngB/vMf6NEDKlQIO8KSKY85laIrVUWNSFhmzZoV\ndggSReUpnzk5MGcOnHYadO0atM2fDytWwEUXQVKC/C1fnnIqxRe3P+5m1tLMkvdpi9NWaCIHlpyc\nfPCDpMwoD/ncvRuefRZOOSUoXqpXh0WL4O234bzzwCzsCKOrPORUSi4uRY2Z3QG8S7Bezd7qmdnI\neMQgIpIIdu2Cp56CE0+Eyy4Lti946y14/XU499zEK2ZEiiJeT2oOB64D8u0d7+5vAOlmdn5hJ4mI\nSGDnTpgyJVgw79proWlTWLYMFiyAs84KOzqR0iFeRU1Fd5/l7pP3fcHd/wH8Nk5xiBRq0KBBYYcg\nUZRI+dy+HSZMCLYyuOGGYK2Z99+HuXODqdrlRSLlVGInXkXNEWZ23AFe3xmnOEQK1aBBg7BDkChK\nhHxmZcHDDwebTPbrB+3awUcfwfPPB+NoyptEyKnEXryKmrHAogN0M1WLUxwiherbt2/YIUgUleV8\nbt0KI0dCo0YweHAw6HfVKnj66WAcTXlVlnMq8ROXZZjcPd3MhgAvmNmnwBzgA2ArcBYqakSknPvu\nOxg3DsaOhZ9+gl69gr2ZGjUKOzKRsiNua0u6+wwzywBGAXcBe8bovwJcFq84RERKk82bg26m8eMh\nOxtuugkGDQo2nBSRoon3hpbp7v5boC5wBtDY3Tu7+3fxjENkX6tWrQo7BImispDPr7+G228PnsSM\nGwd//COsWwd/+5sKmsKUhZxK+MLa0DIT+BjYEsb9RfY1ePDgsEOQKCrN+dy4Efr2hZQUePxxGDgQ\n1q+HUaOgbt2woyu9SnNOpfQIc2uzKsBAM8sBRrj7zyHGIuXc+PHjww5Boqg05nPtWrj//mCn7OrV\nYcgQ6NMHatYMO7KyoTTmVEqfuBQ1ZpYCDAWOAP4DzHX3d4G7zKw+MBy4LR6xiBRG00UTS2nK5+rV\ncN99MGMGHHEEjBgBN98cFDYSudKUUym94tX99CTBOJqKQE9gmZl9ZmZ3AbWAOnGKQ0QkLj76CC6/\nHJo3h1dfhTFj4IsvgkHAKmhEYiNeRU26u5/n7l3cvT7BIOF5QH+Cqd2bonkzM+ttZl+Y2c9mtszM\nTovwvN+YWbaZpUczHhEpP9LT4Xe/gxYtYOlSmDQJ1qwJFtDTnowisRWvombH3t+4+7vuPhCoB9R2\n96itf21mPYCHCLq7fk3Q3bXAzGof5LwawHRgUbRikbJj1KhRYYcgURRGPpctg/PPh9atYeVKmDYN\nPvssmKJdqVLcw0k4+oxKJOJV1HxgZr/bt9ED0Z7OPQCY7O5Pufsq4GYgC+h1kPMeBWYAy6Icj5QB\nWVlZYYcgURSvfLrD4sXQsSO0aRN0Lz39NGRkQM+eULFiXMIoF/QZlUjEq6h5AehlZsMO9sSkJMys\nItAaeHVPm7s7wdOXNgc4ryeQAtwTq9ikdLvnHqU+kcQ6n+6wcGGwH1OHDpCZCc89F4yjufJKOCTM\neaUJSp9RiUS8ipqZwOnAn4FNZvaxmU00s8vM7Kgo3qc2UIGCY3Q2EXR1FWBmvwLuA65095woxiIi\nCcYd5s2DM86Azp1hx45gt+z334dLLoGkUFb+EpE94vUR/NrdjwQOBy4E5gOnAn8HvjSzV+IURz5m\nlkTQ5TTU3dfsaQ4jFhEpvXJygt2xf/1rSE2FQw+FBQtg+XK44AIw/a0hUirEq6h5z8xGAKcA8919\nkLufTlDkXEBQWERDJrCbYPr43uoC/yvk+OoExdX43FlP2QRPk04xs51m1v5AN+vatSupqan5vtq0\nacOcOXPyHbdw4UJSU1MLnN+7d2+mTp2ary09PZ3U1FQyMzPztQ8dOrTAQLkNGzaQmppaYPnwRx55\nhEGD8o+9zsrKIjU1lSVLluRrnzlzJj179iwQW48ePcrV+8jMzEyI9wGJkY+Svo/MzMyovI+nn55J\nu3Y9Oekk+P3voXbtYAzN0Uf3ICtrTr5iRvmI7fvIzMxMiPcBiZGPSN/HzJkz83431qtXj9TUVAYM\nGFDgnGixYMhJbJnZTcChBLtxT3f3/8bwXsuA5e7eP/d7AzYA49z9gX2ONaD5PpfoDXQALgbWFbbS\nsZm1AtLS0tJo1apVDN6FxFtqaipz584NOwyJkpLmMzs7WCzvvvuCGUxduwYrALfZ78g8iTV9RhNH\neno6rVu3Bmjt7lFdQiVew9l+S9DtVAHoZ2ZvAW8Cb7r7h1G+1xjgSTNLA94lmA2VTLAAIGZ2P3C0\nu1+bO4j4k71PNrNvgO3unhHluKQUGzZsWNghSBQVN587dsCTT8LIkcHmkt27w8yZwTRtCZc+oxKJ\nuBQ17n6JmVUDzgTOyv0aCVQxs++Bt4E5wAx337H/K0V0r3/kzrC6l6Db6QOgs7tvzj2kHnBsSe4h\niUdP3BJLUfOZlQVTpsDo0fDf/8KllwYDgFu0iFGAUmT6jEok4tL9VOiNzQ4hmH59H0GR0RhYD1wU\ng6c3UaXuJ5HEsG0bPPooPPggbNkCV1wBd90Fxx8fdmQiiSuW3U+hTUB0913uvhw4D/gncBQwFXjR\nzLRzmYjEzA8/wPDh0KgR3H13MKNp9Wp46ikVNCJlWVyKGjPrYGZzzGy0mbXc+zV33wkkuftmd78P\nuAj4SzziEtlj31kEUrZNmTKl0PYtW+DPf4aGDYOi5vLL4fPP4fHH4bjj4hykFIk+oxKJeD2puRNY\nC3QD3jezT8xsjJndZGZ3sNdqv+6eBmyNU1wiQPA4VMq2bdu2MbRfPzqmpDDi1lvpmJLC0H792LZt\nG998A3/6U/BkZswYuP76YEuD8eOhgZ4Llwn6jEok4jX7KSN3A8uBZnYGcDlBt9PNwDpy92Uys/FA\nOrA9TnGJADBhwoSwQ5AS2LZtGxe3acPAjAyG5eRggP/0EzPG/5OTZpzG5p+v4pBDjL59YcAAqFMn\n7IilqPQZlUjEq6h5wsz+Bsxy92Xsf9PIU4FrgMFxiktEEsCDd9/NwIwMuuQEO52spwEjuYNp3otD\nv83i9NP/zQsvn0etWiEHKiIxFZfuJ3f/gKBQaWRmxxzg0DOBRu7+aDziEpHE8Pa8eXTOyeFzjqMX\nU2nC5zzPJQzlHr6kIRW/uUUFjUg5EM+9ZCu6+6wDHZC7oeS3cYpHRBKAu5OT1Yir+SszuZwj+YZR\n/ImbmExVsgBIzj4Md8e0SZNIQovX7KcbgK/N7Nbc7y8wsxfNbLyZpcQjBpEDKWxvFSn9PvgAfv97\n4/VvXuVN2jGW/nxBCot5OK+gceCnihVV0JRx+oxKJOI1+6kB0B74Z+6U7heAJkAWMNvMtDKEhKpP\nnz5hhyBF8O67wdoyv/41vP8+dDvnWSZYU/owgcrsYO9s/jspibP0C7HM02dUIhGvoqaqu6e7+3rg\nqtz7XunugwnWpRkYpzhECtWpU6ewQ5AILFkCnTvD//0ffPopTJ8eLJr3zJxuPHJCE15OSsKBTgRP\naF5OSuLh5s25bfjwkCOXktJnVCIRr6KmmpnVzt0Vuwvwv9z1aMgtdEq035OIJC53eO016NAB2rYN\n9maaNQs+/hiuuQYOOQSqV6/O7KVLWd6nD50aNeLC+vXp1KgRy/v0YfbSpVSvXj3styEicVCkgcJm\n9heCRfRWAavc/ccITx0FLAIqAs2Bu/d5XYvtiUg+7vDvf8Nf/wpLl0KrVvDCC3DhhZBUyD/Hqlev\nzrCxY2HsWA0KFimnivqkZhgwnaBA+cTMPjOzV8ys6YFOcvc1BNO17wTau/v9ALmDhe9Ai+1JyObM\nmRN2CJIrJwfmzIHTToOuXYPi5qWXYMUKuOiiwguaff3rX/+KfaASV/qMSiSK0/30R3ev6e4N3P1X\nQA9gzcFOcvcsd5/r7m/uez3gjWLEIRI1M2fODDuEcm/3bnj2WTjllKB4qVYNFi2Cd94JipuiPHhR\nPhOPciqRMHeP/GCz9e7eMIbxlAlm1gpIS0tLo1WrVmGHI1Km7doFzzwD990XDPrt1AmGDAnGz4hI\n4klPT6d169YArd09qpt6FfVJzcZo3lxEyq+dO2HKFGjWDK69Fpo2hWXLYMECFTQiUjxFXVF4Z0yi\nEJFyY/t2mDoVRo2CL7+Eiy+G2bODbicRkZKI5zYJIlKO/fQTTJ4MDzwA33wDl18Od90FJ5wQdmQi\nkiiK2v3UwMwqxyQSkRD17Nkz7BAS1tatcP/90KgR/OlPwaDfVavg6adjV9Aon4lHOZVIFPVJTWPg\nGzNbAryZ+/Wuu+8qykXMbHTuasIipYJWK42+776DceNg7NjgKU2vXkFR06hR7O+tfCYe5VQiUdTZ\nTzl7fbvnxO3Acn4pcpa6+88Huc5id29ftFBLD81+Etm/zZvh4Ydh/HjIzoabboJBg6B+/bAjE5HS\nIJazn4r6pOYr4FGgLcFietWAKgSbVZ6de0y2maXzS5GzxN33XTFYXVgiCebrr+HBB+HRR4M1ZW65\nBW67DerWDTsyESkvilrUrHH3EQBmlgS0AtoRFDRnAYcDhwJnAP8HDAJyzGwlQYHzBpAGlPu1bkQS\nxcaNwUymKVOgUiUYOBBuvRWOOCLsyESkvCnqQOG87id3z3H3Fe4+xt0vdPcjgJOBvsBzwDeAARWA\nU3Lbnwe+AI6MRvAi0bJkyZKwQyhz1q6FG2+E446DmTODBfPWrw/2agq7oFE+E49yKpEoalFT7UAv\nuvtKd5/g7j3c/SjgeOBG4GmChfss90ukVBk9enTYIZQZq1f/sljev/4FI0YExcyQIVCzZtjRBZTP\nxKOcSiSK2v1UpKF+7v4p8CkwBcDMGgJXAMOLeF+RmJo1a1bYIZR6K1cGBcw//gFHHw1jxsANN0CV\nKmFHVpDymXiUU4lEUZ/UHGVmpxX3Zu6+PneH7i+Kew2RWEhOTg47hFIrLS3YYLJly2Abg4kTYc0a\n6NevdBY0oHwmIuVUIlGcXbonmdkBu6Ei8E0JzxeRGFu6FM4/H049FT76CKZNg88+g5tvDgYEi4iU\nNkUtau4CKgErzayXmRX332naQ0qkFHKHxYuhY0c480z44otg5d+MDOjZEypWDDtCEZH9K1JR4+4j\n3b0FkAo0AZaZ2VQzK+pKFK8X8XiRmBo0aFDYIYTKPdgdu1076NABMjPhueeCJzRXXgmHlLFd4sp7\nPhORciqRKE73055ZTne5+8kEg4Czi3j+PcW5r0isNGjQIOwQQuEO8+bBGWdAly6wYwfMnQvvvw+X\nXAJJxfobInzlNZ+JTDmVSBRpmwQJaJsEKetycuCf/4Thw+GDD+Css+DPf4bf/jZYDVhEJFZiuU1C\nGf13mEjiiuU/NHbvhmeegRYtgicxRxwRjKF56y3o1EkFjYiUbSpqREqBbdu2MbRfPzqmpND92GPp\nmJLC0H792LZtW1Sun50NTzwBzZsHY2QaNYJ33oFFi+Dssw96uohImaCiRgRYtWpVaPfetm0bF7dp\nQ5sJE3hl3Tr+9dVXvLJuHW0mTODiNm1KVNjs2BFsMPmrX0GvXnDiibBiBbz0ErRpE8U3UcqEmU+J\nDeVUIqGiRgQYPHhwaPd+8O67GZiRQZecnLw9RAzokpPDgIwMHhoypMjXzMqCsWODfZluuSUYCPzh\nh8E4mqArO7GFmU+JDeVUIqGiRgQYP358aPd+e948OufkFPpal5wc3p47N+Jr/fgjPPAApKTAbbfB\nOefAJ5/ArFnBOJryIsx8SmwopxKJMrb6hEhshDVd1N2pmp29311eDUjOzsbdsQOM4v3hBxg/Hh5+\nGLZuheuugzvugMaNYxF16afpv4lHOZVIqKgRCZGZ8VPFijiFb1/vwE8VK+63oNmyJehmGjcOtm+H\nP/wBBg8G/f0vIuWRup9EQvabCy5gwX5Wuft3UhJnpaYWaN+0Cf70p2AW00MPwfXXB1sajB+vgkZE\nyi8VNSLAqFGjQrv37SNGMKZ5c15OSmLPCjUOvJyUxMPNm3Pb8OF5x371Fdx6azBmZtIk6NMH1q0L\nCpujjgoj+tIpzHxKbCinEgl1P4kAWVlZod27evXqzF66lIeGDGHM3LkkZ2eTVbEiv0lNZfbw4VSv\nXp1162DUqGCn7OTk4ClN375Qq1ZoYZdqYeZTYkM5lUhom4Ri0DYJEkt7Dwr+7DO4/374+9+hZk0Y\nOBB694bDDgs5SBGRYorlNgl6UiNSypgZn3wCI0YEU7GPPDJ4SnPTTVC1atjRiYiUXipqREqR//wn\n2GRy9mw45phgVtP110PlymFHJiJS+mmgsAiQmZkZ6v3few8uvBBOOQXS0+Gxx+Dzz4OuJhU0RRd2\nPiX6lFOJhIoaEaBXr16h3HfJEujSBU4/HVavhunTg//+4Q9w6KGhhJQQwsqnxI5yKpFQUSMCDBs2\nLG73codXX4UOHaBt22Ca9qxZ8PHHcM01cIg6hUssnvmU+FBOJRIqakQgLrPY3GH+fPjNb6Bjx2A7\ngxdeCMbR9OgBFSrEPIRyQ7MSE49yKpFQUSMSYzk5MGcOnHYanH9+UNy89BKsWAEXXQT7WUxYRESK\nSH+disTI7t3wj38Eg38vugiqV4dFi+Cdd6BrVzjA/pQiIlIMCVnUmFlvM/vCzH42s2VmdtoBjr3I\nzBaa2Tdm9oOZvWNmneIZr4Rv6tSpUbvWrl3BYnknnRR0Kx19NLz1Frz+Opx7roqZeIhmPqV0UE4l\nElIiTS4AABZvSURBVAlX1JhZD+AhYCjwa+A/wAIzq72fU9oBC4HzgFbA68A8Mzs5DuFKKZGeXvJF\nLXfuhClToFmzYMBv06awfDn8+99w1llRCFIiFo18SuminEokEm6bBDNbBix39/653xuwERjn7qMj\nvMZHwCx3H76f17VNguTZvh2mTg1W/f3yS7j4Yrj77qDbSURE8ovlNgkJ9aTGzCoCrYFX97R5ULUt\nAtpEeA0DqgPfxiJGSRw//QRjxgQ7Zv9/e3ceJkV17nH8+6qIgKhREMQFuEqIGjRCUFFB4oJLLm0S\nMbjEEAhJQAgGDahX7iXJgxFQ3MElRDH3MaOoCe6CIWhEWXRGI0ZwiY54UYkYFRQXhPf+UTXaM0zP\n1Mx0d02f+X2ep59hqk5VvT0v0/M+p06dM24cDBgAzz8Pd96pgkZEJA2hzYjRAdgWWFtj+1qgZ8Jz\nTADaAXPzGJcEZP16mDkzKmjefx/OPhsuugh69Eg7MhGRli20oqZJzOxM4L+BjLtrTm6p5r334Oqr\no9fGjTBiBFxwAXTrlnZkIiICgd1+AtYBm4FONbZ3At6u60AzOx24CTjN3RcludjJJ59MJpOp9urX\nrx/z5s2r1m7BggVkMpmtjh8zZsxWI/orKirIZDJbrXMyefJkpk2bVm3b6tWryWQyrFq1qtr2a6+9\nlgkTJlTbtnHjRjKZDIsXL662vaysjOHDh28V29ChQ1vU+8hkMjnfx4knZjj33HV07RqNmxk2DM45\nZzLduk2rVtA0h/cBYeSjqe8jk8kE8T4gjHzk431kMpkg3geEkY+k76OsrOyLv42dO3cmk8kwfvz4\nrY7Jl5YyUHg10UDhy3IccwYwGxjq7vcnuIYGCgdmwYIFDBpU/Un+t96CGTPg+uujx7DHjIHzzoNO\nNUtmaXZqy6eUNuU0HIUcKBzi7acrgDlmVg4sB8YDbYE5AGZ2KdDF3YfF358Z7xsHPGVmVX+yPnb3\n9cUNXdKS/WH5xhswfTr87nfQunVUyPziF7DbbikGKA2iP37hUU4lieCKGnefG89J8xui207PAie4\n+ztxk87A3lmH/IRocPHM+FXlVkDLwrYgr74KU6fCnDnR7L+TJsHYsbDLLmlHJiIiSQRX1AC4+yxg\nVo59w2t8/62iBCXN1qpVcOmlcNttUW/MlCkwenRU2IiISOkIbaCwSGIrVsDpp8MBB8B9981jxgx4\n7TWYOFEFTamrOdhSSp9yKkmoqJEWp7w8WmDyoINg6VKYNQuOPbaMc8+Ftm3Tjk7yoaysLO0QJM+U\nU0lCRY20GEuWwLe/Dd/8ZjTz7803w8svw6hRcOedd6QdnuTRHXcon6FRTiUJFTUSvMceg+OOgyOO\ngMrKaOzMypUwfDi0apV2dCIiki8qaiRI7rBgQbQe08CB8O67cNdd0TiaM8+E7YIcIi8i0rKpqJGg\nuMN998Hhh8MJJ0QraN97L1RURKtnb6P/8SIiwdJHvARhyxa4+27o3RsyGdh+e5g/H5Ytg8GDoxmB\n61LbVN9SupTP8CinkoSKGilpn38ejZHp1QuGDIFdd4VHH4XHH4dBg+ovZqpottKwKJ/hUU4lCRU1\nUpI2bYJbboH994cf/CBaKfvJJ2HhQjj66Iaf74wzzsh7jJIe5TM8yqkkoaJGSsqnn8INN0CPHjBi\nBHz96/D00/DAA9CvX9rRiYhImvQMiJSEjRth9uxoock334ShQ6MBwb16pR2ZiIg0F+qpkWbtww/h\nssuge/dotexjj43mmCkry29Bs3jx4vydTFKnfIZHOZUkVNRIs/TBB9HCkl27wsUXwymnwEsvwa23\nQs+e+b/e9OnT839SSY3yGR7lVJLQ7SdpVt59F666Cq69NppjZuTIaIHJffYp7HVvv/32wl5Aikr5\nDI9yKkmoqJFmYe1amDEjWlzSPVqP6Ze/hD32KM7122oly6Aon+FRTiUJFTWSV+6OJZ0cBlizJhr8\ne9NN0TpM48bB+PHQsWMBgxQRkSCpqJEm27BhA5dffDFP3Hcf7TZt4qNWrThy8GB+eckltG/fvtZj\nKith6tRorpm2beHCC+HnP48mzxMREWkMDRSWJtmwYQOn9utHv5kzeaSyknvWrOGRykr6zZzJqf36\nsWHDhmrtX3klml+mR49oWYNf/xpefx0mT063oJkwYUJ6F5e8Uz7Do5xKEipqpEkuv/hizlu5khO3\nbKHqppMBJ27ZwviVK5kxaRIAL7wQzfzbsyc89BBMmxb11lx4Iey0U1rRf2mfQo9ElqJSPsOjnEoS\n5u5px1ByzKw3UF5eXk7v3r3TDidVx3XvziOVldQ2isaBfl1OYu8jHuTuu2GvveCCC6KemjZtih2p\niIg0BxUVFfTp0wegj7tX5PPcGlMjjebutNu0qdaCZjl9mcIklr2Z4Z0K56abjB/+MFo9W0REpBB0\n+0kazcz4qFUrsvv6FnMkJ/Awh7Gcl/gqB3Q4n1WrjJEjVdCIiEhhqaiRJjly8GAetm1YyDEMZBH9\nWcybdOF2hnK59eK0Mz+nVau0o6zfqlWr0g5B8kj5DI9yKkmoqJFGc4eDjp7KGTuUcxwL2UB7/sx3\neJaD2Wmbu7jmgJ6cP2VK2mEmMnHixLRDkDxSPsOjnEoSKmqkwbZsgXnzoG9fGDKkLT179eLMzA18\npesQbtnzaU7s1pVlY8dy95IlOeepaW6uu+66tEOQPFI+w6OcShIaKCyJbd4Md90Fl1wCK1bAwIGw\ncCF861vbYjYKGNXgGYWbCz0uGhblMzzKqSShnhqp1+efwx/+AAceCKefDl26wOOPw6JFcMwxkF3D\nlGJBIyIiYVBRIzl99hnMnh1NmDdsWPR12TJ4+GE46qi0oxMREalORY1s5ZNPYOZM2G8/+OlPoXdv\neOYZuOceOPTQtKMrjGnTpqUdguSR8hke5VSS0Jga+cJHH8GNN8Jll8G//gVnnAH/9V9wwAFpR1Z4\nGzduTDsEySPlMzzKqSShZRIaIbRlEtavj3pmrrgC3n8fzj4bLrooWnRSREQkn7RMghTEe+/BNdfA\n1VdHvTQjRkRrM3XrlnZkIiIiDaeipgV65x248kq47jrYtAl+9jOYMAH23DPtyERERBpPA4VbkLfe\ngvPPj3pirrkGRo+Gykq46ioVNOvWrUs7BMkj5TM8yqkkoaKmBXjjDRg7Frp3jx7RPu88eP11mDYN\nOnVKO7rmYcSIEWmHIHmkfIZHOZUkdPspYK++ClOnwpw50L49TJoUFTe77JJ2ZM3Pr371q7RDkDxS\nPsOjnEoSKmoC9OKL8Nvfwm23wW67wZQp0a2mElmGKRUhPMUmX1I+w6OcShIqagLy/PNRATN3brSU\nwRVXwMiR0LZt2pGJiIgUnsbUBKCiAr73PejVC5YuhVmz4J//hHHjVNCIiEjLoaKmhG3eDKecAn36\nRKtm33wzvPwyjBoFrVunHV1p+f3vf592CJJHymd4lFNJQkVNCdt2Wzj44GjszMqVMHw4tGqVdlSl\nqaIir5NaSsqUz/Aop5KElklohNCWSRARESmWQi6ToJ4aERERCYKKGhEREQmCihoREREJgooaESCT\nyaQdguSR8hke5VSSUFEjAowdOzbtECSPlM/wKKeShIoaEWDQoEFphyB5pHyGRzmVJFTUiIiISBBU\n1IiIiEgQVNSIAPPmzUs7BMkj5TM8yqkkEWRRY2ZjzOw1M/vYzJaaWd962g80s3Iz+8TMXjKzYcWK\nVZqHadOmpR2C5JHyGR7lVJIIrqgxs6HADGAycAjwd2C+mXXI0b4bcD+wEDgYuBqYbWbHFyNeaR46\nduyYdgiSR8pneJRTSSK4ogYYD9zo7n9w91XAKGAjMCJH+9HAq+4+0d1fdPeZwF3xeURERKREBFXU\nmFkroA9RrwsAHq3Y+RegX47DDo/3Z5tfR/tmp6ysrNmcryHHJmlbX5u69ufal++fV74VIr7GnjPf\n+ayvnfJZ2HM29LhC/o6Waj5Bn7kN3VfMnAZV1AAdgG2BtTW2rwU65zimc472O5lZ6/yGVxj6BWvY\nvub+oVmqfwRV1NSuVPOZtL2KmnTPp8/c6rYr2pXCsgPAypUr044DgA8++ICKivyt3t6U8zXk2CRt\n62tT1/5c+2rbvnz58rz+DJsi3/lsyjnznc/62imfhT1nQ48r5O9oQ7eHnNOW9pmb9bdzh3qDbiCL\n7s6EIb79tBE41d3vzdo+B9jZ3b9byzGPAeXufl7Wth8BV7r7V3Jc50zgtvxGLyIi0qKc5e5/zOcJ\ng+qpcfdNZlYOHAvcC2BmFn9/TY7DlgAn1dg2KN6ey3zgLKAS+KQJIYuIiLQ0OwDdiP6W5lVQPTUA\nZvZ9YA7RU0/LiZ5iGgJ8zd3fMbNLgS7uPixu3w1YAcwCbiYqgK4CTnb3mgOIRUREpJkKqqcGwN3n\nxnPS/AboBDwLnODu78RNOgN7Z7WvNLNvA1cC44D/A36sgkZERKS0BNdTIyIiIi1TaI90i4iISAul\nokZERESCoKKmAMzsP81slZm9aGY/TjseaRoz+5OZ/dvM5qYdizSdme1lZovM7B9m9qyZDUk7Jmk8\nM9vZzJ4yswoze87MRqYdkzSdmbUxs0ozm96g4zSmJr/MbFvgBeBo4EOgAjjM3d9LNTBpNDMbALQH\nhrn799OOR5rGzDoDu7v7c2bWCSgHerj7xymHJo0QT9vR2t0/MbM2wD+APvrMLW1mNgXYF3jD3Scm\nPU49Nfl3KPC8u7/t7h8CDxDNeyMlyt3/RlSgSgDi383n4n+vBdYBu6YblTSWR6rmC2sTf7W04pGm\nM7P9gJ7AQw09VkVN/nUB1mR9vwbYM6VYRKQOZtYH2Mbd19TbWJqt+BbUs8Bq4DJ3/3faMUmTXA5c\nRCOKUxU1Wcysv5nda2ZrzGyLmWVqaTPGzF4zs4/NbKmZ9U0jVqmf8hmefObUzHYFbgV+Uui4pXb5\nyqe7f+Du3wC6A2eZWcdixC/V5SOf8TEvuvsrVZsaEoOKmuraEU3Wdw6w1WAjMxsKzAAmA4cAfwfm\nx5P9VXkT2Cvr+z3jbVJ8+cinNC95yamZbQ/8Gfituy8rdNCSU15/R+NJVv8O9C9UwFKnfOTzcOB0\nM3uVqMdmpJlNShyBu+tVywvYAmRqbFsKXJ31vRHNQDwxa9u2wIvAHsCOwErgK2m/n5b+amw+s/YN\nBO5M+33olZ+cAmXA/6T9HvRqej6B3YEd43/vTLTszYFpv5+W/mrqZ268fxgwvSHXVU9NQvEK4H2A\nhVXbPPqp/wXol7VtM3A+8CjRk0+Xu0bhNztJ8xm3fQS4AzjJzFab2WHFjFWSSZpTMzsSOA34jpk9\nEz8KfGCx45W6NeB3tCvwuJk9AzxG9EfzH8WMVerXkM/cpghu7acC6kDUC7O2xva1RKO0v+Du9wP3\nFykuaZyG5PP4YgUlTZIop+7+BPrsKwVJ8/kU0a0Mad4Sf+ZWcfdbG3oR9dSIiIhIEFTUJLcO2Ey0\n8ne2TsDbxQ9Hmkj5DI9yGhblMyxFyaeKmoTcfRPRzKPHVm2LZ7I8FngyrbikcZTP8CinYVE+w1Ks\nfOq+chYzawfsx5fPxf+HmR0M/Nvd3wCuAOaYWTmwHBgPtAXmpBCu1EP5DI9yGhblMyzNIp9pP/bV\nnF5E6zVtIeoiy37dnNXmHKAS+BhYAnwz7bj1Uj5byks5DeulfIb1ag751IKWIiIiEgSNqREREZEg\nqKgRERGRIKioERERkSCoqBEREZEgqKgRERGRIKioERERkSCoqBEREZEgqKgRERGRIKioERERkSCo\nqBEREZEgqKgRERGRIKioERERkSCoqBGRojCzjJktNrNnzexTM9tiZuvNbJcc7TuZ2RNm9mHcdouZ\nbTCzcjM7vtjxi0jzp1W6RaSozMyAd4Gd400T3P2KOtrvCrwE3A+MdvePCx+liJQiFTUiUlRmdjCw\nCPgM2B14xd2/Ws8xrwCHuPuGIoQoIiVKt59EpNj6A48CN8bf72tmg3I1NrNuwBoVNCJSHxU1IlJs\nA4C/AtcDm+JtY+pofzTwWKGDEpHSp6JGRIqtP/BXd38bmAsYcLKZ7Z2j/QBU1IhIAipqRKRozKwH\n0Vi+F+JNV8dftwFG5TjsCODJQscmIqVPRY2IFNMAovE0ALj708ASot6aH5vZdtmNzawL8J6eeBKR\nJFTUiEgxDSB68inbVfHXjsBptbRv8q0nM2vV1HOISPOnokZEiqlqkHC2PwFvxP8+p8a+o4G/1XdS\nM+trZrPiyfoWmNlcM5tsZjuYWVfglhrttzezB83slXhSvzXx9w+a2SIze8nM7jSz/c3su2b2VzP7\nKG77QnyNPbLOd7uZfRDvf83Mpjb4JyMiTaZ5akSkKMxsL2CJu281INjMJgJTAQe+4e4r4u3PAUfm\nepzbzHYDZgK9gInu/kDWvj7AL4BDgDJ3v6SW44cCfwROdfd5WdtbAWXAMXE8q81sKdDZ3bvliOU7\nwPfc/Yf1/jBEpCDUUyMixVLbracqvwOqxs2cA2BmHYBP6ihoegLLgHZAn+yCBsDdy4HlwP51XPeo\n+OviGsduIurd2YVorE9boHcd5wHoHr8PEUmJihoRKZacRY27vwf8L9GA4bPMbMe4/eO1tTezjsDD\nwHrg++7+SY5rPgh8RFT81OYoohmN19Wyb8f4a3uiJ7C2o+7xPYfVcR0RKQIVNSJSLP2pu6ej6vHu\ndsCPqHuQ8E3A3sBP63kyaj3whLtvrrnDzHYium2V63HxgUS3w56MY/Fc8cTrWW3v7p/VEYuIFJiK\nGhEpuHjsyw7uXpmrjbuvBB6Jvx1N1IuyVU+NmR0JnALMjx8Jr8uHwOQc+44g+gx8opZrdACGAovc\n/S6iAmeNu7+W41wHASvqiUVECkxFjYgUQ7X5aepwFdEtqK8B28W3pWoaTtRrckst+6px94/dfXmO\n3f35sifmC/EEgfcAC4DBZtYa6Evdt54SPaUlIoW1Xf1NRESa7DjqvvUEgLs/ZGYvAT3IXSQcHX99\ntIkx9Sdae+rn0d0jHNieaAzO+e6+FMDMBgCtqbuoOQINEhZJnYoaESkoMzsBGAY8m/CQa+NXriJi\nT2B9bYN7zex44AJgd6ANsAV4DRgcP9FU1W57ot6XJ919dD3x1DmeJtZGsx6LpE+3n0SkIMzsPjOr\nJHoCqQ1wvZmtMLOx9Rw6B3ib3D0164GNte1w90fc/TjgfGBf4HZ3PzG7oIn1Jep9SXLL6CDgXXd/\nubadZtYPqEhwHhEpMPXUiEhBuPvgRh73EdCljibLgJPMrF3ctjZHEPWuPJRjf9V4miRLMLQFai1o\nYuPjl4ikTD01IlJqriT67PpJbTvjx6uHEI2NeSrHOarG0yxJcL1ngA45rjUOWODuaxKcR0QKTEWN\niJQUd38UuBi4xMyGm9kXn2Pxo9g3AK8QjZepbX6aVkQ9ORV1TNqX7TqgnZmdlXWOHcxsMrC7u89u\n0hsSkbzR2k8iUpLMbCDRbZ+uwFrgfeBfwDXAauBQd388q31XYHbcfl9gA/A0cF32uk85rrUPMJ2o\nx+ZT4DPgBnefn993JSJNoaJGREREgqDbTyIiIhIEFTUiIiISBBU1IiIiEgQVNSIiIhIEFTUiIiIS\nBBU1IiIiEgQVNSIiIhIEFTUiIiISBBU1IiIiEgQVNSIiIhIEFTUiIiISBBU1IiIiEoT/BwROOZi3\nfm4AAAAAAElFTkSuQmCC\n", "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -438,7 +464,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 81, "metadata": { "collapsed": false }, @@ -446,18 +472,18 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 27, + "execution_count": 81, "metadata": {}, "output_type": "execute_result" }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGNCAYAAAA7NIpFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3XmcFPW1///XGUBgEHEZAUEQEANoYhQ0kbgvQeVqS26S\nS0yMCiT5akQTbsQkvyygkZsLRo0B1xtUvDEjMTEEvCbigkYMapzRuIFxZVOJuCA6Isic3x/VzfTs\nNdXV0zU17+fjUY/urvpU1WmOM3Os+nw+Ze6OiIiISJKUlToAERERkYZUoIiIiEjiqEARERGRxFGB\nIiIiIomjAkVEREQSRwWKiIiIJI4KFBEREUkcFSgiIiKSOCpQREREJHFUoIiIiEjiJLpAMbMjzWyx\nma03s1ozy4TY5xgzqzKzLWb2TzM7qz1iFRERkfgkukABegFPAt8GWn1okJkNAe4E7gM+DVwF/NrM\nPl+8EEVERCRu1lEeFmhmtcAEd1/cQpvZwMnufmDeukqgj7uPb4cwRUREJAZJv4LSVocB9zZYdzcw\ntgSxiIiISERpK1D6AxsarNsA7GJm3UsQj4iIiETQtdQBlJqZ7QGcCLwKbCltNCIiIh1KD2AIcLe7\nvxXngdNWoLwB9Guwrh/wnrt/1Mw+JwK3FjUqERGRdPsa8Ns4D5i2AmUFcHKDdeOy65vzKsBvfvMb\nRo0aVaSwwps2bRpXXnllIo7Xln3DtG2tTUvbm9vW1Pq4/w0L0VQs55xyCte+/jrWRHsHzt1rL667\n887QxysklkLaKp+lPWbc+WytnfJZ3GO2db9i/oy2Zf3KlSs544wzIPu3NE6JLlDMrBcwHHb8Lh9m\nZp8G3nb3tWb2c2CAu+fmOrkOOC87mudG4HjgS0BLI3i2AIwaNYrRo0cX42u0SZ8+fWKNo5DjtWXf\nMG1ba9PS9ua2NbU+7n/DQjQVy8lf+hJvXn01J9XWNmr/57Iyxn/5y23+d4gaSyFtlc/SHjPufLbW\nTvks7jHbul8xf0bbuj4r9i4SSe8kewjwBFBF8D+XlwPVwMXZ7f2BQbnG7v4q8G/ACQTzp0wDprh7\nw5E9iXX66acn5nht2TdM29batLS9uW1NrX/jjTdajaW9NBXfhbNmccWoUfy5rGzH5D5OUJxcOWoU\n37v00jYdr5BYCmmrfJb2mHHns7V2ymdxj9nW/Yr5M9rW9cXSYeZBKRYzGw1UVVVVlbSq37oVJk6E\n4cNhv/3qXgcOhLKkl5EJM3DgQNavX1/qMFq0efNmLv/xj3l48WLKt22jpls3Ds9k+N6ll9K7d+9S\nh5coHSGfEp7ymS7V1dWMGTMGYIy7V8d57ETf4ulM3nsPPvwQ7rgDXn0Vclf/u3eHffetK1ryC5hB\ng1S8NCX7w5JovXv3ZuZVV8FVV+HumDXVI0WgY+RTwlM+JSwVKAlRUQF/+UvwfuvWoEh54QV48cW6\nZdGiYP327UG77t1h2LD6RUtuGTQIunbS7Lb3ZchCqThpWUfLp7RM+ZSwdIsnIbd4wtq2DVavDoqX\nF16Al16qK2ReeQU+/jho160bDBnSdPEyZEiwXUREpBC6xSM7dOtWV2ic3GBA9ccfw5o1dVdccgXM\nPffAddcFV2YAunSBwYPrjrPvvnWvw4ZBeXn7fy8REZF8KlBSpGvXoMAYNgzGjau/bft2WLcuKFhe\nfLHuysvDD8OCBVBTU9d24MD6RUv++113bd/vFMWkSZO46aabSh2GxET5TBflU8JSgdJJdOkC++wT\nLMcdV3+bO2zYUFe45F6fegr++Ed45526tnvsUVe0NCxi+veHJHSnGNewOpMOTflMF+VTwlIflA7W\nB6UU3nmnfuGS//611+ralZcHV29yt4ryCxn1exERSR/1QZGS2m03OOSQYGmopibonJsrXHLLkiXB\niKNcp92ysqDfS65gyS9ghg2DPn3a9SuJiEjCqUCRgpSXwwEHBEtDH39c1+8lf3nsMaishM2b69ru\nsUfjoiX3qsnqREQ6HxUoUjRduwa3doYMgeOPr7/NHd56KyhYXn65rnh5+WVYvhzWrw/aAOy0Ewwd\nWr9oyb0fOhR69ap/7OXLl3PEEUe0x1eUdqB8povyKWGpQJGSMAsmp6uogM9+tvH2LVuCW0S54iX3\numwZzJ8fzLqb069fXdEybBgsWTKHq646gmHDYMAAXX3p6ObMmaM/aCmifEpY6iSrTrIdTm7U0Usv\nBf1fXn65fiHz2ms1QDCZS/fuwRWc/AImdzVm2DDQY2+Sr6amhnJNzpMayme6qJOsSB6zYEhz//5w\n+OGNt3/4YTmvvlpXwOSKmAcfhJtvhg8+qGtbUVFXsDR8HTRII4+SQH/M0kX5lLBUoEjq9OwJo0YF\nS0Pu8OabdVddcsXLK6/AI4/A2rV1D2rs0iUoUvILl/z3ffsmY94XEZE0UoEinYpZUFj07QuHHdZ4\n+7ZtweMCckVLroB56qngYY1vvVXXtrw8uH2UK1waFjK77NJuX0tEJHVUoEjqTJ8+ncsuuyzSvt26\n1Q11bsrmzfULl9z7ZcvgxhvrPzJg992DQqVhEZNb16NHpBA7nULyKcmjfEpYKlAkdQYPHly0Y/fu\nDQceGCwN5W4fNVXA3HFHcGUmN3EdBH1o8guW/NfBg9X/JaeY+ZT2p3xKWBrFo1E80k4+/jiY3+XV\nV+sKl/z3+XO/lJUFE9Q1vAKTm1dm772DPjIiIqWkUTwiKdC1a90DG48+uvH2rVuDqywNC5h//hOW\nLoU33qh/rEGD6l95yS1Dh8Jee6mAEZGOTQWKSELstFPwdOjhw5ve/uGHsHp1/QJm9Wp45hm48074\n17/q2nbrFtwm2mef+sVLbhkwQAWMiCSbChRJnVWrVjFy5MhShxG7nj1h5MhgacoHH9QVMLkiZvVq\nePrp4OGNb75Z17Zr16CAyS9a8ouZAQOCNkmQ1nx2VsqnhJWQX0Ei8bnoootYvHhxqcNod716wf77\nB0tTmitgmroCk5sDJle0NHzde+/gik976Kz5TCvlU8JSgSKpM2/evFKHkEitFTA1NXV9YHLL6tXw\n/PNBH5jXX69raxZcZcm/+pJbhgwJrs707BlP3MpnuiifEpYKFEkdDWOMpry85VtIW7YEM+3mCpfc\n6+rV8NBDsG5d3Sy8EEyGl1+4NFx23TVcXMpnuiifEpYKFBEJpUcP2G+/YGnKtm3BUOlc0ZJfwDz5\nZHB1ZuvWuva77FL/qss++9R17B08OHhKtZ5ELdJ5qUARkVh061Z3y6cptbXBU6hzRUt+IbNsWfD+\n/ffr2u+0U/2CJVfM5N4PGtR+/WBEpP2pQJHUmT17Nt///vdLHYY0UFYWzM+y115NPwfJHd59NyhU\n1qypK2CWLp3N5s3f5667ggInJ/dU64ZXXnKvgwcHt5H0QMdk0c+nhKUCRVKnJv+BONJhmMFuuwXL\nQQfVrZ8xo4aLLw7e5/rB5IqXNWvqipnHHw+2bdtWt2/v3nXFSn7hklsGDNAjBdqbfj4lLE11r6nu\nRVIjdxspV7TkFzC592+/Xde+rCwoUhoWLrlbSLoKI9IyTXUvIhJC/m2kz3626Tbvvx8UKmvX1i9g\n1q6Fxx5rfBVm553rFy+5wiX3fu+9oXv39vl+Ip2JChQR6VR23rnl+WBqa4PnHjVVwDz+OPzxj/Vn\n5YWgL0yucGn4OnhwMORaI5JE2kYFiqTOxo0bqaioKHUYEpP2zmfuts+AAc1fhampCeZ9yRUw+Vdk\nnnkmeP3ww7r2O+0UXGkZNKhuyRUxufd9+nSOW0n6+ZSwVKBI6kyePFlTaadIEvNZXg6f+ESwNMUd\n3norKFpyhUvu/auvwl//GswZs3173T4779y4cGm4lJe3y9crqiTmU5JJBYqkzsyZM0sdgsSoI+bT\nDCoqguXgg5tus3178PiAXOGSX8g88QQsXlz/+UgAu+/efPEyaBAMHJj8/jAdMZ9SGipQJHU0Gitd\n0prPLl2C2z577w1jxzbd5qOPgltJ+UVMbnn44eD1nXfq79O3b13n3YYFzN57B0VMKYdWpzWfEj8V\nKCIiCdW9O+y7b7A054MPgkKlYSGzbl0wQ++6dbBpU117s+AxAg2LmNz7vffW/DCSDCpQREQ6sF69\nWn7II8B77wWFSn4Rk/t8773B582b69o3LGLyi5fc+wED9KgBKS4VKJI68+fPZ8qUKaUOQ2KifBZu\nl11aHloNQRGTfyUmv6C5996gU+9779Xfp1+/uqIlf8m/ndSjR/19lE8JSwWKpE51dbV+AaaI8tk+\ndtkFDjggWJrz3ntBoZJfwOSKmAcfDN6/+279fSoq6gqXgQPhiSeq6dJlSr2CZuedi/vdpGPSVPea\n6l5EJDbvvx8UKuvX172uXVv3ed26xhPd7bJL/SKmqdc99ugc88R0NJrqXkREOoSdd269T8xHH8Fr\nr9W/ArN+fbA89xzcfXcwm29tbd0+3bs3Xbzklr33Dmb0Vefe9FCBIiIi7ap7dxg6NFia8/HHwYMf\nc0VMroDJvf/734P3W7bU7ZPr3JtfvDQsZAYODJ5yLcmnAkVERBKna9e6gqK5Rw64B0+nzi9g8ouY\nhx4KXvOfYA1BgdLw6kvDIqZv32CuGikdFSiSOplMRlNpp4jymS5x5tMs6Juyxx7w6U83366mJril\nlF/E5JYXXgjmi3n99eCqTU6XLsFTsQcMaFy8DBxYt15XY4pHBYqkztSpU0sdgsRI+UyXUuSzvByG\nDw+W5tTWBo8WyC9e8ouaZcuC14ajlPKvxuSKloav6hsTjUbxaBSPiIiElH81Jr+Aafh+69a6fcyC\nW0bNFTG59x1xpJJG8YiIiCRAmKsxuadZ5xcur71W9/6xx4JbShs2BG1zdtopuK2UX7g0XHK3lTpa\nIROFChQREZEY5T/NuqW+Mdu2BUVKfgGTX9CsXNn0baVeveDWW+G004r7PUpNBYqkzqJFi5gwYUKp\nw5CYKJ/ponzW6datboK6ltTUBFdc8ouYlmb8TYuyUgcgErfKyspShyAxUj7TRflsu/Ly4InWRx4J\nEyfCtGkt32JKCxUokjoLFy4sdQgSI+UzXZRPCSvxBYqZnWdmr5jZh2b2iJkd2kr7r5nZk2b2gZm9\nZmbzzWz39opXRERECpfoAsXMJgKXAzOAg4F/AHebWUUz7Q8HFgD/A+wPfAn4DHBDuwQsIiIisUh0\ngQJMA65391vcfRVwDlADTG6m/WHAK+5+tbuvdve/AdcTFCkiIiLSQSS2QDGzbsAY4L7cOg9mlbsX\nGNvMbiuAQWZ2cvYY/YAvA/9X3GglSSZNmlTqECRGyme6KJ8SVmILFKAC6AJsaLB+A9C/qR2yV0zO\nABaa2VbgdeAdQHNldyLjxo0rdQgSI+UzXZRPCSvJBUqbmdn+wFXATGA0cCIwlOA2j3QSp59+eqlD\nkBgpn+mifEpYSS5QNgLbgX4N1vcD3mhmnx8AD7v7Fe7+jLvfA3wbmJy93dOs8ePHk8lk6i1jx45l\n0aJF9dotXbqUTCbTaP/zzjuP+fPn11tXXV1NJpNh48aN9dbPmDGD2bNn11u3Zs0aMpkMq1atqrd+\n7ty5TJ8+vd66mpoaMpkMy5cvr7e+srKyycunEydO1PfQ99D30PfQ99D3KOh7VFZW7vjb2L9/fzKZ\nDNOmTWu0T1wS/bBAM3sEeNTdv5P9bMAa4FfuflkT7X8PbHX3r+atGwssBwa6e6PCRg8LFBERiaaY\nDwuMdAXFzF4ysx+YWZN9QWJ0BfBNMzvTzEYC1wHlwM3ZOH5uZgvy2i8Bvmhm55jZ0Oyw46sIipzm\nrrpIyjT8PwPp2JTPdFE+Jayot3iGALOANWZ2h5mdlL26ESt3/x1wIXAJ8ARwIHCiu7+ZbdIfGJTX\nfgHwn8B5wNPAQmAl8MW4Y5PkmjNnTqlDkBgpn+mifEpYkW7xmNnewBSC+UgGAQ6sA+YDN7r7ujiD\nLCbd4kmfmpoaysvLSx2GxET5TBflM10Sd4vH3de5+8UEV1LGA4sIrmbMBF4xsyVmljGzJHfClZTS\nL790UT7TRfmUsAoqIDzwF3f/IsGVlB8ArwD/BvwRWGtmPzOzIYUGKiIiIp1HbFc43P1f7j4HGAVc\nCRiwF/Aj4EUz+5OZHRTX+URERCS9YitQzGwfM7sEeBX4bnb134BLgReBU4HHzOwLcZ1TpCkNx/1L\nx6Z8povyKWEVVKCYWVcz+6KZ/QV4Cfgx0Bu4FjjQ3Y9w95+6+0hgIsHEa5cUGrRISwYPHlzqECRG\nyme6KJ8SVtRRPPsB3wDOAvYkuJ1TTTBPyW/dvaaZ/W4HMu7ePXLEMdMoHhERkWiKOYqna8T9nicY\nWvwhcBNwnbs/HmK/TUC3iOcUERGRTiLqLZ7ngO8AA9z9GyGLE7JtNfRYREREWhR1HpRPuvs8d38v\n7oBECtXwgVnSsSmf6aJ8SlhRn8VTZma7mFmzt2vMrFu2ja6YSLu66KKLSh2CxEj5TBflU8KKWjxM\nA94Bjm6hzdHZNudHPIdIJPPmzSt1CBIj5TNdlE8JK2qB8gVgrbvf21yD7LZ16EF90s40jDFdlM90\nUT4lrKgFyn7AsyHaPZNtKyIiIhJa1AKlD8GQ4dZsAnaLeA4RERHppKIWKK8DB4ZodyDwr4jnEIlk\n9uzZpQ5BYqR8povyKWFFLVDuB0aZ2cTmGpjZfwD7A8sinkMkkpqaJicylg5K+UwX5VPCijrV/Ujg\nCYIC53+AGwiexQOwL/At4JsEs80e4u7PxBJtEWiqexERkWgSN9W9u68yszOBBcC52SWfAVuASUku\nTkRERCSZIk+i5u63E/QxuR54Efgou7xI8DTjT7v7wjiCFBERkc4l6sMCAXD3F4FvxxSLSCw2btxI\nRUVFqcOQmCif6aJ8Sliahl5SZ/LkyaUOQWKkfKaL8ilhFXQFBcDMugJ7AN2ba+Puawo9j0hYM2fO\nLHUIEiPlM12UTwkrcoFiZicAPwYOA5p9aCDBSJ6CCyGRsDQaK12Uz3RRPiWsSIWDmZ0C/BHoQvBA\nwFeAzTHGJSIiIp1Y1CsbMwj6r0wD5rn79vhCEhERkc4uaifZA4AV7n6VihNJmvnz55c6BImR8pku\nyqeEFbVAeR9Qx1dJpOrqWCczlBJTPtNF+ZSwok51/1uCKew/EX9I7UtT3YuIiERTzKnuo15B+T6w\ni5nNzg4zFhEREYlN1OJiEvBn4ELgi2b2ALAOqG2irbv7zyKeR0RERDqhqAXKTIL5TQwYll2a44AK\nFBEREQmtkCsoIomUyWRYvHhxqcOQmCif6aJ8SliRChR3XxB3ICJxmTp1aqlDkBgpn+mifEpYelig\npM64ceNKHYLESPlMF+VTwipoBE52BM+/AZ8BKoBH3f3G7LYB2XXPufvHhQYqIiIinUchDws8AvgN\nMIigs6wTPDTwxmyTscDvgC8DdxQWpoiIiHQmkW7xmNn+wF+AvYC5wH8QFCn5lgA1wBcLCVCkrRYt\nWlTqECRGyme6KJ8SVtQ+KD8BegCnuvt33f33DRu4+1agGji4gPhE2qyysrLUIUiMlM90UT4lrKgF\nyrHAY+6+tJV264EBEc8hEsnChQtLHYLESPlMF+VTwopaoOwKrA3RrhdBvxQRERGR0KIWKP8Chodo\nN4pwhYyIiIjIDlELlPuBg8zs2OYamNkXCIqYeyKeQ0RERDqpqAXKfwNbgUVmdq6Z9c9tMLPdzGwy\nMB/4ALii8DBFwps0SU9iSBPlM12UTwkrUoHi7quA07P7zyPoDOvAWcBG4H+A7sDX3P2VeEIVCUcz\nVaaL8pkuyqeEZe4efWezfYBpwOeBIQQFyzqC2zqXu/tLMcRYVGY2Gqiqqqpi9OjRpQ5HRESkw6iu\nrmbMmDEAY9y9Os5jFzTVvbuvBr4bUywiIiIigB4WKCIiIgmkAkVSZ/ny5aUOQWKkfKaL8ilhhSpQ\nzOxlM3vJzIbmfQ67JL4fiqTLnDlzSh2CxEj5TBflU8IK2wdlSPa1W4PPIolz2223lToEiZHymS7K\np4QVqkBx97KWPoskSXl5ealDkBgpn+mifEpYiS80zOw8M3vFzD40s0fM7NBW2u9kZrPM7FUz25K9\nzXR2O4UrIiIiMShomHGxmdlE4HLgW8BjBHOu3G1mn3D3jc3sdjuwJzAJeAnYiw5QiImIiEidSH+4\nzew4M7vDzI5soc1R2TZHRQ+PacD17n5Ldvbac4AaYHIz5zwJOBIY7+7L3H2Nuz/q7isKiEE6mOnT\np5c6BImR8pkuyqeEFfXKwv8jmD32yRbaPAmMIygq2szMugFjgPty6zyY9vZeYGwzu50KPA5838zW\nmdnzZnaZmfWIEoN0TIMHDy51CBIj5TNdlE8JK+otns8AT7j75uYauPt7ZlYNfDbiOSqALsCGBus3\nACOa2WcYwRWULcCE7DGuBXYHpkSMQzqY888/v9QhSIyUz3RRPiWsqFdQ+gNrQ7RbS9AHpL2UAbXA\nV939cXf/C/CfwFlm1r0d4xAREZECRC1QPgD6hWjXl+BqRhQbge1NnKcf8EYz+7wOrHf39/PWrQQM\n2Lulk40fP55MJlNvGTt2LIsWLarXbunSpWQymUb7n3feecyfP7/euurqajKZDBs31u/PO2PGDGbP\nnl1v3Zo1a8hkMqxatare+rlz5za6Z1tTU0Mmk2k0I2NlZWWTjzKfOHGivoe+h76Hvoe+h75HQd+j\nsrJyx9/G/v37k8lkmDZtWqN94hLpacZmdg9wBDDC3dc002Yw8E9ghbsfGyk4s0eAR939O9nPBqwB\nfuXulzXR/pvAlUBfd6/JrjsN+D2ws7t/1MQ+eppxyqxatYqRI0eWOgyJifKZLspnuhTzacZRr6Dc\nCHQH7jSzQxpuzK5bQjDz7I3Rw+MK4JtmdqaZjQSuA8qBm7Pn+bmZLchr/1vgLeAmMxuVHUE0B5jf\nVHEi6XTRRReVOgSJkfKZLsqnhBWpk6y7V5rZF4AvAY+a2T8I5hwB2Bf4NMFtlT+6+/9GDc7df2dm\nFcAlBLd2ngROdPc3s036A4Py2n9gZp8H5gJ/JyhWFgI/iRqDdDzz5s0rdQgSI+UzXZRPCSvSLR4A\nMysD/j+CTqi7Ntj8LsGtlv9y9+0FRVhkusUjIiISTTFv8USeSdbda4FLzWw2cAh1VzLWAlXuvjWG\n+ERERKQTKniqe3ffBqzILiIiIiIF0zNqJHUaDsGTjk35TBflU8IKdQXFzH4KOHC1u7+d/RyWu/vP\nIkUnEkFNTU2pQ5AYKZ/ponxKWKE6yZpZLUGBMsrd/5n32UKcw929S2FhFo86yYqIiESThE6ykwkK\nkteznxtPNyciIiISk1AFirvf3ODzgmaaioiIiBQsVCdZM9tuZvPzPv/UzBo/UEAkARo+u0I6NuUz\nXZRPCSvsKB6jfn+TmcCE2KMRicHkyZNLHYLESPlMF+VTwgpboLxP8GRikcSbOXNmqUOQGCmf6aJ8\nSlhhO8k+BZxgZjOAV7LrhpvZmWF2dvdbogQnEoVGY6WL8pkuyqeEFbZAuRi4A5hBMJoH4PDs0hLL\ntleBIiIiIqGFHcVzj5ntD5xA8MydmcA/gD8VLzQRERHprEI/i8fd1wI3AZjZTOBJd7+4SHGJRDZ/\n/nymTJlS6jAkJspnuiifElbYYcY3mll+1+tJwK+LE5JIYaqrY53MUEpM+UwX5VPCastU9ze7++Ts\n5+3Zzx2+DNZU9yIiItEUc6r7sMOMtwE98j43nBdFREREJDZhC5S1wJFmtk8xgxERERGB8AXKb4GB\nwMvZ2zsAZ2WnwG9t+bg4oYuIiEhahS1QZgIXAsuBV7PraoA1IZa1sUUrEkImo8dEpYnymS7Kp4QV\ndh6UWuCK7JLrNHt7rtOsSJJMnTq11CFIjJTPdFE+JaywV1AaWkBwNUUkccaNG1fqECRGyme6KJ8S\nVuiJ2vK5+6S4AxERERHJCTtR25lm9rlmtu1iZj2a2Xa6mV1RSIAiIiLS+YS9xXMz8I1mtr0DXN3M\ntnHAd9oYk0hBFi1aVOoQJEbKZ7oonxJW1D4o+TRpmyRKZWVlqUOQGCmf6aJ8SlhxFCgiibJw4cJS\nhyAxUj7TRfmUsFSgiIiISOKoQBEREZHEUYEiIiIiiaMCRVJn0iRN05Mmyme6KJ8SVlsmajvJzO5v\n47aREWISKYhmqkwX5TNdlE8Jy9y99UbBs3eicnfvUsD+RWVmo4GqqqoqRo8eXepwREREOozq6mrG\njBkDMMbdq+M8dtgrKMfGeVIRERGRloR9mvGDxQ5EREREJEedZCV1li/Xg7bTRPlMF+VTwlKBIqkz\nZ86cUocgMVI+00X5lLBUoEjq3HbbbaUOQWKkfKaL8ilhqUCR1CkvLy91CBIj5TNdlE8JSwWKiIiI\nJI4KFBEREUkcFSiSOtOnTy91CBIj5TNdlE8JSwWKpM7gwYNLHYLESPlMF+VTwgo71X1B/0W5+5pC\n9i8mTXUvIiISTRKmun8VaL2SaZq34TwiIiIioQuHvxK9QBERERFpk7DP4jmmyHGIxGbVqlWMHDmy\n1GFITJTPdFE+JSx1kpXUueiii0odgsRI+UwX5VPCiqVviJl1B3YHPnL3t+M4pkhU8+bNK3UIEiPl\nM12UTwmroCsoZvYtM3sC+ABYB/wib9u/m9kdZja8wBhF2kTDGNNF+UwX5VPCilSgmFkXM/sjcC0w\nClgJWINm/wAmABMLilBEREQ6nahXUKYCpwF/BvZx9081bODuLwEvAidHDw/M7Dwze8XMPjSzR8zs\n0JD7HW5m28ws1nHZIiIiUnxRC5SzgQ3ARHff0EK754B9Ip4DM5sIXA7MAA4muCpzt5lVtLJfH2AB\ncG/Uc0vHNXv27FKHIDFSPtNF+ZSwohYoI4BH3f2DVtp9AOwZ8RwA04Dr3f0Wd18FnAPUAJNb2e86\n4FbgkQLOLR1UTU1NqUOQGCmf6aJ8SlhRC5RtQI8Q7QYDm6OcwMy6AWOA+3LrPJiX/15gbAv7TQKG\nAhdHOa90fBdfrNSnifKZLsqnhBW1QHkWGGNmvZtrYGZ9gYOAJyOeowLoQnArKd8GoH8z59wP+C/g\na+5eG/Hg1Bz6AAAgAElEQVS8IiIiUmJRC5T/BfYArjOznRpuNLMuwNVAOUFfkKIzszKC2zozsh10\nofHIIhEREekAohYoNwAPAKcDz5vZddn1nzazq4B/Al8E7iEoGqLYCGwH+jVY3w94o4n2vYFDgHnZ\n0TvbgJ8AB5nZVjM7pqWTjR8/nkwmU28ZO3YsixYtqtdu6dKlZDKZRvufd955zJ8/v9666upqMpkM\nGzdurLd+xowZjTqKrVmzhkwmw6pVq+qtnzt3LtOnT6+3rqamhkwmw/Lly+utr6ysZNKkSY1imzhx\nYqf6Hhs3bkzF94B05KPQ77Fx48ZUfA9IRz4K/R6543f075HTmb5HZWXljr+N/fv3J5PJMG3atEb7\nxMWCbh0RdjTrQTDC5htAtwabtwM3At9x9y2RgzN7hKAz7neynw1YA/zK3S9r0NYI5mTJdx5wLEGx\n9Kq7f9jEOUYDVVVVVYwePTpqqJIgmUyGxYsXlzoMiYnymS7KZ7pUV1czZswYgDHuHuu0HpGnus8W\nHueZ2UzgGGAIwRWZdcAyd38thviuAG42syrgMYJRPeXAzQBm9nNggLufle1A+1z+zmb2L2CLu6+M\nIRbpIGbOnFnqECRGyme6KJ8SVsHP4nH3N4HbY4ilqWP/LjvnySUEt3aeBE7MnhOCzrKDinFu6bh0\nJSxdlM90UT4lrFgeFlhM7n4NcE0z2xrfMKu//WI03FhERKTDCVWgmNmZhZzE3W8pZH8RERHpXMJe\nQbkZiNKb1rL7qUCRdjN//nymTJlS6jAkJspnuiifElbYAuUSohUoIu2uurpavwBTRPlMF+VTwoo8\nzDgtNMxYREQkmmIOM446UZuIiIhI0ahAERERkcQJO4rnpwWcw939ZwXsLyIiIp1M2E6yMwk6yUZ5\n+J4DKlCk3Wgq7XRRPtNF+ZSwwhYoLU6IJpIkU6dOLXUIEiPlM12UTwkrVIHi7guKHYhIXMaNG1fq\nECRGyme6KJ8SljrJioiISOKoQBEREZHEKahAMbMjzOwyM1tkZveZ2f1NLPfFFaxIGIsWLSp1CBIj\n5TNdlE8JK1KBYoEbgQeB7wEZ4JgGy9F570XaTWVlZalDkBgpn+mifEpYUa+gnAOcDVQBnwfuyK4f\nAZxM8HDBWuAyYFhBEYq00cKFC0sdgsRI+UwX5VPCCjvMuKGzgQ+Ak939LTM7A8DdXwBeAO42s7uA\nhcDfgNUxxCoiIiKdRNQrKKOAv7n7W9nPDmBmXXIN3P33BFdYLiwoQhEREel0ohYoZcBbeZ9rsq+7\nNWj3AvCpiOcQERGRTipqgbIeGJD3OXcL5+AG7T4BfBzxHCKRTJqkiY/TRPlMF+VTwopaoFQD++fd\n0llK8JyeOWY20sx6m9l0YAzwRAxxioSmmSrTRflMF+VTwjJ3b/tOZqcDtwIT3H1xdt1vga+Q7Y+S\ntR04wt0fiyHWojCz0UBVVVUVo0ePLnU4IiIiHUZ1dTVjxowBGOPu1XEeO9IoHnevNLM7qH/75izg\nKWACQV+UfwJzklyciIiISDJFHWaMu3/U4PM24L+zi4iIiEhkUWeSrTaz2+MORiQOy5cvL3UIEiPl\nM12UTwkraifZEcC2OAMRicucOXNKHYLESPlMF+VTwopaoLwA7BFnICJxue2220odgsRI+UwX5VPC\nilqgzAeONrORcQYjEofy8vJShyAxUj7TRfmUsCIVKO4+l+CBgA+a2TQzG25mO8UamYiIiHRakUbx\nmNn23FvgF9kFM2uqubt75NFCIiIi0vlEvcWzFlhDMMX9mlaWtYWHKRLe9OnTSx2CxEj5TBflU8KK\nOlHbkJjjEInN4MGDSx2CxEj5TBflU8KKNNV9mmiqexERkWiKOdV91Fs8IiIiIkVTcOdVMzsA2A/o\nTdBpthF3v6XQ84iIiEjnEblAMbMTgGuAfVtqRvB0YxUo0m5WrVrFyJGaoictlM90UT4lrKjP4jkE\n+D9gMPBb4Onspv8GbgfeyX6+CbikwBhF2uSiiy4qdQgSI+UzXZRPCSvqFZQfZvc9yd3vMbObgE+5\n+48AzGxX4HrgFOCQWCIVCWnevHmlDkFipHymi/IpYUXtJPs54Al3v6epje7+LnAmUAtcGvEcIpFo\nGGO6KJ/ponxKWFELlN0JHhiYsxXAzHrlVrj7R8BDwOcjRyciIiKdUtQC5U1glwafAYY1aNcT6BPx\nHCIiItJJRS1QXgSG5n1+jGDEzv/LrTCz4cBxwMuRoxOJYPbs2aUOQWKkfKaL8ilhRS1Q7gJGmNmo\n7Oe/EDyX51wze9TM/gD8HegBzC88TJHwampqSh2CxEj5TBflU8KKNNW9mfUHTgOWu/uz2XWfAn4H\njMg2qyUoTs7xBM+nr6nuRUREoinmVPdRHxb4BsEw4vx1TwOjzGwksBvworu/2dT+IiIiIi0peKr7\nhtx9VdzHFBERkc6lTX1QzGy8md1gZn82s0VmdomZDW19T5H2s3HjxlKHIDFSPtNF+ZSwQhcoZnYr\nsASYApwIZIAfAc+aWaY44Ym03eTJk0sdgsRI+UwX5VPCCnWLx8ymAKcDHwP/CzxB8PTiU4CxwC1m\nto+7bypWoCJhzZw5s9QhSIyUz3RRPiWssH1QziIYlXOyu9+Xt/7n2efwnAn8O8HDAUVKSqOx0kX5\nTBflU8IKe4vnU8AjDYqTnP8imKTtU7FFJSIiIp1a2AJlF+ClZra9lNdGREREpGBhCxQDtje1wd1r\n23isNjGz88zsFTP70MweMbNDW2j7BTNbamb/MrNNZvY3MxtXjLgkuebP1+TFaaJ8povyKWHFPg9K\nnMxsInA58C2C5/1MA+42s0+4e1Nj1Y4ClgI/BN4FJgNLzOwz7v6PqHGsWbNGQ+M6kLvvvpuDDz54\nx+eKigo94r0Dq66uZsqUKaUOQ2KifEpYoaa6N7NaIOp09e7ukQohM3sEeNTdv5P9bMBa4FfuPifk\nMZ4BbnP3S5vZ3uJU92vWrGHUqFF6fkQHVl5ezsqVK1WkiIjELClT3VvEc0Taz8y6AWMIOuECQaVj\nZvcSDG0OcwwjGA79dpQYIJhUqKamht/85jeMGjWq9R0kUVauXMkZZ5zBxo0bVaCIiHQgoQoUdy9K\n/5JWVABdgA0N1m+g7oGErZkO9CJ4iGFBRo0apeFxIiIi7STRfVAKYWZfBX4CZJrpryIiIiIJVYor\nI2FtJBg51K/B+n7AGy3taGZfAW4Avuzuy8KcbPz48WQymXrL2LFjWbYs1O7SAVRXV5PJZBp1eJ4x\nYwazZ8+ut27NmjVkMhlWrar/7Mu5c+cyffr0eutqamrIZDIsX7683vrKykomTZrUKI6JEyeyaNGi\neuuWLl1KJtP4iRHnnXdeo1EPne17ZDKZVHwPSEc+Cv0euXg6+vfI6Uzfo7Kycsffxv79+5PJZJg2\nbVqjfeISqpNsqTTTSXYNQSfZy5rZ53Tg18BEd78zxDla7CSb6wDU3HZJNuWv41u6dCnjxmm2gLRQ\nPtMlKZ1kS+EK4GYzq6JumHE5cDOAmf0cGODuZ2U/fzW77QLg72aWu/ryobu/176hi0gc9McsXZRP\nCSvJt3hw998BFwKXEDyg8EDgRHd/M9ukPzAob5dvEnSsvRp4LW/5ZXvFnBYzZ86krCzR/3mIiEiK\nJf0KCu5+DXBNM9smNfh8bLsE1QmYGcEdtba59tprKS8v56yzzipCVCIi0lnof5ElVtdccw0LFiwo\ndRiSIg07GkrHpnxKWCpQiqCYHY+T3KlZpBgqKytLHYLESPmUsFSgxGTz5s3MuOACThg6lAmDBnHC\n0KHMuOACNm/enOhjAyxfvpxDDz2Unj17st9++3HDDTc0anPTTTdx/PHH069fP3r06MEBBxzAdddd\nV6/N0KFDefbZZ3nggQcoKyujrKyM4447DoB33nmHCy+8kAMPPJDevXvTp08fxo8fz1NPPRXLd5D0\nWrhwYalDkBgpnxJW4vugdASbN2/mi2PH8p8rVzKzthYjeHDR3VdfzRfvv58/rFhB7969E3dsgGee\neYYTTzyRvn37cskll7Bt2zZmzpxJ375967W77rrr+OQnP8lpp51G165dWbJkCd/+9rdxd84991wA\nrrrqKqZOnUrv3r358Y9/jLvTr18wkOrll19m8eLFfPnLX2bo0KFs2LCB66+/nmOOOYbnnnuO/v37\nR/4OIiKSQu7eqRdgNOBVVVXelKqqKm9pu7v7T88/3/9cVuYOjZa7ysp8xgUXNLtva4p5bHf3CRMm\neHl5ua9bt27HulWrVnnXrl29rKxsx7otW7Y02vekk07y4cOH11v3yU9+0o899thGbbdu3dpo3erV\nq71Hjx5+6aWXFvIVWhQmfyIiEk3udyww2mP++6xbPDF4eMkSTqytbXLbSbW1PLx4cSKPXVtby9Kl\nS/nCF77AwIEDd6wfMWIEJ554Yr223bt33/H+vffe46233uKoo47i5ZdfDnWrqVu3bvXO+/bbb1Ne\nXs6IESOoro51bh8REUkBFSgFcnd6bdvW7CObDSjfti1S59ZiHhvgzTff5MMPP2T48OGNto0YUf95\njA8//DAnnHACO++8M7vuuit77rknP/rRjwDYtGlTq+dyd6688ko+8YlP0L17dyoqKujbty9PP/10\nqP2l82pq+m3puJRPCUsFSoHMjA+6daO5EsGBD7p1izSnSDGP3RYvvfQSJ5xwAm+//TZXXnkld911\nF/fee++OZzDUNnOFJ9+sWbP43ve+xzHHHMOtt97K0qVLuffee9l///1D7S+dl2YeTRflU8JSJ9kY\nHH7qqdx99dWc1MQf2r+UlXFEEw+HSsKx99xzT3r27MkLL7zQaFv+Q6eWLFnC1q1bWbJkSb1bQffd\nd1+j/Zorlv7whz9w3HHHNRoh9O6777LnnntG/QrSCZx++umlDkFipHxKWLqCEoMLZ83iilGj+HNZ\n2Y6rHQ78uayMK0eN4nuXXprIY5eVlXHiiSeyaNEi1q1bt2P9ypUrWbp06Y7PXbsGdWz+lY5NmzZx\n8803Nzpmr169ePfddxut79KlS6NbUbfffjvr16+PHL+IiKSXCpQY9O7dmz+sWMGjU6cybsgQThs4\nkHFDhvDo1KkFDwMu5rEBLr74YtydI444gjlz5jBr1iyOO+44PvnJT+5oM27cOLp168Ypp5zCNddc\nw+zZsznkkEN2DCHON2bMGJ566ilmzZrFwoULWbZsGQCnnHIKDzzwAJMnT+bXv/413/nOdzj33HPZ\nd999C4pfRERSKu5hQR1tIYZhxg3V1taGbttWxTj2Qw895Iceeqj36NHDhw8f7jfccIPPnDmz3jDj\nO++80w866CAvLy/3YcOG+S9+8Qu/6aabvKyszFevXr2j3YYNG/zUU0/1Pn36eFlZ2Y4hxx999JFP\nnz7dBw4c6L169fKjjjrKH330UT/22GP9uOOOi/075WiYccf30EMPlToEiZHymS7FHGZsHnEESFqY\n2WigqqqqitGjRzfaXl1dzZgxY2huuySb8tfxZTIZFhcwnF6SRflMl9zvWGCMu8c6Z4Ru8YhIot12\n222lDkFipHxKWCpQRCTRysvLSx2CxEj5lLBUoIiIiEjiqEARERGRxFGBIiKJNn369FKHIDFSPiUs\nFSgikmiDBw8udQgSI+VTwlKBIiKJdv7555c6BImR8ilhqUARERGRxFGBIiIiIomjAkVEEi3/ydrS\n8SmfEpYKFBFJtIsuuqjUIUiMlE8JSwWKlMzq1aspKyvjlltuafO+Dz74IGVlZfz1r38tQmSSJPPm\nzSt1CBIj5VPCUoEiHZaZlToEaQcalpouyqeEpQJFREREEkcFioiIiCSOCpRObubMmZSVlfHCCy9w\nxhlnsOuuu9K3b19++tOfArB27VomTJhAnz592Guvvbjiiivq7f/mm28yZcoU+vfvT8+ePTnooIOa\n7FOyadMmzj77bHbddVd22203Jk2axLvvvttkTM8//zxf+tKX2GOPPejZsyeHHnooS5Ysif/LS4cw\ne/bsUocgMVI+JSwVKJ1crh/HxIkTgeCXx2GHHcasWbP45S9/ybhx49h7772ZM2cO++23H9OnT2f5\n8uUAbNmyhaOPPppbb72Vr3/96/ziF79g11135eyzz2bu3Ln1zpPJZLj11ls588wzmTVrFuvWreOs\ns85q1I/k2Wef5bDDDuP555/nhz/8IVdccQU777wzEyZM4E9/+lM7/ItI0tTU1JQ6BImR8imhuXun\nXoDRgFdVVXlTqqqqvKXtHd3MmTPdzPzcc8/dsW779u0+aNAg79Kli1922WU71r/77rteXl7ukyZN\ncnf3X/7yl15WVuaVlZU72nz88cf+uc99znfZZRd///333d190aJFbmZ++eWX72hXW1vrRx11lJeV\nlfmCBQt2rD/++OP9oIMO8m3bttWL8/DDD/cRI0bs+PzAAw94WVmZP/jggy1+v7TnT0SklHK/Y4HR\nHvPf564lrI1SqaYGij0P0ciRUF4e3/HMjClTpuz4XFZWxiGHHMKf/vQnJk+evGN9nz59GDFiBC+/\n/DIAd911F/379+crX/nKjjZdunThggsu4Ktf/SoPPvgg48eP56677qJbt26cc8459c55/vnn89BD\nD+1Y984777Bs2TJ+9rOfsWnTpnoxjhs3josvvpjXX3+dvfbaK74vLyIiiaQCJWarVsGYMcU9R1UV\njB4d7zEbDv3r06cPPXr0YPfdd2+0/u233wZgzZo17Lfffo2ONWrUKNyd1atX72i31157Ud6gqhox\nYkS9zy+++CLuzk9+8hN+/OMfNzqumfGvf/1LBYqISCegAiVmI0cGBUSxzxG3Ll26hFoH5G6Nxa62\nthaACy+8kBNPPLHJNsOHDy/KuSW5Nm7cSEVFRanDkJgonxKWCpSYlZfHf3UjqfbZZx+efvrpRutX\nrlwJwJAhQ3a0u//++6mpqal3FaXhMzmGDRsGQLdu3TjuuOOKFLV0NJMnT2bx4sWlDkNionxKWBrF\nI5GNHz+eN954g4ULF+5Yt337dubOnUvv3r056qijdrTbtm0b11577Y52tbW1zJ07t94onj333JNj\njjmG66+/njfeeKPR+TZu3FjEbyNJNXPmzFKHIDFSPiUsXUGRyL71rW9x/fXXc/bZZ/P4448zZMgQ\nbr/9dlasWMFVV11Fr169ADj11FM5/PDD+cEPfsArr7zC/vvvzx133MHmzZsbHfPqq6/myCOP5FOf\n+hTf/OY3GTZsGBs2bGDFihWsX7+eJ554YkfbYt1qkmQZ3VkuSXYSyqeEpQJFmtXcs25y63v06MGD\nDz7ID37wA2655Rbee+89RowYwc0338zXv/71eu2XLFnCd7/7XW699VbMjNNOO40rrriCgw8+uN6x\nR40axeOPP87FF1/MggULeOutt+jbty8HH3wwM2bMCBWfiIh0fNbZ/y/UzEYDVVVVVU1W9tXV1YwZ\nM4bmtkuyKX8iIsWT+x0LjHH36jiPrT4oIpJo8+fPL3UIEiPlU8JSgSIiiVZdHev/lEmJKZ8SlgoU\nEUm0q6++utQhSIyUTwlLBYqIiIgkjgoUERERSRwVKCIiIpI4KlBEJNEymUypQ5AYKZ8SlgoUEUm0\nqVOnljoEiZHyKWGpQBGRRBs3blypQ5AYKZ8Slqa6Dyn3hF7pWJQ3EZGOSQVKKyoqKigvL+eMM84o\ndSgSUXl5ORUVFaUOQ0RE2kAFSisGDx7MypUr2bhxY6lDkZCWLVvGscceu+NzRUUFgwcPLmFEUohF\nixYxYcKEUochMVE+JazEPyzQzM4DLgT6A/8Aznf3v7fQ/hjgcuAAYA0wy90XtNC+xYcFSsczduxY\nVqxYUeowJCbKZ7oon+nSaR8WaGYTCYqNGcDBBAXK3WbW5PV6MxsC3AncB3wauAr4tZl9vj3ilWTY\nc889Sx2CxEj5TBflU8JKdIECTAOud/db3H0VcA5QA0xupv25wMvufpG7P+/uVwO/zx5HREREOojE\nFihm1g0YQ3A1BAAP7kfdC4xtZrfDstvz3d1C+8SprKxMzPHasm+Ytq21aWl7c9vi/veKm/LZtm2d\nLZ+FHDPufLbWTvks7jHbul8xf0aTks/EFihABdAF2NBg/QaC/ihN6d9M+13MrHu84RWH/qC1bVtn\n+wWofJZWR/2DpgKlaR01n2Hbd/QCRaN4oAckZ76MTZs2UV0dXz+jQo7Xln3DtG2tTUvbm9vW1PrH\nHnss1n/DQiifymexjhl3Pltrp3wW95ht3a+YP6NtWZ/3t7NHq0G3UWJH8WRv8dQAX3T3xXnrbwb6\nuPsXmtjnQaDK3f8zb93ZwJXuvlsz5/kqcGu80YuIiHQqX3P338Z5wMReQXH3bWZWBRwPLAYwM8t+\n/lUzu60ATm6wblx2fXPuBr4GvApsKSBkERGRzqYHMITgb2msEnsFBcDM/gO4mWD0zmMEo3G+BIx0\n9zfN7OfAAHc/K9t+CPA0cA1wI0Ex80tgvLs37DwrIiIiCZXYKygA7v677JwnlwD9gCeBE939zWyT\n/sCgvPavmtm/AVcCFwDrgCkqTkRERDqWRF9BERERkc4pycOMRUREpJNSgSIiIiKJowKlFWZ2ipmt\nMrPnzWxKqeORwpjZHWb2tpn9rtSxSGHMbG8zW2Zmz5rZk2b2pVLHJNGZWR8z+7uZVZvZU2b2jVLH\nJPEws55m9qqZzWnTfuqD0jwz6wI8BxwNvA9UA59193dKGphEZmZHAb2Bs9z9P0odj0RnZv2Bvu7+\nlJn1A6qA/dz9wxKHJhFkp5Ho7u5bzKwn8CzBE3L1+7aDM7NLgX2Bte5+Udj9dAWlZZ8BnnH3N9z9\nfeD/COZVkQ7K3f9KUGxKB5f9uXwq+34DsBHYvbRRSVQeyM1F1TP7aqWKR+JhZsOBEcCf27qvCpSW\nDQDW531eDwwsUSwi0gwzGwOUufv6VhtLYmVv8zwJrAEuc/e3Sx2TFOwXwA+JUGymtkAxsyPNbLGZ\nrTezWjPLNNHmPDN7xcw+NLNHzOzQUsQqrVM+0yXOfJrZ7sAC4JvFjluaFlc+3X2Tux8EDAW+ZmZ7\ntkf80lgcOc3u87y7v5hb1ZYYUlugAL0IJnb7NtCoo42ZTQQuB2YABwP/AO7OTgyX8xqwd97ngdl1\n0v7iyKckRyz5NLOdgD8C/+XujxY7aGlWrD+f2ck4/wEcWayApVVx5PQw4Ctm9jLBlZRvmNmPQ0fg\n7qlfgFog02DdI8BVeZ+NYObZi/LWdQGeB/YCdgZWAruV+vt09iVqPvO2HQPcXurvoaXwfAKVwE9L\n/R20FJ5PoC+wc/Z9H4LHlhxQ6u+jpfDfudntZwFz2nLeNF9BaVb2ScljgPty6zz4F7wXGJu3bjvw\nPeABghE8v3D1KE+csPnMtr0HWAicbGZrzOyz7RmrtC5sPs3scODLwAQzeyI7PPWA9o5XWtaGn899\ngIfM7AngQYI/fs+2Z6wSTlt+5xYi0c/iKaIKgqsjGxqs30DQ23gHd78TuLOd4pJo2pLPz7dXUBJZ\nqHy6+8N03t9hHUnYfP6d4FaBJF/o37k57r6grSfplFdQREREJNk6a4GyEdhO8ITkfP2AN9o/HCmQ\n8pkuyme6KJ/p0y457ZQFirtvI5h18vjcuuwshscDfytVXBKN8pkuyme6KJ/p0145Te39WzPrBQyn\nbtz1MDP7NPC2u68FrgBuNrMq4DFgGlAO3FyCcKUVyme6KJ/ponymTyJyWurhS0UcFnU0wdCo7Q2W\nG/PafBt4FfgQWAEcUuq4tSifnWFRPtO1KJ/pW5KQUz0sUERERBKnU/ZBERERkWRTgSIiIiKJowJF\nREREEkcFioiIiCSOChQRERFJHBUoIiIikjgqUERERCRxVKCIiIhI4qhAERERkcRRgSIiIiKJowJF\nREREEkcFioiIiCSOChQRaTMzy5jZcjN70sw+MrNaM3vPzHZtpn0/M3vYzN7Ptq01s81mVmVmn2/v\n+EUk+fQ0YxGJzMwMeAvok1013d2vaKH97sA/gTuBc939w+JHKSIdkQoUEYnMzD4NLAO2An2BF939\nE63s8yJwsLtvbocQRaSD0i0eESnEkcADwPXZz/ua2bjmGpvZEGC9ihMRaY0KFBEpxFHA/cC1wLbs\nuvNaaH808GCxgxKRjk8FiogU4kjgfnd/A/gdYMB4MxvUTPujUIEiIiGoQBGRSMxsP4J+bM9lV12V\nfS0Dzmlmt88Bfyt2bCLS8alAEZGojiLofwKAuz8OrCC4ijLFzLrmNzazAcA7GrkjImGoQBGRqI4i\nGMGT75fZ1z2BLzfRvuDbO2bWrdBjiEjyqUARkahyHWTz3QGszb7/doNtRwN/be2gZnaomV2Tndht\nqZn9zsxmmFkPM9sHuKlB+53M7C4zezE7Adz67Oe7zGyZmf3TzG43s1Fm9gUzu9/MPsi2fS57jr3y\njnebmW3Kbn/FzP67zf8yIlIwzYMiIm1mZnsDK9y9UWdYM7sI+G/AgYPc/ens+qeAw5sbYmxmewBX\nA58CLnL3/8vbNgb4LnAwUOnus5rYfyLwW+CL7r4ob303oBI4LhvPGjN7BOjv7kOaiWUC8O/ufmar\n/xgiUhS6giIiUTR1eyfnf4BcP5NvA5hZBbClheJkBPAo0AsYk1+cALh7FfAYMKqF8x6RfV3eYN9t\nBFdddiXoG1MOjG7hOABDs99DREpEBYqIRNFsgeLu7wD/S9BZ9mtmtnO2/UNNtTezPYG/AO8B/+Hu\nW5o5513ABwSFTFOOIJjJdmMT23bOvvYmGEnUlZb7w3y2hfOISDtQgSIiURxJy1cgckOOewFn03IH\n2RuAQcC3Whnh8x7wsLtvb7jBzHYhuDXU3BDmYwhuOf0tG4s3F0/2+UI7ufvWFmIRkSJTgSIibZLt\nK9LD3V9tro27rwTuyX48l+DqRqMrKGZ2OHAacHd2mHJL3gdmNLPtcwS/zx5u4hwVwERgmbv/nqBY\nWejdzqcAAAJ0SURBVO/urzRzrAOBp1uJRUSKTAWKiLRVvflPWvBLgts8I4Gu2Vs/DU0iuJpxUxPb\n6nH3D939sWY2H0ndFZIdspPJ/QlYCpxqZt2BQ2n59k6o0UYiUlxdW28iIlLPCbR8ewcAd/+zmf0T\n2I/m/+AfnX19oMCYjiR4FtD5wR0aHNiJoM/K99z9EQAzOwroTssFyudQB1mRklOBIiKhmdmJwFnA\nkyF3mZtdmisIBgLvNdWx1cw+D3wf6Av0BGqBV4BTsyNzcu12Irgq8jd3P7eVeFrsf5LVU7PdipSe\nbvGISKvMbImZvUowkqYncK2ZPW1mU1vZ9WbgDZq/gvIeUNPUBne/x91PAL4H7Avc5u4n5RcnWYcS\nXBUJc1vmQOAtd3+hqY1mNhaoDnEcESkyXUERkVa5+6kR9/sAGNBCk0eBk82sV7ZtUz5HcNXjz81s\nz/U/CTONfjnQZHGSNS27iEiJ6QqKiJTSlQS/h77Z1MbskN8vEfQl+Xszx8j1P1kR4nxPABXNnOsC\nYKm7rw9xHBEpMhUoIlIy7v4A8CNglplNMrMdv5Oyw4OvA14k6F/S1Pwn3QiusFS3MMFbvnlALzP7\nWt4xepjZDKCvu/+6oC8kIrHRs3hEpOTM7BiCWyv7ABuAd4F/Ab8C1gCfcfeH8trvA/w6235fYDPw\nODAv/zk8zZxrMDCH4ErKR8BW4Dp3vzvebyUihVCBIiIiIomjWzwiIiKSOCpQREREJHFUoIiIiEji\nqEARERGRxFGBIiIiIomjAkVEREQSRwWKiIiIJI4KFBEREUkcFSgiIiKSOCpQREREJHFUoIiIiEji\nqEARERGRxPn/AS1bgKlvhb3VAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGNCAYAAAA7NIpFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3Xl8FfX1//HXCVIgiriAgAqiYlncibTiggqKSvWW2lqq\ntipYW62oRcVfW63ggi24F3CrKNgiotYiuFTEHXcTt1pwB9SvG6KIRhTh/P743EASCExu5mbunbyf\nj8c84M79zMy5OSQ5zHwWc3dERERECklJ0gGIiIiI1KYCRURERAqOChQREREpOCpQREREpOCoQBER\nEZGCowJFRERECo4KFBERESk4KlBERESk4KhAERERkYKjAkVEREQKTkEXKGa2r5nNMLP3zWylmWUi\nHLO/mZWb2TIze93MjmuMWEVERCQ+BV2gABsCLwK/A9a7aJCZdQHuBh4EdgWuAm4ws4PyF6KIiIjE\nzYplsUAzWwkMcvcZ62gzBjjU3Xeptm8q0MbdBzZCmCIiIhKDQr+DUl97ArNr7bsf6JNALCIiIpKj\ntBUoHYCPau37CNjYzFokEI+IiIjkYIOkA0iamW0OHAzMB5YlG42IiEhRaQl0Ae5390/jPHHaCpQP\ngfa19rUHvnD3b+o45mBgSl6jEhERSbdjgFviPGHaCpSngENr7RuQ3V+X+QD//Oc/6dGjR57Cim74\n8OFcccUViZzvpMMO45oPPsCqjgWqjnTg5I4dufbuu3O+zvrarOv9ut5b2/64v4b10ZCvYRQN+Wz1\nOVb5XLt8xJLrOePO5/raKZ/5PWd9j8vn92h99s+dO5df/vKXkP1dGqeCLlDMbEOgK6z6eb+dme0K\nLHb3d83sL8CW7l4118m1wCnZ0Tw3Av2BnwHrGsGzDKBHjx706tUrHx+jXtq0aRNrHPU536E/+xmf\nTJjAIStXhmOBqiPvKylh4JFH1nmuKNdZX5t1vV/Xe2vbH/fXsD4a8jWMoiGfrT7HKp9rl49Ycj1n\n3PlcXzvlM7/nrO9x+fwere/+rNi7SBR6J9k9gBeAcsJ/QC8DKoDzs+93ADpVNXb3+cCPgAMJ86cM\nB05w99ojewrWUUcdldj5zho9mst79OC+khIcOIrwRb+vpIQrevTgzIsuatB11tdmXe/X9d7a9n/4\n4YfrjSVfGvI1jKIh/z7qc6zyuXZxf3825Jxx53N97ZTP/J6zvsfl83u0vvvzpWjmQckXM+sFlJeX\nlxdMVZ+kpUuXctm55/LEjBmULl9OZfPm7J3JcOZFF9G6deukw4tkq6224v3330/s+mn4GhaSpPMp\n8VI+06WiooKysjKAMneviPPcBf2IRxpf69atGXXVVXDVVbg7Zrb+gwpM9pslMWn4GiZt6dKlXHrO\nOTwxcyYrFi/mwG23Ze/DD+es0aNV5BW5pL8/pXgU+iMeSVCx/mJt7NuQ61KsX8MkLV26lJ/26UOf\nCRN4YP58rli2jAfmz6fPhAn8tE8fli5dmnSI0gCF9P0phU13UCR19AOwuF16zjmcMXfuqo7GVdk8\nZOVKfO5cLjv33HCHSorGwoULWbRoEQDdunWjoiLWJwGSR23btqVz586JXFsFiogUlCdmzmRUtjip\n7ZCVK7l8xgxQgVI0Fi5cSI8ePaisrEw6FMlBaWkpc+fOTaRIUYEiqTNkyBBuuummpMOQHLg7Gy5f\nTvUHY0OAqmwaULp8ufr2FJFFixZRWVlZMHNNSXRVc5wsWrRIBYpIHAYMGJB0CJIjM+Or5s1xVk9+\nVD2bDnzVvLmKkyJUKHNNSfFQJ1lJHfVBKW57H34495es/tFUPZv/KSlhn0ym8YMSkUanAkVECkrt\nye4g3snuRKQ4qEARkYLSunVr/vXUUzwzbBgDunThx1ttxYAuXXhm2DD+9dRTmgdFpIlQHxRJnTlz\n5rDPPvskHYY0QPXJ7h5//HH23XffpEMSkUamOygF4rvv4Nhj4Z57oI4RlhLR2LFjkw5BYnTJJZck\nHYJIvY0aNYqSEv2KbQh99QrEhx/C3Llw2GHQvTuMHw9ffpl0VMXp1ltvTToEiZHyKcXIzHIabXbN\nNdcwefLkPERUfFSgFIitt4Znn4UnnoDdd4ff/z7sO+MMePvtpKMrLqWlpUmHIDFSPqUpufrqq1Wg\nZKlAKSBmsNdeMG0avPMOnHQSTJ4MXbvCj38MDz4ITXzxaRFJOc/zD7l8n1/iowKlQHXqBH/9K7z7\nLlx/fShYDjwQdtwRrr5aj39EJD2WLl3KyNNO48Btt2VQp04cuO22jDzttNgWhsz3+efMmUPv3r1p\n1aoVO+ywA9dff/0abW666Sb69+9P+/btadmyJTvuuCPXXnttjTbbbrstr776Ko888gglJSWUlJTQ\nr18/AD777DPOOussdtllF1q3bk2bNm0YOHAgL7/8ciyfoSC5e5PegF6Al5eXeyFbudL94YfdjzjC\nvaTEfeON3U87zf2115KOrPCcddZZSYcgMVI+i1t5ebmv62fsF1984QftuKPfV1LiK8NNYl8Jfl9J\niR+0447+xRdfNOj6+T7/K6+84qWlpd6lSxcfO3asjx492jt27Oi77rqrl5SUrGr3gx/8wIcOHepX\nXXWVT5gwwQ855BA3M7/66qtXtbnrrru8U6dO3rNnT7/lllt8ypQpPnv2bHd3f/75532HHXbwP/3p\nT/73v//dL7roIu/UqZNvuumm/sEHHzToM9Rlfbmr3gbo5XH/fo77hMW2FUuBUt2CBe5//KN727Yh\ngwMGuN91l/t33yUdWWH429/+lnQIEiPls7it75fceaee6veVlIQfZrW2e0tKfORppzXo+vk+/6BB\ng7y0tNTfe++9VfvmzZvnG2ywQY0CZdmyZWsce8ghh3jXrl1r7Ntpp538gAMOWKPtt99+u8a+BQsW\neMuWLf2iiy5qyEeoU9IFih7xFKHOneHii8Pjn8mT4fPPQx+Vrl1hzBj45JOkI0zWqaeemnQIEiPl\nM92emDmTg9exevUTM2YU7PlXrlzJrFmz+MlPfsJWW221an+3bt04+OCDa7Rt0aLFqr9/8cUXfPrp\np/Tt25e333470qOm5s2b17ju4sWLKS0tpVu3blRUVOT8GQqZCpQi1rJlmDvlmWfCCKD99oORI8Po\nn2OPhaefVqdaESlc7muuXl1d9dWrC/H8n3zyCV9//TVdu3Zd471u3brVeP3EE09w4IEHstFGG7HJ\nJpvQrl07zjnnHACWLFmy3mu5O1dccQXf//73adGiBW3btmWLLbbglVdeiXR8MVKBkhK9e8OkSfD+\n+zB6dBiu3KcP9OoVOtmqU62IFJrqq1evTUNXr873+aN66623OPDAA1m8eDFXXHEF9957L7Nnz2b4\n8OFAuCOyPqNHj+bMM89k//33Z8qUKcyaNYvZs2fTs2fPSMcXIxUoKbP55nDWWfDGG3DvvWE00Mkn\nw5ZbwrBh8MorSUeYf/PmzUs6BImR8plutVevri6O1avzef527drRqlUr3njjjTXeq/7vdubMmXz7\n7bfMnDmTE088kUMOOYR+/frRsmXLNY6rq1j617/+Rb9+/bj++uv5+c9/zoEHHki/fv34/PPPc46/\n0KlASamSEjj0UJgxIwxRPv10uOMO2GUX2Gcf+Oc/YdmypKPMj7PPPjvpECRGyme65Xv16nyev6Sk\nhIMPPpjp06fz3nvvrdo/d+5cZs2ater1BhuEZe+q3+lYsmQJkyZNWuOcG2644VqLjmbNmq3xKOr2\n22/n/fffzzn+QqcCpQno3BkuvDB0qr399tB35Ve/gq22CjPVpu0/qOPHj086BImR8plu+V69Ot/n\nP//883F39tlnH8aOHcvo0aPp168fO+2006o2AwYMoHnz5hx22GFcffXVjBkzhj322IP27duvcb6y\nsjJefvllRo8ezbRp03j44YcBOOyww3jkkUcYOnQoN9xwA6effjonn3wy22+/fYPiL2hxDwsqto0i\nHGYch9dfdz/zTPfNNw8j7vr2dZ8yxf3rr5OOTETSJMpQ1epWrlyZ13jycf7HH3/ce/fu7S1btvSu\nXbv69ddf76NGjaoxzPjuu+/23XbbzUtLS3277bbzSy+91G+66SYvKSnxBQsWrGr30Ucf+eGHH+5t\n2rTxkpKSVUOOv/nmGx8xYoRvtdVWvuGGG3rfvn39mWee8QMOOMD79esX+2dyT36YsXkTH+ZhZr2A\n8vLycnr16pV0OI1u2TL497/huuvg0Udhs83CCKATT4SePZOOTkSKXUVFBWVlZTTVn7HFLEruqtoA\nZe4e63hnPeJp4lq2hKOOgkceCY96hg4N/VN23DH0VZk8GSork45SRESaGhUoskq3bnDJJfDee3Dr\nraF4Of546NgRfvc7KJa5gMaMGZN0CBIj5VOkaVKBImto0QIGD4bZs+Gtt+DUU+Guu6CsLMyrMmEC\nfPZZ0lHWrVK3fFJF+RRpmlSgyDpttx1cdBEsWBCGLHfuHIYsb7klHHMMPPQQFNocQeeff37SIUiM\nlE+RpkkFikSywQZw+OEwfXp4BHT++fD889C/P2y/PVxwASxcmHSUIiKSFipQpN46dICzzw6daufM\ngX79YOxY6NIFDjoIbrkFvv466ShFRKSYqUCRnJnB3nvDxInw4Yfhz2++CY9+OnaEk04KCxk29kj2\nRYsWNe4FJa+UT5GmSQWKxGKjjWDIEHjssbAO0LBhcM89sOeeYT6VMWPCQoaNYejQoY1zIWkUyqdI\n06QCRWLXtWvoWDt/PsyaFUb+jBoVOtgecghMnZrfuVVGjRqVv5NLo1M+RZomFSiSN82ahT4pU6aE\nR0DXXgtffglHHx0eAf361/D44/GPAtJslemifIo0TSpQpFG0aROmz58zJzwCOv30MM9K377hjsvI\nkfDmm0lHKSIihUIFijS6rl3DsOS33w7r//TrB1dcATvsAH36wNVXw6efJh2liEjhWrBgASUlJdx8\n8831PvbRRx+lpKSExx57LA+RxUcFiiSmpCTcQbnhBvjoo9A3ZbPN4LTTwiOgQYPgjjvCgob1MXHi\nxPwELIlQPkXiZ2ZJh7BeKlCkILRqBb/4RRj58/77q9cEOvLIMO/KiSeGBQ2j9FepKJZFgyQS5VOk\naVKBIgWnffvQR+X552Hu3LAW0OzZcMABsM02YZK4l16qe36VCRMmNG7AklfKp0jTpAJFClr37nDh\nhaG/ypw5kMnAjTfCbrvBzjvDxRfDO+8kHaWINEWjRo2ipKSEN954g1/+8pdssskmbLHFFpx33nkA\nvPvuuwwaNIg2bdrQsWNHLr/88hrHf/LJJ5xwwgl06NCBVq1asdtuu621T8mSJUs4/vjj2WSTTdh0\n000ZMmQIn3/++Vpjeu211/jZz37G5ptvTqtWrejduzczZ86M/8M3AhUoUhSqZq2dMAE++CA8Ctp1\nVxg9OixouNdeMH586MsiItIYqvpxDB48GIAxY8aw5557Mnr0aK688koGDBjA1ltvzdixY9lhhx0Y\nMWIEc+bMAWDZsmXst99+TJkyhV/96ldceumlbLLJJhx//PGMGzeuxnUymQxTpkzh2GOPZfTo0bz3\n3nscd9xxa/QjefXVV9lzzz157bXX+OMf/8jll1/ORhttxKBBg7jrrrsa4SsSM3dv0hvQC/Dy8nKX\n4vPll+633OJ+2GHuG2zgXlLiftBB7jfe6P7ZZ0lHJyLl5eWe1p+xo0aNcjPzk08+edW+FStWeKdO\nnbxZs2Z+ySWXrNr/+eefe2lpqQ8ZMsTd3a+88kovKSnxqVOnrmrz3Xff+V577eUbb7yxf/nll+7u\nPn36dDczv+yyy1a1W7lypfft29dLSkp88uTJq/b379/fd9ttN1++fHmNOPfee2/v1q3bqtePPPKI\nl5SU+KOPPrrOzxcld1VtgF4e8+/nDRKsjUQabMMN4aijwvbpp/Cvf8Gf/pRh9uwZnHQSHHpo6Hx7\n+OGhrRSfTCbDjBkzkg5DGkFlZViENN+6d4fS0njOZWaccMIJq16XlJSwxx57cNddd9VYpqFNmzZ0\n69aNt99+G4B7772XDh068Itf/GJVm2bNmnHaaadx9NFH8+ijjzJw4EDuvfdemjdvzkknnVTjmqee\neiqPP/74qn2fffYZDz/8MBdeeCFLliypEeOAAQM4//zz+eCDD+jYsWM8H7wRqECR1Nh8c/jNb6BL\nl2HsuCPcdhtMmxaKl9LSUKQMHhyKlpYtk45Woho2bFjSIUgjmTcPysryf53y8rAER1w6d+5c43Wb\nNm1o2bIlm2222Rr7Fy9eDMDChQvZYYcd1jhXjx49cHcWLFiwql3Hjh0prVVRdevWrcbrN998E3fn\nz3/+M+eee+4a5zUzPv74YxUoIkkaMGAAAMOHh+3tt0OxcuutcMQR0Lo1/PjHoVgZMAC+972EA5Z1\nqsqnpF/37qF4aIzrxKlZs2aR9gFVXQtitzI7B8NZZ53FwQcfvNY2Xbt2zcu180UFiqTedtvBH/4Q\ntnnzwl2VadPgn/+ETTYJE8INHgz9+0Pz5klHK9J0lZbGe2ejkG2zzTa88sora+yfO3cuAF26dFnV\n7qGHHqKysrLGXZR5tZ6FbbfddgA0b96cfv365SnqxqVRPNKkdO8e1v159VV4+eUwx8qTT4bHPh06\nwAknwP33w/LlSUcqImk2cOBAPvzwQ6ZNm7Zq34oVKxg3bhytW7emb9++q9otX76ca665ZlW7lStX\nMm7cuBqjeNq1a8f+++/Pddddx4cffrjG9RYtWpTHT5MfuoMiqTN9+nQGDRq0zjZmYR6VnXeG888P\nxcptt4XtxhvDlPuDBoWZbPv102OgJEXJp0ix+c1vfsN1113H8ccfz/PPP0+XLl24/fbbeeqpp7jq\nqqvYMNur//DDD2fvvffmD3/4A++88w49e/bkzjvvZOnSpWucc8KECey7777svPPOnHjiiWy33XZ8\n9NFHPPXUU7z//vu88MILq9rm61FTnHQHRVJn6tSp9WpvtnpOlddfhxdegJNOgsceW31nZciQMPfK\nN9/kKWipU33zKVII6lrrpmp/y5YtefTRRznmmGO4+eabOeuss/j888+ZNGlSjY7hZsbMmTM55phj\nmDJlCueeey6dOnVi8uTJa5y7R48ePP/88xx22GFMnjyZYcOGcd1119GsWTNGjhwZKb5CYsVQReWT\nmfUCysvLy+nVVB5+SiTu4c7KHXfA7bfDa6/BxhuH2Wx/+lM4+OCwhpCI1K2iooKysjL0M7b4RMld\nVRugzN1jXTir4O+gmNkpZvaOmX1tZk+bWe/1tD/GzF40s6/M7P/MbKKZbbauY0TWpurOyoUXhjWB\nXnkFzjgDXnwRfvITaNcOfv7z0OF2LXdbRUSkAQq6QDGzwcBlwEhgd+Al4H4za1tH+72BycDfgZ7A\nz4AfANc3SsCSWmaw006hg+0rr4TRQH/6E7z1VpgIrl27MHR58mTITnMgIiINUNAFCjAcuM7db3b3\necBJQCUwtI72ewLvuPsEd1/g7k8C1xGKFJHYdOsWCpTy8rBY4cUXw6JFoa/KFlvAQQfB1VfD//1f\n0pGKiBSngi1QzKw5UAY8WLXPQ4eZ2UCfOg57CuhkZodmz9EeOBK4J7/RSiEZMmRIo16vS5fw6OeJ\nJ+D998OihWZw+umw1Vaw554wdiy88UajhpUajZ1PESkMBVugAG2BZkDt9Wk/Ajqs7YDsHZNfAtPM\n7FvgA+AzQHNlNyFJzjzasWMYATRrFnz8Mdx8M2y5JYwaBd//fnhMdO658PzzoROurJ9mkhVpmgq5\nQKk3M+sJXAWMIqxSfDCwLeExjzQRRx11VNIhALDppvCrX8Gdd4bHP3feCXvsAddcA717Q+fOMGwY\nPPAAfPtt0tEWrkLJp4g0rkIuUBYBK4D2tfa3B9acJi/4A/CEu1/u7v919weA3wFDs4976jRw4EAy\nmUyNrU+fPkyfPr1Gu1mzZpHJZNY4/pRTTmHixIk19lVUVJDJZNaYwW/kyJGMGTOmxr6FCxeSyWTW\nmL543LhxjBgxosa+yspKMpkMc+bMqbF/6tSpa70dPnjwYH2OhD9HaWkY+TNpEvTvP5gLL5zOEUfA\n3XeH9YA23XQWW2+dYdo0qL4QaaF9juqKOR/6HI33Ob7++us1ri/F5eGHHwbCv72q340dOnQgk8kw\nfPjwvF23oOdBMbOngWfc/fTsawMWAn9z90vW0v4O4Ft3P7ravj7AHGArd1+jsNE8KJIkd3jpJbjr\nLpg+PQxhbt4c9t8/jAo6/PBwp0WkWGkelOJVlPOgmNlbZvYHM1trX5AYXQ6caGbHmll34FqgFJiU\njeMvZlZ9Or2ZwE/N7CQz2zY77PgqQpFT110XSZna/+MsZGaw225h+PILL8D8+XD55aFw+f3vYZtt\nwvvnnRf6rWQXLG1SiimfIhKfXNfi6QKMBi4ws7sJ84zc7zHfjnH327JznlxAeLTzInCwu3+SbdIB\n6FSt/WQz2wg4BbgU+JwwCugPccYlhW3s2LHss88+SYeRk222Cf1Shg0Lj3r+8x+YMQPGjQsTxnXs\nCIcdFu6s9O8fHh2lXTHnU1arWqVXikfSOcvpEY+ZbQ2cQJiPpBPgwHvAROBGd38vziDzSY940qf2\nsuRpsHw5zJkDM2eG7c03wzT7/fuHYuVHPwpDmtMojflsShYuXEiPHj2orKxMOhTJQWlpKXPnzqVz\nHc+a8/mIp0F9ULJ9Qg4GTgQOA5oTOrb+hzCb693uXtA3pVWgSLFxD+sCzZwZOtk+8QSsWAG9eoVC\n5bDDwmihkkLuAi9NysKFC9fofCvFoW3btnUWJ1DABUqNE5ltARwP/BroSrir8iFwIzDR3efHcqGY\nqUCRYrd4Mdx/fyhY/vMf+OyzMJvtoYeGgmXAAGjTJukoRSSNCq6T7Nq4+8fuPhboAVwBGNAROAd4\n08zuMrPd4rqeiASbbQZHHQW33BImh3v88TDlfnl5WMywbVs44AC49NKw6GEBD9wTEVkltgLFzLYx\nswuA+cDvs7ufBC4C3gQOB541s5/EdU2Rtak9D0NTssEGsM8+8Ne/hkUN58+Hv/0NNtwQ/vxn6NkT\nttsOTjkF7rkHiqFbQFPOZxopnxJVgwoUM9vAzH5qZv8B3gLOBVoD1wC7uPs+7n6eu3cHBhP6p1zQ\n0KBF1mVdz0ubmm22gZNPDn1VFi+Ge+8NfVTuuy/8udlmcMghcNVVhbtWkPKZLsqnRJXrKJ4dCH1N\njgPaER7nVBDmKbnF3df6/zIzux3IuHuLnCOOmfqgSFPkDq+/HgqVe++FRx8N0+1vv30oWA49NDwW\n0uAZEVmXfPZByXUelNcInWC/Bm4CrnX35yMct4Qw0kdEEmQG3bqF7fe/hy+/hIceCp1s77kHJkyA\nFi2gb99QsBxyCPToEY4TEWkMuT7i+R9wOrClu/86YnFCtq0GP4oUmI02gkwGrr4a3n4b5s0L/Via\nNYNzzoEddwyPi048Ee64Az7/POmIRSTtcioW3H0ndx/v7l/EHZBIQ9VewEzqp+ruyu9/Hx4BLV4c\n/jziiDBZ3JFHhpFBe+8NF1wATz8d5mHJF+UzXZRPiSrXtXhKzGxjM6vzcY2ZNc+20R0TaVRnn312\n0iGkSqtW4RHPlVeGYcrz54c7LR06hHWD+vSBdu3CkOYbboCFC+O9vvKZLsqnRJVr8TAc+AzYbx1t\n9su2OTXHa4jkZPz48UmHkGrbbAO/+Q3861+waFGYyfbUU+Hdd+G3vw3v9+gBp50WRg8tXdqw6ymf\n6aJ8SlS5juKZA2zt7l3W024BsMDd++YWXv5pFI9IfD77LHS2vf9+mDULFiwIc7P06QMHHRS2PfYI\n+0Sk+BXiTLI7AK9GaPffbFsRaQI23RR++lO4/np4550wlPmqq8J8K5deGgqVtm1Df5Zrrglzr2hm\nWxFZm1z/H9OGMGR4fZYAm+Z4DREpYmawww5h+93v4Lvv4Lnn4IEHwnbaaWFf585w4IFh69cP2rdP\nOnIRKQS53kH5ANglQrtdgI9zvIZITsaMGZN0CLIWVY96zjsvrBe0eHHoo3LEEfDMM3D00aHj7S67\nwBlnhPlYli5VPtNG+ZSocr2D8hBwvJkNdvdpa2tgZj8HegL/yDU4kVxUFsMCM0Lr1mG15R/9KLz+\n4IPQf+XBB8NcK1dcEYqaDh0q+fJL6N8/FDgtCmYeasmFvj8lqlw7yXYHXiDcgfk7cD1hLR6A7YHf\nACcSZpvdw93/G0u0eaBOsiKFxx3efDMUKw8+CA8/DJ9+Ci1bhsUQ+/ULBUuvXupwK5KkfHaSzalA\nATCzI4HJwNr+P2PAMmBIXXdYCoUKFJHCt3IlvPxyuMPy0ENh7aAvv4SNNw7T8ffrF9YO2mUXKNHM\nSyKNphDX4sHdbzezF4AzgP5Ap+xb7wKzgSvdvUDXRxWRYlJSArvtFrYzzoDly+H558OdlYcegj/9\nCZYtC6OF9tsvFCsHHAA9e6pgESlWDbo56u5vAr+LKRaRWCxatIi2bdsmHYbEZG35bN489Efp02d1\ncfLMM6FgefhhOPPMUMS0bQv7779669lTCx4mTd+fEpX+byGpM3To0KRDkBhFyWfLluHOyahR4fHP\n55/D7NlhZtsPPgjrCu20UxjCfOSRYbXmV1/VHCxJ0PenRNXg7mVmtgGwOWvviwKAu8e8OodI3UaN\nGpV0CBKjXPJZWho60fbvH15/9RU89RQ88kjYhg9ffYdlv/1WbzvtpEdC+abvT4mqIZ1kDwTOBfYE\n6lw0EHB3L9h+9uokK9L0VBUsjz4aCpZnngkFy2abwb77ho63++0Hu+6qUUIi61JwnWTN7DDg30Az\nwoKA7wANXBJMRKRxbLjh6tlrAb7+Gp5+OhQsjz4K55wT+rW0bh2GNfftG7Y99oDvfS/Z2EWailz/\nbzCS0H9lODDe3VfEF5KISONq1Wr1yB+Ab74J0/I/9ljYRo+GP/4x9HXp0ycUK/vuC3vuGYodEYlf\nrk9bdwSecverVJxIoZk4cWLSIUiMkshnixbhzsmf/gT/+U9Ypfm550Kh0qYNjB8f7r5ssgn88Idw\n1llw112VI0JpAAAgAElEQVSwaFGjh1p09P0pUeVaoHwJqOOrFKSKilgfg0rCCiGfG2wQHu+ccQb8\n+9/w8cfw3//CuHHQtSvcdhsMGgTt2oWhzL/9LfzjHzB/vkYK1VYI+ZTikOtU97cQprD/fvwhNS51\nkhWROCxYAHPmhIUQH38c/ve/sH+rrcLdmH32gb33DrPdNmuWbKwicSm4TrLA/wOeM7MxwDnu/l2M\nMYmIFJ1ttgnbMceE159+Ck8+ubpoqZoBd6ONQj+WvfcO2557hn0iUlOuBcoQ4D7gLOCnZvYI8B6w\nci1t3d0vzPE6IiJFafPN4fDDwwZhpNDzz8MTT4Si5aqrwsRyJSVhOHNVwbLXXtC5c6KhixSEXAuU\nUYSVig3YLrvVxQEVKCLSpLVqFUb+7LtveL1yJcybFwqWJ56A++8PnW8Btt46FCp77RWKll13DdP7\nizQlDbmDIlKQMpkMM2bMSDoMiUla81lSEjrU9uwJJ54Y9n38cXgs9MQTYSK5//f/wpDnVq2gd+9Q\nsFStQdSuXbLx5yqt+ZT45VSguPvkuAMRicuwYcOSDkFi1JTyucUWYTTQoEHh9TffQEVFKFqeegom\nT4a//jW817Xr6mKlTx/Yeefi6HzblPIpDZPzVPdpoVE8IlIs3GHhwtUFy1NPwYsvwnffhQnjfvCD\nUKzsuWfYivUuixSPQhzFA6xaKPBHwA+AtsAz7n5j9r0ts/v+p1E+IiINZ7Z6tNBRR4V9lZWh8+3T\nT4eCZeJEuPji8N72268uVvbcMwxx1lT9UixyLlDMbB/gn0AnQmdZJywaeGO2SR/gNuBI4M6GhSki\nImtTWrp6rSAId1kWLAgFS9V2221hiHOLFlBWFma//eEPQ9HSuXMofEQKTU4zyZpZT+A/QEdgHPBz\nQpFS3UygEvhpQwIUqa/p06cnHYLESPmsHzPo0gV+8Qu48spQoHzxRbi78te/hoLkzjvD+126QMeO\n8OMfh7suDz4Y2uaT8ilR5XoH5c9AS2Cgu88CsFoluLt/a2YVwO4NilCknqZOncqgql6GUvSUz4Zr\n2XL1Y54qH30EzzyzehszJhQnZtCjR7jD8oMfhG3nneMb5qx8SlS5TnX/IfC2u+9Vbd9KYJK7D622\n7xZCEbNJHMHmgzrJioiEeVlee211wfLss/Dyy6EDbsuWsPvuqwuWH/wg9G/RoyEpxE6ymwDvRmi3\nIaFfioiIFLCSknDnpEcPOP74sO/rr8MooaqC5e67wwy4AJtuGuZm6d07FCy9e4fHRSJxybVA+Rjo\nGqFdD6IVMiIiUmBatVo9z0qVTz+F555bvd1wA4weHd7bcsvVRUvv3qFD7uabJxO7FL9cC5SHgF+Z\n2QHu/vDaGpjZTwhFzIRcgxMRkcKy+eZwyCFhgzBq6L33ahYtl1wCS5aE97fbDvbYIxQse+wBvXrB\nxhsnF78Uj5xG8QB/Bb4FppvZyWbWoeoNM9vUzIYCE4GvgMsbHqZIdEOGaCWGNFE+C5sZdOoERxwB\nf/kLzJ4NixfD66/DlCmQycD778PIkXDAAdCmzRC6d4df/hKuuCKs9Pzll0l/CilEuU51P8/MjgL+\nAYzPbg4cl90AlgFHufs7cQQqEtWAAQOSDkFipHwWn5IS2GGHsB19dNi3YkVYHHHcuAFssAGUl8O/\n/gXLloUip3v3cIelrCxsu+0GG22U7OeQZDVoqnsz2wYYDhwEdCHckXkPeAC4zN3fiiHGvNIoHhGR\nZCxfDv/7X5gJt7w8bC+9FNYgqipaysrCY6GysjCSqHXrpKOW6gpxFA8A7r4A+H1MsYiISBPSvDns\numvYTjgh7KsqWqoKlvJyuOOO1XdadthhddHSq1coWjbdNNnPIfnRoAJFREQkTtWLlqHZWbW++w7m\nzg3FSkVF+POuu8I6RADbbhsKlepFS4cOdV9DioMKFEmdOXPmsM8++yQdhsRE+UyXXPK5wQZhNtud\nd149R8uKFaEj7gsvhILlhRdqjh7q2HF10bL77mHr0kWTyxWTSAWKmb1N6AR7oLu/k30dlbv79jlF\nJ5KDsWPH6hdaiiif6RJXPps1Wz2xXFVHXHeYPz/cZXnhhfDnDTfAhx+G9zfZJHS+3X331X927x7f\nNP4Sr0idZLPT2AN0d/fXq72OxN1zHc6cd+okmz6VlZWUlpYmHYbERPlMlyTy+cEHYUbcF15Yvb2V\nHcLRogXstFMoWKq2XXbRXC1RJd5JtnaBUcgFh4h+maWL8pkuSeSzY8ewHXro6n1LloS1hqoKlooK\nuPnm0EkXwlpDu+66umjZddcw34seETWegu+DYmanAGcBHYCXgFPd/bl1tP8eMBI4JnvM/wEXuPuk\n/EcrIiLFoE0b2HffsFX59tvQGffFF8Nw5xdfDJPJffZZeH/TTVd34K0qXnr2DHdhJH4FXaCY2WDg\nMuA3wLOEOVfuN7Pvu/uiOg67HWgHDAHeAjqS+4y5IiLSRHzve6uLjyru8O67oWCp2u65Z/WiiRts\nEPqx7LpreDRUdXz79rrb0lA5/eI2s35mdqeZ7buONn2zbfrmHh7Dgevc/WZ3nwecBFQCQ+u45iHA\nvsBAd3/Y3Re6+zPu/lQDYpAiM2LEiKRDkBgpn+lSbPk0g86d4fDD4dxz4fbb4Y03YOlSePJJGDcO\n9t4b3n4bLrggrFHUsWMoUA46CM48EyZPDo+Rli1L+tMUl1zvoPyWMHvsceto8yIwgDDl/WP1vYCZ\nNQfKgIur9rm7m9lsoE8dhx0OPA/8PzP7FWEtoBnAn91d/zSaiM6dOycdgsRI+UyXtORzo43WXOl5\n5Up4551wl+Xll8M2fTpcnl2Rrlkz6NYt3GnZZZcwbHqXXdS3pS45TXVvZu8A77r7Ou+OmNljwFa5\nDDM2s47A+0Afd3+m2v4xQF93X6NIMbP7gP0JU+1fALQFrgEecvcT6riORvGIiEjeLF0K//3v6qLl\n5ZfhlVdWz9nSps3qeV6qtp12CsOiC13io3jWogPwZIR27wJ75HiNXJQAK4Gj3f1LADM7A7jdzH7n\n7t80YiwiIiK0br3m3Zaqvi2vvLK6YHn8cfj738PMuRDurFQVK1WFS/fuTadTbq6dR78C2kdotwXh\nEU8uFgEr1nKd9sCHdRzzAfB+VXGSNRcwYOt1XWzgwIFkMpkaW58+fZg+fXqNdrNmzSKTyaxx/Cmn\nnMLEiRNr7KuoqCCTybBoUc3+vCNHjmTMmDE19i1cuJBMJsO8efNq7B83btwaz2wrKyvJZDLMmTOn\nxv6pU6eudWn6wYMH63Poc+hz6HPocxTQ56jq2/KjH8F7751C//4TeeUV+Oqr8IjowgsraNEiw7ff\nLmLqVPjVr8KooVatRrLFFmN44IHG/xxTp05d9buxQ4cOZDIZhg8fvsYxccn1Ec8DwD5AN3dfWEeb\nzsDrwFPufkBOwZk9DTzj7qdnXxuwEPibu1+ylvYnAlcAW7h7ZXbfj4E7gI3WdgdFj3jSZ968eXTv\n3j3pMCQmyme6KJ+5WbIkPCb673/D3Zbf/jbcUUlaPh/x5HoH5UagBXC3ma3xCCe7bybQPNs2V5cD\nJ5rZsWbWHbgWKAUmZa/zFzObXK39LcCnwE1m1iM7gmgsMFGPd5qOs88+O+kQJEbKZ7oon7lp0yaM\nFvrtb2H8+MIoTvItpz4o7j7VzH4C/Ax4xsxeIsw5ArA9sCvhscq/3f0fuQbn7reZWVtCh9f2hJFB\nB7v7J9kmHYBO1dp/ZWYHAeOA5wjFyjTgz7nGIMVn/PjxSYcgMVI+00X5lKhyesQDYGYlwJ+AM4Da\nfY0/JzxqudjdVzQowjzTIx4REZHcFOIoHtx9JXBRdtjvHqy+k/EuUO7u38YQn4iIiDRBDZ7q3t2X\nA09lNxEREZEG0xo1kjq1hxJKcVM+00X5lKgi3UExs/MABya4++Ls66jc3S/MKTqRHFRWViYdgsRI\n+UwX5VOiitRJ1sxWEgqUHu7+erXXUVYPcHdv1rAw80edZEVERHJTCJ1khxIKkg+yr9ecbk5EREQk\nJpEKFHefVOv15DqaioiIiDRYpE6yZrbCzCZWe32ema25MIJIAai9BocUN+UzXZRPiSrqKB6jZn+T\nUcCg2KMRicHQoUOTDkFipHymi/IpUUUtUL4krEwsUvBGjRqVdAgSI+UzXZRPiSpqJ9mXgQPNbCTw\nTnZfVzM7NsrB7n5zLsGJ5EKjsdJF+UwX5VOiilqgnA/cCYwkjOYB2Du7rYtl26tAERERkciijuJ5\nwMx6AgcS1twZBbwE3JW/0ERERKSpirwWj7u/C9wEYGajgBfd/fw8xSWSs4kTJ3LCCSckHYbERPlM\nF+VTooo6zPhGM6ve9XoIcEN+QhJpmIqKWCczlIQpn+mifEpU9ZnqfpK7D82+XpF9XfRlsKa6FxER\nyU0+p7qPOsx4OdCy2uva86KIiIiIxCZqgfIusK+ZbZPPYEREREQgeoFyC7AV8Hb28Q7Acdkp8Ne3\nfZef0EVERCStohYoo4CzgDnA/Oy+SmBhhO3d2KIViSCT0TJRaaJ8povyKVFFnQdlJXB5dqvqNHt7\nVadZkUIybNiwpEOQGCmf6aJ8SlRR76DUNplwN0Wk4AwYMCDpECRGyme6KJ8SVeSJ2qpz9yFxByIi\nIiJSJepEbcea2V51vLexmbWs472jzOzyhgQoIiIiTU/URzyTgF/X8d5nwIQ63hsAnF7PmEQaZPr0\n6UmHIDFSPtNF+ZSocu2DUp0mbZOCMnXq1KRDkBgpn+mifEpUcRQoIgVl2rRpSYcgMVI+00X5lKhU\noIiIiEjBUYEiIiIiBUcFioiIiBQcFSiSOkOGaJqeNFE+00X5lKjqM1HbIWb2UD3f655DTCINopkq\n00X5TBflU6Iyd19/o7D2Tq7c3Zs14Pi8MrNeQHl5eTm9evVKOhwREZGiUVFRQVlZGUCZu1fEee6o\nd1AOiPOiIiIiIusSdTXjR/MdiIiIiEgVdZKV1JkzRwttp4nymS7Kp0SlAkVSZ+zYsUmHIDFSPtNF\n+ZSoVKBI6tx6661JhyAxUj7TRfmUqFSgSOqUlpYmHYLESPlMF+VTolKBIiIiIgVHBYqIiIgUHBUo\nkjojRoxIOgSJkfKZLsqnRKUCRVKnc+fOSYcgMVI+00X5lKiiTnXfoH9R7r6wIcfnk6a6FxERyU0h\nTHU/H1h/JbN2Xo/riIiIiEQuHB4j9wJFREREpF6irsWzf57jEInNvHnz6N69e9JhSEyUz3RRPiUq\ndZKV1Dn77LOTDkFipHymi/IpUcXSN8TMWgCbAd+4++I4zimSq/HjxycdgsRI+UwX5VOiatAdFDP7\njZm9AHwFvAdcWu29I8zsTjPr2sAYRepFwxjTRflMF+VTosqpQDGzZmb2b+AaoAcwF7BazV4CBgGD\nGxShiIiINDm53kEZBvwYuA/Yxt13rt3A3d8C3gQOzT08MLNTzOwdM/vazJ42s94Rj9vbzJabWazj\nskVERCT/ci1Qjgc+Aga7+0fraPc/YJscr4GZDQYuA0YCuxPuytxvZm3Xc1wbYDIwO9drS/EaM2ZM\n0iFIjJTPdFE+JapcC5RuwDPu/tV62n0FtMvxGgDDgevc/WZ3nwecBFQCQ9dz3LXAFODpBlxbilRl\nZWXSIUiMlM90UT4lqlwLlOVAywjtOgNLc7mAmTUHyoAHq/Z5mJd/NtBnHccNAbYFzs/lulL8zj9f\nqU8T5TNdlE+JKtcC5VWgzMxa19XAzLYAdgNezPEabYFmhEdJ1X0EdKjjmjsAFwPHuPvKHK8rIiIi\nCcu1QPkHsDlwrZl9r/abZtYMmACUEvqC5J2ZlRAe64zMdtCFNUcWiYiISBHItUC5HngEOAp4zcyu\nze7f1cyuAl4Hfgo8QCgacrEIWAG0r7W/PfDhWtq3BvYAxmdH7ywH/gzsZmbfmtn+67rYwIEDyWQy\nNbY+ffowffr0Gu1mzZpFJpNZ4/hTTjmFiRMn1thXUVFBJpNh0aJFNfaPHDlyjY5iCxcuJJPJMG/e\nvBr7x40bx4gRI2rsq6ysJJPJMGfOnBr7p06dypAhQ9aIbfDgwU3qcyxatCgVnwPSkY+Gfo5Fixal\n4nNAOvLR0M9Rdf5i/xxVmtLnmDp16qrfjR06dCCTyTB8+PA1jomLhW4dORxo1pIwwubXQPNab68A\nbgROd/dlOQdn9jShM+7p2dcGLAT+5u6X1GprhDlZqjsFOIBQLM1396/Xco1eQHl5eTm9evXKNVQp\nIJlMhhkzZiQdhsRE+UwX5TNdKioqKCsrAyhz91in9ch5qvts4XGKmY0C9ge6EO7IvAc87O7/F0N8\nlwOTzKwceJYwqqcUmARgZn8BtnT347IdaP9X/WAz+xhY5u5zY4hFisSoUaOSDkFipHymi/IpUTV4\nLR53/wS4PYZY1nbu27JznlxAeLTzInBw9poQOst2yse1pXjpTli6KJ/ponxKVLEsFphP7n41cHUd\n7635wKzm++ej4cYiIiJFJ1KBYmbHNuQi7n5zQ44XERGRpiXqHZRJQC69aS17nAoUaTQTJ07khBNO\nSDoMiYnymS7Kp0QVtUC5gNwKFJFGV1FRoR+AKaJ8povyKVHlPMw4LTTMWEREJDf5HGac60RtIiIi\nInmjAkVEREQKTtRRPOc14Bru7hc24HgRERFpYqJ2kh1F6CSby+J7DqhAkUajqbTTRflMF+VToopa\noKxzQjSRQjJs2LCkQ5AYKZ/ponxKVJEKFHefnO9AROIyYMCApEOQGCmf6aJ8SlTqJCsiIiIFRwWK\niIiIFJwGFShmto+ZXWJm083sQTN7aC3bg3EFKxLF9OnTkw5BYqR8povyKVHlVKBYcCPwKHAmkAH2\nr7XtV+3vIo1m6tSpSYcgMVI+00X5lKhyvYNyEnA8UA4cBNyZ3d8NOJSwuOBK4BJguwZFKFJP06ZN\nSzoEiZHymS7Kp0QVdZhxbccDXwGHuvunZvZLAHd/A3gDuN/M7gWmAU8CC2KIVURERJqIXO+g9ACe\ndPdPs68dwMyaVTVw9zsId1jOalCEIiIi0uTkWqCUAJ9We12Z/XPTWu3eAHbO8RoiIiLSROVaoLwP\nbFntddUjnN1rtfs+8F2O1xDJyZAhmvg4TZTPdFE+JapcC5QKoGe1RzqzCOv0jDWz7mbW2sxGAGXA\nCzHEKRKZZqpMF+UzXZRPicrcvf4HmR0FTAEGufuM7L5bgF+Q7Y+StQLYx92fjSHWvDCzXkB5eXk5\nvXr1SjocERGRolFRUUFZWRlAmbtXxHnunEbxuPtUM7uTmo9vjgNeBgYR+qK8Dowt5OJEREREClOu\nw4xx929qvV4O/DW7iYiIiOQs15lkK8zs9riDEYnDnDlzkg5BYqR8povyKVHl2km2G7A8zkBE4jJ2\n7NikQ5AYKZ/ponxKVLkWKG8Am8cZiEhcbr311qRDkBgpn+mifEpUuRYoE4H9zKx7nMGIxKG0tDTp\nECRGyme6KJ8SVU4FiruPIywI+KiZDTezrmb2vVgjExERkSYrp1E8Zrai6q/ApdkNM1tbc3f3nEcL\niYiISNOT6yOed4GFhCnuF65ne7fhYYpEN2LEiKRDkBgpn+mifEpUuU7U1iXmOERi07lz56RDkBgp\nn+mifEpUOU11nyaa6l5ERCQ3+ZzqPtdHPCIiIiJ50+DOq2a2I7AD0JrQaXYN7n5zQ68jIiIiTUfO\nBYqZHQhcDWy/rmaE1Y1VoEijmTdvHt27a4qetFA+00X5lKhyXYtnD+AeoDNwC/BK9q2/ArcDn2Vf\n3wRc0MAYRerl7LPPTjoEiZHymS7Kp0SV6x2UP2aPPcTdHzCzm4Cd3f0cADPbBLgOOAzYI5ZIRSIa\nP3580iFIjJTPdFE+JapcO8nuBbzg7g+s7U13/xw4FlgJXJTjNURyomGM6aJ8povyKVHlWqBsRlgw\nsMq3AGa2YdUOd/8GeBw4KOfoREREpEnKtUD5BNi41muA7Wq1awW0yfEaIiIi0kTlWqC8CWxb7fWz\nhBE7v63aYWZdgX7A2zlHJ5KDMWPGJB2CxEj5TBflU6LKtUC5F+hmZj2yr/9DWJfnZDN7xsz+BTwH\ntAQmNjxMkegqKyuTDkFipHymi/IpUeU01b2ZdQB+DMxx91ez+3YGbgO6ZZutJBQnJ3kBz6evqe5F\nRERyk8+p7nNdLPBDwjDi6vteAXqYWXdgU+BNd/9kbceLiIiIrEuDp7qvzd3nxX1OERERaVrq1QfF\nzAaa2fVmdp+ZTTezC8xs2/UfKdJ4Fi1alHQIEiPlM12UT4kqcoFiZlOAmcAJwMFABjgHeNXMMvkJ\nT6T+hg4dmnQIEiPlM12UT4kq0iMeMzsBOAr4DvgH8AJh9eLDgD7AzWa2jbsvyVegIlGNGjUq6RAk\nRspnuiifElXUPijHEUblHOruD1bb/5fsOjzHAkcQFgcUSZRGY6WL8pkuyqdEFfURz87A07WKkyoX\nEyZp2zm2qERERKRJi1qgbAy8Vcd7b1VrIyIiItJgUQsUA1as7Q13X1nPc9WLmZ1iZu+Y2ddm9rSZ\n9V5H25+Y2Swz+9jMlpjZk2Y2IB9xSeGaOFGTF6eJ8pkuyqdElZeiIi5mNhi4DBgJ7A68BNxvZm3r\nOKQvMAs4FOgFPAzMNLNdGyFcKRAVFbFOZigJUz7TRfmUqCJNdW9mK4Fcp6t3d89pQjgzexp4xt1P\nz7424F3gb+4+NuI5/gvc6u4X1fG+proXERHJQT6nuq/PHRTLccvpLo2ZNQfKgFUdc7Nr+swmDG2O\ncg4jDIdenEsMIiIikoxIdzbcPYlHQW2BZsBHtfZ/xOoFCddnBLAhYRFDERERKRKxr8VTKMzsaODP\nQMbdNbeyiIhIESnkTrKLCCOH2tfa3x74cF0HmtkvgOuBI9394SgXGzhwIJlMpsbWp08fpk+fXqPd\nrFmzyGTWnNn/lFNOWaN3ekVFBZlMZo21J0aOHMmYMWNq7Fu4cCGZTIZ582qutThu3DhGjBhRY19l\nZSWZTIY5c+bU2D916lSGDBmyRmyDBw9uUp8jk8mk4nNAOvLR0M+RyWRS8TkgHflo6OeoiqfYP0eV\npvQ5pk6duup3Y4cOHchkMgwfPnyNY+ISqZNsUuroJLuQ0En2kjqOOQq4ARjs7ndHuIY6yabMrFmz\nGDBAo8vTQvlMF+UzXfLZSbbQH/FcDkwys3LgWWA4UApMAjCzvwBbuvtx2ddHZ987DXjOzKruvnzt\n7l80buiSFP3wSxflM12UT4mqoAsUd78tO+fJBYRHOy8CB7v7J9kmHYBO1Q45kdCxdkJ2qzIZ0BKa\nIiIiRaKgCxQAd78auLqO94bUen1AowQlIiIieVXInWRFclK7Y5oUN+UzXZRPiUoFiqTO1KlTkw5B\nYqR8povyKVGpQJHUmTZtWtIhSIyUz3RRPiUqFSgiIiJScFSgiIiISMFRgSIiIiIFRwWKpM7apmuW\n4qV8povyKVGpQJHU0UyV6aJ8povyKVGpQJHUOeqoo5IOQWKkfKaL8ilRqUARERGRgqMCRURERAqO\nChRJnTlz5iQdgsRI+UwX5VOiUoEiqTN27NikQ5AYKZ/ponxKVCpQJHVuvfXWpEOQGCmf6aJ8SlQq\nUCR1SktLkw5BYqR8povyKVGpQBEREZGCowJFRERECo4KFEmdESNGJB2CxEj5TBflU6JSgSKp07lz\n56RDkBgpn+mifEpU5u5Jx5AoM+sFlJeXl9OrV6+kwxERESkaFRUVlJWVAZS5e0Wc59YdFBERESk4\nKlBERESk4KhAkdSZN29e0iFIjJTPdFE+JSoVKJI6Z599dtIhSIyUz3RRPiUqFSiSOuPHj086BImR\n8pkuyqdEpQJFUkfDGNNF+UwX5VOiUoEiIiIiBUcFioiIiBQcFSiSOmPGjEk6BImR8pkuyqdEpQJF\nUqeysjLpECRGyme6KJ8Slaa611T3IiIiOdFU9yIiItKkqEARERGRgqMCRVJn0aJFSYcgMVI+00X5\nlKhUoEjqDB06NOkQJEbKZ7oonxKVChRJnVGjRiUdgsRI+UwX5VOiUoEiqaPRWOmifKaL8ilRqUAR\nERGRgqMCRURERAqOChRJnYkTJyYdgsRI+UwX5VOiUoEiqVNREetkhpIw5TNdlE+JSlPda6p7ERGR\nnGiqexEREWlSVKCIiIhIwVGBIiIiIgVHBYqkTiaTSToEiZHymS7Kp0SlAkVSZ9iwYUmHIDFSPtNF\n+ZSoVKBI6gwYMCDpECRGyme6KJ8SlQoUERERKTgqUERERKTgqECR1Jk+fXrSIUiMlM90UT4lqoIv\nUMzsFDN7x8y+NrOnzaz3etrvb2blZrbMzF43s+MaK1YpDGPGjEk6BImR8pkuyqdEVdAFipkNBi4D\nRgK7Ay8B95tZ2zradwHuBh4EdgWuAm4ws4MaI14pDO3atUs6BImR8pkuyqdEVdAFCjAcuM7db3b3\necBJQCUwtI72JwNvu/vZ7v6au08A7sieR0RERIpEwRYoZtYcKCPcDQHAw8qGs4E+dRy2Z/b96u5f\nR/uCM3Xq1II5X32OjdJ2fW3W9X5d78X99Yqb8lm/95paPhtyzrjzub52ymd+z1nf4/L5PVoo+SzY\nAgVoCzQDPqq1/yOgQx3HdKij/cZm1iLe8PJDv9Dq915T+wGofCarWH+hqUBZu2LNZ9T2xV6gbNCo\nVytMLQHmzp2bdBwALFmyhIqK+Fasbsj56nNslLbra7Ou9+t6b237n3322Vi/hg2hfCqf+Tpn3Plc\nXzvlM7/nrO9x+fwerc/+ar87W6436Hqy8NSk8GQf8VQCP3X3GdX2TwLauPtP1nLMo0C5u59Rbd/x\nwBXuvmkd1zkamBJv9CIiIk3KMe5+S5wnLNg7KO6+3MzKgf7ADAAzs+zrv9Vx2FPAobX2Dcjur8v9\nwDHAfGBZA0IWERFpaloCXQi/S2NVsHdQAMzs58AkwuidZwmjcX4GdHf3T8zsL8CW7n5ctn0X4BXg\navfJJoYAAAnlSURBVOBGQjFzJTDQ3Wt3nhUREZECVbB3UADc/bbsnCcXAO2BF4GD3f2TbJMOQKdq\n7eeb2Y+AK4DTgPeAE1SciIiIFJeCvoMiIiIiTVMhDzMWERGRJkoFioiIiBQcFSjrYWaHmdk8M3vN\nzE5IOh5pGDO708wWm9ltScciDWNmW5vZw2b2qpm9aGY/SzomyZ2ZtTGz58yswsxeNrNfJx2TxMPM\nWpnZfDMbW6/j1AelbmbWDPgfsB/wJVAB/NDdP0s0MMmZmfUFWgPHufvPk45HcmdmHYAt3P1lM2sP\nlAM7uPvXCYcmOchOI9HC3ZeZWSvgVaBMP2+Ln5ldBGwPvOvuZ0c9TndQ1u0HwH/d/UN3/xK4hzCv\nihQpd3+MUGxKkct+X76c/ftHwCJgs2Sjklx5UDUXVavsn5ZUPBIPM+sKdAPuq++xKlDWbUvg/Wqv\n3we2SigWEamDmZUBJe7+/nobS8HKPuZ5EVgIXOLui5OOSRrsUuCP5FBsprZAMbN9zWyGmb1vZivN\nLLOWNqeY2Ttm9rWZPW1mvZOIVdZP+UyXOPNpZpsBk4ET8x23rF1c+XT3Je6+G7AtcIyZtWuM+GVN\nceQ0e8xr7v5m1a76xJDaAgXYkDCx2++ANTramNlg4DJgJLA78BJwf3ZiuCr/B2xd7fVW2X3S+OLI\npxSOWPJpZt8D/g1c7O7P5DtoqVOs35/ZyThfAvbNV8CyXnHkdE/gF2b2NuFOyq/N7NzIEbh76jdg\nJZCpte9p4Kpqr40w8+zZ1fY1A14DOgIbAXOBTZP+PE19yzWf1d7bH7g96c+hreH5BKYC5yX9GbQ1\nPJ/AFsBG2b+3ISxbsmPSn0dbw3/mZt8/Dhhbn+um+Q5KnbIrJZcBD1bt8/AVnA30qbZvBXAm8Ahh\nBM+lrh7lBSdqPrNtHwCmAYea2UIz+2FjxirrFzWfZrY3cCQwyMxeyA5P3bGx45V1q8f35zbA42b2\nAvAo4Zffq40Zq0RTn5+5DVHQa/HkUVvC3ZGPau3/iNDbeBV3vxu4u5HiktzUJ58HNVZQkrNI+XT3\nJ2i6P8OKSdR8Pkd4VCCFL/LP3CruPrm+F2mSd1BERESksDXVAmURsIKwQnJ17YEPGz8caSDlM12U\nz3RRPtOnUXLaJAsUd19OmHWyf9W+7CyG/YEnk4pLcqN8povymS7KZ/o0Vk5T+/zWzDYEurJ63PV2\nZrYrsNjd3wUuByaZWTnwLDAcKAUmJRCurIfymS7KZ7oon+lTEDlNevhSHodF7UcYGrWi1nZjtTa/\nA+YDXwNPAXskHbc25bMpbMpnujblM31bIeRUiwWKiIhIwWmSfVBERESksKlAERERkYKjAkVE5P+3\nd7+hetZ1HMffn9jm5qyk5irLZgwrH2SZrGh6ziS0f7D+lw+ESqJo9odCsgc92CNBelBWllaCQZBi\nEUU1c8K20m1qatLAYA0mxsAisc5c6qR9e3Bdx+4d7+u+zx/PzjV4v+Bwn/v6/a7r+t3nwX0+/P5d\nknrHgCJJknrHgCJJknrHgCJJknrHgCJJknrHgCJJknrHgCJJknrHgCJJknrHgCJJknrHgCJJknrH\ngCJpzpK8P8ndSR5K8kySY0mmkpzeUf8VSXYnebKteyzJ4SQPJLn0RLdfUv/5NGNJ85YkwOPAS9tD\nX62qb46o/zJgP/AbYEtVPbX4rZR0MjKgSJq3JG8GdgJHgbXAgap6/ZhzDgDnV9XhE9BESScph3gk\nLcQEsAv4Qft+fZJ3dVVOcjZwyHAiaRwDiqSFmAR2ADcAz7bHPj+i/ibg94vdKEknPwOKpIWYAHZU\n1WPAbUCA9yU5q6P+JAYUSbNgQJE0L0nOoZnH9nB76Nvt64uAz3WcthHYs9htk3TyM6BImq9Jmvkn\nAFTV/cBeml6UTydZNlg5yZnAE67ckTQbBhRJ8zVJs4Jn0HXt6xnAx4bUX/DwTpLlC72GpP4zoEia\nr+kJsoN+Afyt/f3KGWWbgD+Mu2iSDUm+327stj3JbUm2JlmZZB1w84z6K5JsS3Kg3QDuUPt+W5Kd\nSfYn+VmSc5N8KMmOJEfaug+393jVwPVuTfLvtvxgkmvn/JeRtGDugyJpzpK8BthbVc+bDJvkauBa\noIC3VNW+9vifgQu7lhgneTnwPeBNwNVV9duBsguALwPnA7dU1TVDzr8M+Cnwkar65cDx5cAtwDvb\n9jya5B7glVV1dkdbPgh8uKo+MfaPIWlR2IMiaT6GDe9M+xEwPc/kSoAka4CnR4STNwD3AquBCwbD\nCUBVPQDcB5w74r4Xta93zzj3WZpel9Np5sacCrx1xHUAXtd+DklLxIAiaT46A0pVPQH8hGay7OVJ\nTmvr3zWsfpIzgN8BU8DHq+rpjntuA47QBJlhLqLZyfafQ8pOa19fTLOSaBmj58O8fcR9JJ0ABhRJ\n8zHB6B6I6SXHq4FPMXqC7A+Bs4DPjlnhMwXsrqr/zixI8hKaoaGuJcwX0ww57WnbUl3taZ8vtKKq\njo5oi6RFZkCRNCftXJGVVfVIV52q+gtwZ/t2C03vxvN6UJJcCHwAuKNdpjzKk8DWjrKNNN9nu4fc\nYw1wGbCzqn5OE1YOVdXBjmudB+wb0xZJi8yAImmujtv/ZITraIZ53ggsa4d+ZrqCpjfj5iFlx6mq\np6rqvo7iCf7fQ/KcdjO5XwHbgc1JTgE2MHp4Z1arjSQtrmXjq0jScS5h9PAOAFV1e5L9wDl0/8Pf\n1L7uWmCbJmieBfTFZoSGAlbQzFm5qqruAUgyCZzC6ICyESfISkvOgCJp1pK8G/gk8NAsT/lu+9MV\nCF4NTA2b2JrkUuBrwFpgFXAMOAhsblfmTNdbQdMrsqeqtoxpz8j5J61V7nYrLT2HeCSNleTXSR6h\nWUmzCrghyb4kXxhz6o+Bx+juQZkC/jOsoKrurKpLgKuA9cCtVfWewXDS2kDTKzKbYZnzgMer6q/D\nCpO8A3hwFteRtMjsQZE0VlVtnud5R4AzR1S5F3hvktVt3WE20vR63N5RPj3/ZDbb6J8KDA0nra+0\nP5KWmD0okpbSt2i+hz4zrLBd8vtRmrkkf+y4xvT8k72zuN+fgDUd9/oSsL2qDs3iOpIWmQFF0pKp\nql3A14FrklyR5LnvpHZ58I3AAZr5JcP2P1lO08Py4IgN3gZdD6xOcvnANVYm2QqsraqbFvSBJL1g\nfBaPpCWX5GKaoZV1wN+BfwH/AL4DPAq8raruGqi/Driprb8eOAzcD1w/+Byejnu9FvgGTU/KM8BR\n4MaquuOF/VSSFsKAIkmSeschHkmS1DsGFEmS1DsGFEmS1DsGFEmS1DsGFEmS1DsGFEmS1DsGFEmS\n1DsGFEmS1DsGFEmS1DsGFEmS1DsGFEmS1DsGFEmS1Dv/AyCTnPHauyfMAAAAAElFTkSuQmCC\n", "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -476,16 +502,16 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 82, "metadata": { "collapsed": false }, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAg0AAAFjCAYAAAC+DjgjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3X+U3HV97/HXOyEKG2KUJmQDZANardFSMIvKClKqskAq\nX1GUZdFTmliu1uRo15J47tHeRA6WbhC4mOARrtsL3NYhtMebAj2ViMf2NlpEd4X2tontrYSFEAIr\ngoEJEJL3/WNm2J39+ZnZ736/M599Ps7Zk+x3vt+Z9zcvdufN98d7zN0FAAAwlTl5FwAAAJoDTQMA\nAAhC0wAAAILQNAAAgCA0DQAAIAhNAwAACELTAAAAgtA0AACAIEflXcB4zOzXJJ0vaY+kF/OtBgCA\npnK0pJMl3efuv0jziRuyaVCpYfjLvIsAAKCJfVzSt9J8wkZtGvZI0l/8xV9oxYoVOZci9fT06MYb\nb2yI56tl25B1p1pnsscnemy85Wn/G04HeZLnTD1n2nlOtR55zuxz1rrdTP6M1rJ8165d+sQnPiGV\n30vT1KhNw4uStGLFCq1cuTLvWrRw4cJU65jO89Wybci6U60z2eMTPTbe8rT/DaeDPMlzpp4z7Tyn\nWo88Z/Y5a91uJn9Ga11elvrp/bmbNm1K+zmn7ctf/vJSSZ/61Kc+paVLl+ZdjiTp1FNPbZjnq2Xb\nkHWnWmeyxyd6bPTyG2+8UVdeeeWUtWSFPGt7bLblOZ3nTDvPqdYjz5l9zlq3m8mf0dDl+/bt0623\n3ipJt27atGnflAXVwBrxUy7NbKWk/v7+/obpfjE9J554ovbu3Zt3GUgJecaFPOMyMDCg9vZ2SWp3\n94E0n5tbLpGJ8n/AiAR5xoU8EarmpsHM3mtmd5vZXjM7YmbJFOt/2Mx2mNlTZvacmf3QzDrrLxnN\nqLu7O+8SkCLyjAt5IlQ9RxrmS3pI0mckhZzbOEfSDkkXSlop6fuS7jGz0+p4bTQpfinFhTzjQp4I\nVfPdE+7+HUnfkSQzs4D1e0Yt+qKZfUjSRZIervX1AQBAPjK/pqHcaCyQ9EzWr438rF69Ou8SkCLy\njAt5IlQecxrWq3SK464cXhs56ezkMpaYkGdcyDPciy9Kjz0mDQ5Kjz5a+rPy9/XrpQsuyLvCmZVp\n02Bml0v6E0mJuw9l+drIF+dM40KecSHPEnfpF78Y2xCM/H7//uptWlultjZp+XLp6KPzqTtLmZ2e\nMLPLJN0q6WPu/v2QbVatWqUkSaq+Ojo6tH379qr1duzYoSQZexPH2rVr1dfXV7VsYGBASZJoaKi6\nZ9m4caN6e3urlg0ODipJEu3evbtq+ZYtW7R+/fqqZcViUUmSaOfOnVXLC4XCuIf+urq62A/2g/1g\nP9iPDPfjYx/r0i23bNff/710xx3SNddIq1bt0PHHJ1qxQjr2WGnxYqm9XfrIR9bqqqv6dO+90rPP\nSr/5m9KHPjSg009P9O1vD+k//qN01GHfPumCCzaqvb1X556bzX6MzKNQKLz63tja2qokSdTTM/pS\nwvRMa7iTmR2RdLG73z3Fet2Svimpy93vDXhehjsBAGry7LPjnzao/LlvX+loQsWiRaUjBG1tw0cL\nRv65eLE09eX+jWcmhzvVfHrCzOZL+nVJlX/KN5Zvn3zG3R8zs2slneDuV5TXv1zSbZI+K+nHZrak\nvN1Bd//VdHcAzWHnzp06++yz8y4DKSHPuDRDnq+8UnrTn+i0weCg9KsR7yjz5knLlpXe/N/yFukD\nH6huEJYtk1pa8tufZlXPNQ1nqDRrwctf15eX3y5pjaRWSctGrH+lpLmSbi5/adT6mAU2b97c8L+U\nEI4849IIeR44MH4jUPl+717p8OHh9V//+lITsHy5dO651UcI2tpK1xrMYeZx6vjsCWSiWCyqhbY+\nGuQZl5nO88gR6cknxz9tUPn7s88Orz93rnTiiWMbgZF/X7Bgxspteg11egKoB28wcSHPuEw3z2Kx\ndBviRKcOHntMOnRoeP0FC4aPErznPdJll1U3BUuXSkfx7tSQiAUAMCF36emnJ764cHBQGnlThJl0\nwgnDRwTe9a7qIwXLl0sLF+a3P5gemgYAmMVefFF6/PGJ7zoYHJReeml4/ZaW0hv/smWVWxOrLy48\n6STpNa/Jb38ws2gakIn169fruuuuy7sMpIQ8m4O79MwzEx8hePTRyrCi9ZJKeVaGFbW1SaefPvz3\nypGC445rztsQkQ6aBmSira0t7xKQIvJsDIcOle4qmOwowQsvDK//2tcOv/m//e3SqlWlv/f3t+mz\nny0dKZgNUw1RP+6eAIAG9dxzY48OjPz7E09UDyv6tV8bvm5gvDsOjj+eowSzAXdPAEBkDh8uvelP\nNptg5LCio44qHQlYvlx685tLw4oq31euMeCmFsw0mgYAmAHPPz/59MLHHx87rKhyZOCcc4YbgUpT\nsGRJaX4BkCeaBmRi9+7deutb35p3GUjJbM/zyJHSBYSTDSv65S+H1x89rOjss4fvNqicPnjd6/Lb\nn9meJ8LRNCATGzZs0N13T/q5ZmgisedZGVY00bUEo4cVHXvs8BGBM8+ULr20+kjBCSc09rCi2PNE\nehr4P2PEZOvWrXmXgBQ1c56VYUUTHSEYHCw9XjHZsKLKkYOFC5v7AsNmzhPZomlAJrhFLy6NnOdL\nL5WuF5jo1MHgYGmgUcUxxwy/+Z9+uvShD1XfcTAbhhU1cp5oLDQNAJqGe+lagcmuJXjyyeptliwZ\nbgpOPbW6IWhrkxYtau6jBECWaBoANIx6hxW1tUlve5t0wQXVTQHDioB00TQgE729vfrCF76QdxlI\nSb15Pvfc5A3BE0+U7kyoqAwramuTzjuveljR8uXS4sXSnDkp7tgsxc8nQtE0IBPFYjHvEpCi8fI8\nfFjat2/yUwejhxWddFLpzf9Nb5Le976xw4rmz89wp2Yxfj4RijHSAII8//zEg4oefXTyYUWj7zZo\nayt9MBLDioD0MUYawIyqDCua7DbEZ54ZXn/OnOphRe95T/VcgryHFQGYGTQNwCxw8ODUw4pefnl4\n/cqwomXLpHe/uzSsaOSRgkYfVgRgZvBjj0wMDQ1p0aJFeZcRJXdpaGjyawlGDytaunS4CTjjjLGn\nD17/+slvQyTPuJAnQtE0IBNr1qxhTG2dXn65dL3AyGZg9LCigweH1x85rOi006QkSX9YEXnGhTwR\niqYBmdi0aVPeJTSkyrCiya4lePLJ0noVxx8/3BSsWpXPsCLyjAt5IhRNAzIxW++CeeWVqYcVPf/8\n8Pqvec3wm/+KFdL5548dVnTMMfntT8VszTNW5IlQNA3ANPzqV5MfJdi7t3pY0XHHDc8h+MAHqucS\ntLWVjiIwrAhAo6JpACZw+HDp1MBE1xEMDkrPPju8/lFHlW5DXL5cOuUU6dxzxw4rOvbY3HYHAKaN\npgGZ6Ovr0yc/+cm8y6jywgtTDyt65ZXh9RcuHL6W4Oyzxw4rWrp09gwrasQ8UT/yRCiaBmRiYGAg\n019KR45ITz01/hGCyve/+MXw+nPmlGYPVJqAM88cO6xo4cLMym94WeeJmUWeCMUYaTSlF18sDSQa\n79TBo4+OHVY0f/7YJmDk9yeeKM2bl9/+AEBaGCONWcW9dBRgsmFFTz1Vvc3IYUUrV44dVvSGN8z8\nbYgAEDuaBmRuvGFFo5uDkcOKjj56+OjAqadKH/zg2GFFr31tfvsDALMFTQNS5V66o2CyuQT79lUP\nK1q8eLgJuPDCsacPFi/mKAEANAKaBtTklVekJ54YvyGo/DlyWNG8eaUG4NlnE33wg3frvPPGNgWN\nMKwItUmShLHDESFPhKJpQJUDByY/SrB3b2l+QcUb3jA8h+D97x87rGjJktKdCTt2rFNnZ377hXSt\nW7cu7xKQIvJEKJqGWeTIkephReM1ByOHFc2dOzys6OSTpXPOqZ5L0NYWPqyok44hKuQZF/JEKJqG\niLzwQulWw4mOFDz+uHTo0PD6CxYMHxV4z3ukyy6rbgpOOGH2DCsCAEyNpqFJuI8/rGjk34eGhtev\nDCuqnC4488zhDzyqNAoMKwIA1IKmoUFUhhVNdNrgscekl14aXn/+/OFTBGecIX3kI409rGj79u26\n+OKL8y4DKSHPuJAnQtE0ZKAyrGiyT0Pcv796m9bW4Sbg9NOrryNoxmFFhUKBX0oRIc+4kCdCMUY6\nBS+/XLqrYKJBRYODUrE4vP7RR4+dWDjy+2XLGFYEAKhPQ42RNrP3SlovqV3SUkkXu/ukN/ia2bmS\nrpf0dkmDkr7i7rfXXG0O3KXnnpv8joPRw4oWLRpuBM4/v/oWxGXLpOOPb66jBAAASPWdnpgv6SFJ\nfZK+PdXKZnaypHslfV3S5ZI+IOmbZvaEu3+3jtfPzAsvlD7T4MCB4WXz5pXe+NvapLe8pTSbYPSw\nopaW/GpGOtxdRmcHAFVqbhrc/TuSviNJFvZb9Q8l/dzdN5S//5mZnS2pR1JDNw3z50tXX139kcmV\nYUWIz4EDB/TVL35RP7jnHs0/dEgvzJunsy66SFd95StasGBB3uUBQO6yePs7U9L9o5bdJ6kjg9ee\ntj/6I+nSS0u3LC5dSsNQr9WrV+ddwqQOHDigSzo61HHzzfrunj36m7179d09e9Rx8826pKNDB0Ye\nbkLD54nakCdCZfEW2Cpp1L0B2i/pdWbG5X6zRKNPnPvqF7+oz+/apQuOHFHl8JlJuuDIEfXs2qXr\nv/SlPMtrOI2eJ2pDngjF/zcjE93d3XmXMKkf3HOPzj9yZNzHLjhyRD/gw3yqNHqeqA15IlQWTcOT\nkpaMWrZE0q/c/aVx1n/VqlWrlCRJ1VdHR4e2b99etd6OHTuUJMmY7deuXau+vr6qZQMDA0qSREMj\nxydK2rhxo3p7e6uWDQ4OKkkS7d69u2r5li1btH79+qplxWJRSZJo586dVcsLhcK4h/66urrYjwbZ\nD3fXrv379Tej6t0hKVHpiEPLoUOq3J7cqPshxZEH+8F+sB/h+1EoFF59b2xtbVWSJOrp6RmzTVqm\nNafBzI5oilsuzezPJF3o7qeNWPYtSa9391UTbNNUcxrQ/D5wyin67p49Gu/KXpd03skn6/5HHsm6\nLACo2UzOaaj5SIOZzTez08zs9PKiN5a/X1Z+/FozGzmD4RvldXrN7DfM7DOSPirphmlXj6YxuoNu\nNGdddJHum+Aq1+/MmaOzx/k/kdms0fNEbcgToeo5PXGGpJ9K6lfpf8KulzQg6cvlx1slLaus7O57\nJP2uSvMZHlLpVstPuvvoOyoQsc2bN+ddwqSu+spXdMOKFfq7OXNUOfbmkv5uzhzduGKF/viaa/Is\nr+E0ep6oDXkiFGOkkYlisaiWBp96deDAAV3/pS/pB3ffrZZDh1ScN09nJYn++JprmNMwSjPkiXDk\nGZeGGiMN1KMZfiEtWLBAm266SbrpJiZCTqEZ8kQ48kQobrkExkHDAABj0TQAAIAgNA3IxOj7ktHc\nyDMu5IlQNA3IRFtbW94lIEXkGRfyRCjungAAICINNdwJAADMTjQNAAAgCE0DMjH6Q1vQ3MgzLuSJ\nUDQNyMSGDRvyLgEpIs+4kCdC0TQgE1u3bs27BKSIPONCnghF04BMcEtXXMgzLuSJUDQNAAAgCE0D\nAAAIQtOATPT29uZdAlJEnnEhT4SiaUAmisVi3iUgReQZF/JEKMZIAwAQEcZIAwCA3NE0AACAIDQN\nyMTQ0FDeJSBF5BkX8kQomgZkYs2aNXmXgBSRZ1zIE6FoGpCJTZs25V0CUkSecSFPhKJpQCa4CyYu\n5BkX8kQomgYAABCEpgEAAAShaUAm+vr68i4BKSLPuJAnQtE0IBMDA6kOJUPOyDMu5IlQjJEGACAi\njJEGAAC5o2kAAABBaBoAAEAQmgZkIkmSvEtAisgzLuSJUDQNyMS6devyLgEpIs+4kCdC0TQgE52d\nnXmXgBSRZ1zIE6FoGgAAQBCaBgAAEISmAZnYvn173iUgReQZF/JEKJoGZKJQKORdAlJEnnEhT4Si\naUAmtm3blncJSBF5xoU8EaqupsHM1prZI2Z20MweMLN3TrH+x83sITN7wcyeMLM+MzuuvpIBAEAe\nam4azKxL0vWSNkp6h6SHJd1nZosmWP8sSbdL+h+S3ibpo5LeJenWOmsGAAA5qOdIQ4+kW9z9Dnff\nLenTkoqS1kyw/pmSHnH3m939UXf/oaRbVGocAABAk6ipaTCzeZLaJX2vssxLn619v6SOCTb7J0nL\nzOzC8nMskfQxSX9bT8FoTqtXr867BKSIPONCnghV65GGRZLmSto/avl+Sa3jbVA+svAJSdvM7GVJ\n+yT9UhJzS2cRJs7FhTzjQp4INeN3T5jZ2yTdJGmTpJWSzpd0ikqnKCa1atUqJUlS9dXR0THmnuId\nO3aM+4Era9euVV9fX9WygYEBJUmioaGhquUbN25Ub29v1bLBwUElSaLdu3dXLd+yZYvWr19ftaxY\nLCpJEu3cubNqeaFQGLeL7+rqmlX70d3dHcV+SHHkMd396O7ujmI/pDjymO5+dHd3R7EfFbNpPwqF\nwqvvja2trUqSRD09PWO2SYuVzi4Erlw6PVGUdIm73z1i+W2SFrr7h8fZ5g5JR7v7pSOWnSXpHyUt\ndffRRy1kZisl9ff392vlypU17A4AALPbwMCA2tvbJand3QfSfO6ajjS4+yFJ/ZLeX1lmZlb+/ocT\nbNYi6ZVRy45IcklWy+sDAID81HN64gZJV5rZ75nZWyV9Q6XG4DZJMrNrzez2EevfI+kSM/u0mZ1S\nPspwk6QfufuT0ysfzWL0YTc0N/KMC3kiVM1Ng7vfJekqSVdL+qmk35J0vrs/XV6lVdKyEevfLunz\nktZK+hdJ2yTtknTJtCpHU9m8eXPeJSBF5BkX8kSomq5pyArXNMSnWCyqpaUl7zKQEvKMC3nGpWGu\naQDqxS+kuJBnXMgToWgaAABAEJoGAAAQhKYBmRg9zATNjTzjQp4IRdOATLS1teVdAlJEnnEhT4Ti\n7gkAACLC3RMAACB3NA0AACAITQMyMfqT3tDcyDMu5IlQNA3IxIYNG/IuASkiz7iQJ0LRNCATW7du\nzbsEpIg840KeCEXTgExwS1dcyDMu5IlQNA0AACAITQMAAAhC04BM9Pb25l0CUkSecSFPhKJpQCaK\nxWLeJSBF5BkX8kQoxkgDABARxkgDAIDc0TQAAIAgNA3IxNDQUN4lIEXkGRfyRCiaBmRizZo1eZeA\nFJFnXMgToWgakIlNmzblXQJSRJ5xIU+EomlAJrgLJi7kGRfyRCiaBgAAEISmAQAABKFpQCb6+vry\nLgEpIs+4kCdC0TQgEwMDqQ4lQ87IMy7kiVCMkQYAICKMkQYAALmjaQAAAEFoGgAAQBCaBmQiSZK8\nS0CKyDMu5IlQNA3IxLp16/IuASkiz7iQJ0LRNCATnZ2deZeAFJFnXMgToWgaAABAEJoGAAAQhKYB\nmdi+fXveJSBF5BkX8kQomgZkolAo5F0CUkSecSFPhKqraTCztWb2iJkdNLMHzOydU6z/GjP7ipnt\nMbMXzeznZvb7dVWMprRt27a8S0CKyDMu5IlQR9W6gZl1Sbpe0n+R9KCkHkn3mdlb3H1ogs3+StJi\nSasl/aekpeIoBwAATaXmpkGlJuEWd79Dkszs05J+V9IaSZtHr2xmF0h6r6Q3uvuz5cWD9ZULAADy\nUtP/7ZvZPEntkr5XWealj8m8X1LHBJtdJOknkr5gZo+b2c/M7DozO7rOmgEAQA5qPUWwSNJcSftH\nLd8vqXWCbd6o0pGGt0u6WNLnJH1U0s01vjaa2OrVq/MuASkiz7iQJ0LVc3qiVnMkHZF0ubs/L0lm\n9nlJf2Vmn3H3lzKoATlj4lxcyDMu5IlQtR5pGJJ0WNKSUcuXSHpygm32SdpbaRjKdkkySSdN9mKr\nVq1SkiRVXx0dHWPuKd6xY8e4H7iydu1a9fX1VS0bGBhQkiQaGqq+ZnPjxo3q7e2tWjY4OKgkSbR7\n9+6q5Vu2bNH69eurlhWLRSVJop07d1YtLxQK43bxXV1ds2o/uru7o9gPKY48prsf3d3dUeyHFEce\n092P7u7uKPajYjbtR6FQePW9sbW1VUmSqKenZ8w2abHSJQk1bGD2gKQfufvnyt+bShc2fs3drxtn\n/Ssl3SjpeHcvlpd9SNJfSzp2vCMNZrZSUn9/f79WrlxZ4y4BADB7DQwMqL29XZLa3X0gzeeu57bH\nGyRdaWa/Z2ZvlfQNSS2SbpMkM7vWzG4fsf63JP1C0v80sxVmdo5Kd1n0cWoCAIDmUXPT4O53SbpK\n0tWSfirptySd7+5Pl1dplbRsxPovSDpP0usl/VjS/5L0NypdEIlZYvRhNzQ38owLeSJUXQOW3P3r\n7n6yux/j7h3u/pMRj6129/eNWv/f3f18dz/W3Ze7+waOMswumzePGeGBJkaecSFPhGIqIzJx5513\n5l0CUkSecSFPhKJpQCZaWlryLgEpIs+4kCdC0TQAAIAgNA0AACAITQMyMXqYCZobecaFPBGKpgGZ\naGtry7sEpIg840KeCFXzRMgsMBESAID6NNpESAAAMAvRNAAAgCA0DcjE6E96Q3Mjz7iQJ0LRNCAT\nGzZsyLsEpIg840KeCEXTgExs3bo17xKQIvKMC3kiFE0DMsEtXXEhz7iQJ0LRNAAAgCA0DQAAIAhN\nAzLR29ubdwlIEXnGhTwRiqYBmSgWi3mXgBSRZ1zIE6EYIw0AQEQYIw0AAHJH0wAAAILQNCATQ0ND\neZeAFJFnXMgToWgakIk1a9bkXQJSRJ5xIU+EomlAJjZt2pR3CUgRecaFPBGKpgGZ4C6YuJBnXMgT\noWgaAABAEJoGAAAQhKYBmejr68u7BKSIPONCnghF04BMDAykOpQMOSPPuJAnQjFGGgCAiDBGGgAA\n5I6mAQAABKFpAAAAQWgakIkkSfIuASkiz7iQJ0LRNCAT69aty7sEpIg840KeCEXTgEx0dnbmXQJS\nRJ5xIU+EomkAAABBaBoAAEAQmgZkYvv27XmXgBSRZ1zIE6FoGpCJQqGQdwlIEXnGhTwRqq6mwczW\nmtkjZnbQzB4ws3cGbneWmR0yMwadzzLbtm3LuwSkiDzjQp4IVXPTYGZdkq6XtFHSOyQ9LOk+M1s0\nxXYLJd0u6f466gQAADmr50hDj6Rb3P0Od98t6dOSipLWTLHdNyT9paQH6nhNAACQs5qaBjObJ6ld\n0vcqy7z0MZn3S+qYZLvVkk6R9OX6ygQAAHmr9UjDIklzJe0ftXy/pNbxNjCzN0v6U0kfd/cjNVeI\nKKxevTrvEpAi8owLeSLUjN49YWZzVDolsdHd/7OyeCZfE42JiXNxIc+4kCdC1do0DEk6LGnJqOVL\nJD05zvoLJJ0haWv5rolDkv5E0ulm9rKZnTvZi61atUpJklR9dXR0jLmneMeOHeN+4MratWvV19dX\ntWxgYEBJkmhoaKhq+caNG9Xb21u1bHBwUEmSaPfu3VXLt2zZovXr11ctKxaLSpJEO3furFpeKBTG\n7eK7urpm1X50d3dHsR9SHHlMdz+6u7uj2A8pjjymux/d3d1R7EfFbNqPQqHw6ntja2urkiRRT0/P\nmG3SYqVLEmrYwOwBST9y98+VvzdJg5K+5u7XjVrXJK0Y9RRrJf2OpEsk7XH3g+O8xkpJ/f39/Vq5\ncmVN9QEAMJsNDAyovb1dktrdPdURB0fVsc0Nkm4zs35JD6p0N0WLpNskycyulXSCu19Rvkjy30Zu\nbGZPSXrR3XdNp3AAAJCtmq9pcPe7JF0l6WpJP5X0W5LOd/eny6u0SlqWWoWIwujDbmhu5BkX8kSo\nui6EdPevu/vJ7n6Mu3e4+09GPLba3d83ybZfdnfOOcwymzdvzrsEpIg840KeCMVnTyATd955Z94l\nIEXkGRfyRCiaBmSipaUl7xKQIvKMC3kiFE0DAAAIQtMAAACC0DQgE6OHmaC5kWdcyBOhaBqQiba2\ntrxLQIrIMy7kiVA1T4TMAhMhAQCoz0xOhORIAwAACELTAAAAgtA0IBOjP+kNzY0840KeCEXTgExs\n2LAh7xKQIvKMC3kiFE0DMrF169a8S0CKyDMu5IlQNA3IBLd0xYU840KeCEXTAAAAgtA0AACAIDQN\nyERvb2/eJSBF5BkX8kQomgZkolgs5l0CUkSecSFPhGKMNAAAEWGMNAAAyB1NAwAACELTgEwMDQ3l\nXQJSRJ5xIU+EomlAJtasWZN3CUgRecaFPBGKpgGZ2LRpU94lIEXkGRfyRCiaBmSCu2DiQp5xIU+E\nomkAAABBaBoAAEAQmgZkoq+vL+8SkCLyjAt5IhRNAzIxMJDqUDLkjDzjQp4IxRhpAAAiwhhpAACQ\nO5oGAAAQhKYBAAAEoWlAJpIkybsEpIg840KeCEXTgEysW7cu7xKQIvKMC3kiFE0DMtHZ2Zl3CUgR\necaFPBGKpgEAAAShaQAAAEFoGpCJ7du3510CUkSecSFPhKqraTCztWb2iJkdNLMHzOydk6z7YTPb\nYWZPmdlzZvZDM+ME2ixTKBTyLgEpIs+4kCdC1dw0mFmXpOslbZT0DkkPS7rPzBZNsMk5knZIulDS\nSknfl3SPmZ1WV8VoStu2bcu7BKSIPONCnghVz5GGHkm3uPsd7r5b0qclFSWtGW9ld+9x96+6e7+7\n/6e7f1HSf0i6qO6qAQBA5mpqGsxsnqR2Sd+rLPPSJ17dL6kj8DlM0gJJz9Ty2gAAIF+1HmlYJGmu\npP2jlu+X1Br4HOslzZd0V42vDQAAcpTp3RNmdrmkP5H0MXcfyvK1ka/Vq1fnXQJSRJ5xIU+EqrVp\nGJJ0WNKSUcuXSHpysg3N7DJJt6rUMHw/5MVWrVqlJEmqvjo6OsbcHrRjx45xZ6evXbtWfX19VcsG\nBgaUJImGhqp7lo0bN6q3t7dq2eDgoJIk0e7du6uWb9myRevXr69aViwWlSSJdu7cWbW8UCiM+wPZ\n1dU1q/ajs7Mziv2Q4shjuvvR2dkZxX5IceQx3f2oTIRs9v2omE37USgUXn1vbG1tVZIk6unpGbNN\nWqx0SUI8aD37AAAI+ElEQVQNG5g9IOlH7v658vcmaVDS19z9ugm26Zb0TUld7n5vwGuslNTf39+v\nlStX1lQfAACz2cDAgNrb2yWp3d0H0nzuo+rY5gZJt5lZv6QHVbqbokXSbZJkZtdKOsHdryh/f3n5\nsc9K+rGZVY5SHHT3X02regAAkJmamwZ3v6s8k+FqlU5LPCTpfHd/urxKq6RlIza5UqWLJ28uf1Xc\nrglu0wQAAI2nrgsh3f3r7n6yux/j7h3u/pMRj6129/eN+P533H3uOF80DLPI6HN1aG7kGRfyRCg+\newKZ2Lx5c94lIEXkGRfyRCiaBmTizjvvzLsEpIg840KeCEXTgEy0tLTkXQJSRJ5xIU+EomkAAABB\naBoAAEAQmgZkYvQENDQ38owLeSIUTQMy0dbWlncJSBF5xoU8EarmMdJZYIw0AAD1mckx0hxpAAAA\nQWgaAABAEJoGZGL0x8OiuZFnXMgToWgakIkNGzbkXQJSRJ5xIU+EomlAJrZu3Zp3CUgRecaFPBGK\npgGZ4JauuJBnXMgToWgaAABAEJoGAAAQhKYBmejt7c27BKSIPONCnghF04BMFIvFvEtAisgzLuSJ\nUIyRBgAgIoyRBgAAuaNpAAAAQWgakImhoaG8S0CKyDMu5IlQNA3IxJo1a/IuASkiz7iQJ0LRNCAT\nmzZtyrsEpIg840KeCEXTgExwF0xcyDMu5IlQNA0AACAITQMAAAhC04BM9PX15V0CUkSecSFPhKJp\nQCYGBlIdSoackWdcyBOhGCMNAEBEGCMNAAByR9MAAACC0DQAAIAgNA3IRJIkeZeAFJFnXMgToWga\nkIl169blXQJSRJ5xIU+EomlAJjo7O/MuASkiz7iQJ0LRNAAAgCA0DQAAIAhNAzKxffv2vEtAisgz\nLuSJUHU1DWa21sweMbODZvaAmb1zivXPNbN+M3vRzP7dzK6or1w0q97e3rxLQIrIMy7kiVA1Nw1m\n1iXpekkbJb1D0sOS7jOzRROsf7KkeyV9T9Jpkm6S9E0zO6++ktGMFi9enHcJSBF5xoU8EaqeIw09\nkm5x9zvcfbekT0sqSlozwfp/KOnn7r7B3X/m7jdL+uvy8wAAgCZRU9NgZvMktat01ECS5KVPvLpf\nUscEm51Zfnyk+yZZv+EUCoWGeb5atg1Zd6p1Jnt8osfS/vdKG3nW9thsy3M6z5l2nlOtR54z+5y1\nbjeTP6ONkmetRxoWSZoraf+o5fsltU6wTesE67/OzF5b4+vngjeZ2h6bbb+UyDNfzfomQ9MwvmbN\nM3T9Zm8ajsr01cIdLUm7du3Kuw5J0nPPPZfq581P5/lq2TZk3anWmezxiR4bb/mDDz6Y6r/hdJAn\nec7Uc6ad51TrkefMPmet283kz2gty0e8dx49ZdE1stLZhcCVS6cnipIucfe7Ryy/TdJCd//wONv8\ng6R+d//8iGW/L+lGd3/DBK9zuaS/DC4MAACM9nF3/1aaT1jTkQZ3P2Rm/ZLeL+luSTIzK3//tQk2\n+ydJF45a1llePpH7JH1c0h5JL9ZSIwAAs9zRkk5W6b00VTUdaZAkM7tU0m0q3TXxoEp3QXxU0lvd\n/Wkzu1bSCe5+RXn9kyX9i6SvS/pzlRqM/y5plbuPvkASAAA0qJqvaXD3u8ozGa6WtETSQ5LOd/en\ny6u0Slo2Yv09Zva7km6U9FlJj0v6JA0DAADNpeYjDQAAYHbisycAAEAQmgYAABCkKZsGM/ugme02\ns5+Z2SfzrgfTY2bfNrNnzOyuvGvB9JjZSWb2fTP7VzN7yMw+mndNmB4zW2hmPzazATP7ZzP7g7xr\nwvSZ2TFmtsfMNte0XbNd02BmcyX9m6TflvS8pAFJ73b3X+ZaGOpmZudIWiDpCne/NO96UD8za5V0\nvLv/s5ktkdQv6c3ufjDn0lCn8m31r3X3F83sGEn/Kqmd37nNzcyukfQmSY+5+4bQ7ZrxSMO7JP1f\nd3/S3Z+X9LcqzX1Ak3L3/6NSA4gmV/65/Ofy3/dLGpJ0XL5VYTq8pDIv55jyn5ZXPZg+M/t1Sb8h\n6e9q3bYZm4YTJO0d8f1eSSfmVAuACZhZu6Q57r53ypXR0MqnKB6SNCjpOnd/Ju+aMC1flfRfVUfz\nl2nTYGbvNbO7zWyvmR0xs2Scddaa2SNmdtDMHjCzd2ZZI8KRZ1zSzNPMjpN0u6QrZ7puTCytTN39\nOXc/XdIpkj5uZouzqB/V0sizvM3P3P3/VRbVUkPWRxrmqzQM6jOSxlxMYWZdkq6XtFHSOyQ9LOm+\n8jCpiicknTTi+xPLy5C9NPJE40glTzN7jaT/LelP3f1HM100JpXqz2h5iN/Dkt47UwVjUmnkeaak\ny8zs5yodcfgDM/tScAXunsuXpCOSklHLHpB004jvTaUJkhtGLJsr6WeSlko6VtIuSW/Iaz/4ml6e\nIx47V9Jf5b0ffE0/T0kFSf8t733gK51MJR0v6djy3xeq9LEAb897f2b713R/55Yfv0LS5lpet2Gu\naSh/gma7pO9Vlnlpr+6X1DFi2WFJfyzp71W6c+KrzlW8DSc0z/K635W0TdKFZjZoZu/OslZMLTRP\nMztL0sckXWxmPy3fpvf2rOvF1Gr4GV0u6R/N7KeS/kGlN6V/zbJWTK2W37nTUfNnT8ygRSodRdg/\navl+la7yfJW73yvp3ozqQn1qyfO8rIpC3YLydPcfqLF+r2BioZn+WKVD3Whswb9zK9z99lpfpGGO\nNAAAgMbWSE3DkKTDKn1y5khLJD2ZfTmYJvKMC3nGh0zjkkmeDdM0uPshlabHvb+yrDyJ7P2SfphX\nXagPecaFPONDpnHJKs9Mzz2a2XxJv67h+0LfaGanSXrG3R+TdIOk28ysX9KDknoktUi6Lcs6EYY8\n40Ke8SHTuDREnhnfIvLbKt0mcnjU15+PWOczkvZIOijpnySdkfetLXyR52z4Is/4vsg0rq9GyLPp\nPrAKAADko2GuaQAAAI2NpgEAAAShaQAAAEFoGgAAQBCaBgAAEISmAQAABKFpAAAAQWgaAABAEJoG\nAAAQhKYBAAAEoWkAAABBaBoAAEAQmgYAABDk/wNZKamExrfjsgAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAg0AAAFjCAYAAAC+DjgjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3Xt8FfWd//HXB0UtiFRFiSJ46wVqW5R4i/cbUbLb0Va3\nFO2vNHFtrUHb1MJ217YBH96CVdcCttpmK+22gW67y2K3QsR6i/fmiPVCbFUwBRUN3qgBRfj+/vie\nmBshc5LJzDmT9/PxOA/I98yc85l8OMmHz3znO+acQ0RERKQ3Q5IOQERERAqDigYREREJRUWDiIiI\nhKKiQUREREJR0SAiIiKhqGgQERGRUFQ0iIiISCgqGkRERCSUnZMOYHvMbG/gTGANsDnZaERERArK\nbsBBwHLn3IYoXzgviwZ8wfCrpIMQEREpYBcAv47yBfO1aFgD8J//+Z9MmDAh4VCgqqqKm266KS9e\nL5d9w2zb2zY7er6n57Y3HvX3sD+UT+VzoF4z6nz2tp3yObCvmet+A/kZzWV81apVfPnLX4bs79Io\n5WvRsBlgwoQJTJo0KelYGDlyZKRx9Of1ctk3zLa9bbOj53t6bnvjUX8P+0P5VD4H6jWjzmdv2ymf\nA/uaue43kJ/RXMezIj+9v9Ps2bOjfs1+mzNnzn7A17/+9a+z3377JR0OAJ/5zGfy5vVy2TfMtr1t\ns6Pne3qu6/hNN93ERRdd1GsscVE+c3tusOWzP68ZdT572075HNjXDLPfpk2wYgXsvTccffTAfUbD\njr/yyivcdtttALfNnj37lV4DyoHl410uzWwS0NjY2Jg31a/0z5gxY1i3bl3SYUhElM90UT5z9/zz\n8Ic/wJ13wr33wubN8OMfw8UXJx0ZZDIZiouLAYqdc5koXztfT09IymT/AUtKKJ/ponz2btMmuO++\n9kLh+edhl13gpJPg6qthyhQYPz7pKAeeigaJxbRp05IOQSKkfKaL8rl9L7zgC4Q//AHuucd3E8aN\ng7IyuOEGOO002H33pKOMl05PiIiI4IuC++7zhcKdd8Jf/gJDh8KJJ/pOQlkZTJgAZklHumM6PSEi\nIjIAXnyxvUj44x/9aYixY32RMHeu7yaMGJF0lPlDy0hLLMrLy5MOQSKkfKbLYMrn5s1QXw9VVX4O\nwqGHwre+Be++C3PmwFNPwUsvwa23wtlnq2DoSp0GiUVpaWnSIUiElM90SXs+V6/u3E1obYUxY/zp\nhmuvhdNPhz32SDrKwqA5DSIikirvvQf3398+ifG552DnneH4432hMGUKfPrT+T83oa/yak6DmZ0I\nzASKgf2Ac5xzS3vZZxegGr8OdhHwMnClc+72XN9fRESkq5dear8c8u6727sJU6bANdfAGWeomxCF\nvpyeGA6sBGqB/w65z38B+wDlwAv4YkPzKUREpE/eew8aGtoLhVWrYKed4IQT4Ac/8MXCZz6T3m5C\nUnIuGpxzy4BlAGa9p8PMzgJOBA5xzr2VHW7O9X2lsDU0NHDCCSckHYZERPlMl0LJZ3Nz+ymHu+/2\nkxf33x/OOguuvBImT4aRI5OOMt3i+N/+54A/Af9iZmvN7Dkzu97MdovhvSVPzJ07N+kQJELKZ7rk\naz7ff99PXPzOd+Cww+DAA6GyEt58E664AlauhLVrobYWzjtPBUMc4rh64hB8p2EzcA4wCvgxsBdw\nYQzvL3lg0aJFSYcgEVI+0yWf8vm3v3XuJvz977Dffr6bMGeOn5vw0Y8mHeXgFUfRMATYBpzvnPs7\ngJl9G/gvM7vEOfdeDDFIwoYNG5Z0CBIh5TNdkszn++/Dgw+2FwrPPOPnJpSUwL/+q7/aYeJEzU3I\nF3GcnngFWNdWMGStAgw4YEc7lpWVEQRBp0dJSQlLlizptF19fT1BEHTbv7Kyktra2k5jmUyGIAho\naWnpNF5dXU1NTU2nsebmZoIgoKmpqdP4vHnzmDlzZqex1tZWgiCgoaGh03hdXd12F06ZOnWqjkPH\noePQcQzK41i7Fn76U5gwoZoRI2o47TT45S/hqKPglluamTw54Kc/beLf/g0OP9wXDPl4HG2SzEdd\nXd2HvxuLiooIgoCqqqpu+0SlX+s0mNk2ernk0swuAm4C9nXOtWbHzgZ+C+y+vU6D1mkQEUmPLVs6\ndxOefhqGDPHdhLZ7Okyc6Mek/wZynYacU2Rmw81sopkdnh06JPv12Ozz15rZwg67/BrYAPzczCaY\n2UnAXKBWpyYGj66VtRQ25TNdBiKf69bBz34GX/gC7L03nHoqLFwIxcWweDG0tPhLJq+4Ao44QgVD\noejLnIYjgXsAl33ckB1fCFTgF28a27axc+5dM5sMzAMexxcQi4Hv9z1sKTTjxo1LOgSJkPKZLlHk\nc8sWeOih9m7CU0/5QuDYY2HWLN9RUHFQ+LSMtIiI9MnLL7ff0+Guu+Cdd2CffXyBMGUKlJbCXnsl\nHeXgk1fLSIuIyOC0ZQs8/HB7ofDkk36S4jHHwOWX+7kJkyapm5BmKhpERKRHL78My5a1dxPeftt3\nE848E/7lX3w3Ye+9k45S4qKiQWLR1NTE+PHjkw5DIqJ8pkvHfH7wQeduwsqVvptw9NHw7W/70w7F\nxeomDFZKu8Ri1qxZSYcgEVI+0+Wyy2bx85/DF7/ouwgnneTXUfj0p+FXv4LXXoNHHvE3gjrqKBUM\ng5k6DRKL+fPnJx2CREj5LGwffACPPtp+pcMTT8xnxQpfEHzrW76bcOSRKg6kOxUNEgtdopcuymfh\nWb++fW5Cfb2/6dPee/u5CZdfPo7SUt9lENkRFQ0iIim0dWt7N+HOO6Gx0c9NOPJIuPRS30046ih/\nnweRsFQ0iIikxPr1sHy5LxKWL/fdhL328t2Eb37T/7nvvklHKYVMZ6wkFl1v5iKFTfnMD1u3+isd\nfvAD30EoKoLp0+H552HGDL9C42uvwa9/Df/v//VcMCifEpY6DRKL1tbWpEOQCCmfyXnttc7dhDfe\ngD339F2Eyy7zf44endtrKp8SlpaRFhHJY1u3wuOPt89N+NOfwDm/8mJZmZ+bcPTRsLP+CyhZWkZa\nRGQQaWnxXYQ//MH/uWGD7yaUlkJlpe8mFBUlHaUMRioaREQStnWr7yC0dRMef7y9m3Dxxb6joG6C\n5AP9E5RYtLS0MGrUqKTDkIgon/3X0uLXS2jrJrS0wMiRvovwjW/AWWfF101QPiUsXT0hsaioqEg6\nBImQ8pm7bdvgscdgzhw49lh/JcMFF8Azz8DXvgYPPOALh8WL4atfjff0g/IpYanTILGYPXt20iFI\nhJTPcDZs6NxNeP11302YPNkXCmedBfvvn3SUyqeEp6JBYqGrYNJF+dy+bdsgk2m/p8Njj/mxiRPh\nwgv93IRjj4WhQ5OOtDPlU8JS0SAi0g9vvNG5m/Daa7DHHr6bcNtt/pLIfOgmiERBRYOISA62bYMn\nnmi/0uGRR/zYZz8L5eW+SDjuuPzrJohEQRMhJRa1tbVJhyARGmz5fPNNP0Fx+nTfNTjySJg7109W\nvO02WLsWnnwSrrsOTj658AqGwZZP6TsVDRKLTCbSRckkYWnPZ9vchKuvhhNOgFGj4Etf8h2G6dPh\n3nv9JMff/c7PVRgzJumI+yft+ZToaBlpERF8N+Guu9pPO6xfDyNGwBln+FMOU6bAAQckHaVI77SM\ntIhIxJyDlSvbi4SHH/YrM3760/CVr/gi4fjjYZddko5UJH+oaBCRQeOtt9q7CcuWwSuvwO67+27C\nLbf4dRPGjUs6SpH8paJBRFLLOT9Bsa2b8NBDvpvwqU/B+ef7bsKJJ6qbIBKWJkJKLIIgSDoEiVA+\n5/Ptt+G3v/UTFA84AI44wk9o3GsvWLAA1qzxSzf/8Idw+ukqGCC/8yn5JedOg5mdCMwEioH9gHOc\nc0tD7ns8cC/wlHNOMxwHkRkzZiQdgkQon/LpHDz1lF9cqa2b8MEHMGGCv+KhrMxfAbHrrklHmr/y\nKZ+S3/pyemI4sBKoBf477E5mNhJYCKwARvfhfaWAlZaWJh2CRCjpfL7zDqxY0X7aYd06GDbMdw5+\n9CN/2uGggxINsaAknU8pHDkXDc65ZcAyADOzHHb9CfArYBtwdq7vKyKDl3Pw9NPtRUJDg+8mjB8P\nX/yiLxJOOkndBJGBFstESDMrBw4GLgC+H8d7ikhh27ixczdh7VrfTTjtNHUTRJIy4BMhzezjwDXA\nBc65bQP9fpKflixZknQIEqGByKdzfoLi9df7wmCvveALX4D774fzzvM3hdqwAe64A77xDRUMUdLn\nU8Ia0KLBzIbgT0lUO+deaBseyPeU/FRXV5d0CBKhqPK5cSMsWQJf/zoceKBfWKm62ncU/v3f4YUX\noKkJbrrJ3zVyt90ieVvpQp9PCWugOw0jgCOB+Wa2xcy24E9PHG5m75vZKTvauaysjCAIOj1KSkq6\nVcX19fXbvWSosrKy241YMpkMQRDQ0tLSaby6upqamppOY83NzQRBQFNTU6fxefPmMXPmzE5jra2t\nBEFAQ0NDp/G6ujrKy8u7xTZ16tRBdRyLFy9OxXFAOvLR3+NYvHhxn47jgQcaePbZ9ssd99yzjs9/\nvpx77/VdhWXL/K2mhw+fypgxSzjkkIE9DkhHPvp7HIsXL07FcbQZTMdRV1f34e/GoqIigiCgqqqq\n2z5R6de9J8xsGzu45DI7UXJCl+FK4FTgXGCNc27TdvbTvSdEUuLvf4e7726fm9Dc7DsGp53Wfk+H\nQw9NOkqR9Mire0+Y2XDgY7SfZjjEzCYCbzjn/mZm1wL7O+emO1+RPNtl/9eAzc65Vf2MXUTykHOw\nalV7kfDAA/D++/Dxj8M55/gi4eST4SMfSTpSEclVX66eOBK4B3DZxw3Z8YVABVAEjI0kOhEpCO++\nC3/8Y/sCSy+95LsJp5ziT0VMmQIf+1jSUYpIf+U8p8E5d59zbohzbqcuj4rs8+XOudN2sP8crQY5\n+GzvvJwUrvLycpqa4MYb/QTFvfaCIPA3gwoCXzy88YYvIC69VAVDvtPnU8LSDaskFlpxrvC9+y7c\nc48vBH7/+1Juv90vpnTqqf4yySlT/CkIKTz6fEpYKhokFtOmTUs6BMmRc/CXv7TPTbjvPnjvPTjk\nEPjSl6YxZYo//TBsWNKRSn/p8ylhqWgQkQ+1trZ3E/7wB1i92ncTTjkFamrauwk5LSAvIqmhokFk\nEHMO/vrX9m7Cvff6bsJBB/m7Q06Z4k8/DB+edKQikg8GfBlpEaDboiWSnNZW30W49FLfNfjkJ2HW\nLNi6Fa65xl8u+eKLsGAB/OM/br9gUD7TRfmUsNRpkFjMnTuXE044IekwBq2u3YTNm/2yzR27Cbvv\nHv71lM90UT4lLBUNEotFixYlHcKgsmmTLw7aCoXnn4ehQ/3to6++2hcK48f3fW6C8pkuyqeEpaJB\nYjFMU+wH3PPPtxcJ99zjuwnjxvluwg03+GWbc+km7IjymS7Kp4SlokGkQG3a5C+DbCsU/vpX3004\n8US46irfTZgwQVc6iEh0VDSIFJAXX2xfqvmee3zhMHasLxCuv953E0aMSDpKEUkrXT0hseh6K1gJ\nZ/NmqK+Hqip/lcOhh/q/b9oEV14JTz/t7/Nw661w9tnxFQzKZ7oonxKWOg0Si3HjxiUdQsFYvbpz\nN6G1FQ44wHcTrrsOTj8d9tgj2RiVz3RRPiUs83evzi9mNglobGxsZNIk3dtK0u299+D++9tXYXzu\nOdh5ZzjhBF8olJXBYYdpboKIhJPJZCguLgYods5lonxtdRpEErBmTXuR8Mc/+m7CmDG+SLjmGjjj\njOS7CSIiXaloEInBe+/BAw+0FwpNTb6bcPzx8IMf+G7Cpz+tboKI5DcVDRKLpqYmxo8fn3QYsXrp\npfbLIe++299aev/9fTfhqqt8N2HkyKSj7JvBmM80Uz4lLF09IbGYNWtW0iEMuPff98XBd77j5yAc\ndBDMmAFvvgnf+x6sXAlr18LPfgbnnlu4BQMMjnwOJsqnhKVOg8Ri/vz5SYcwIJqbO3cT/v532G8/\n302YM8d3Ez760aSjjF5a8zlYKZ8SlooGiUVaLul6/31oaGifm/Dss7DTTnDccfBv/+aLhYkT0z83\nIS35FE/5lLBUNIj04m9/a+8mrFjhuwlFRb5AmD0bJk9OZzdBRKQrFQ0iXWzZAg8+2N5NePppGDIE\nSkrgu9/1VzpMnOjHREQGE/3Yk1jU1NQkHcIOrV0LP/0pfOELsPfecOqpsHAhFBfDokXQ0uJPS1xx\nBRxxhAqGfM+n5Eb5lLDUaZBYtLa2Jh1CJx27CXfeCU895QuBY4+FWbN8N+Hww1Uc9CTf8in9o3xK\nWFpGWgaNdetg2TJ/ymHFCnjnHdh3XzjrLF8kTJ4Me+2VdJQiIv2jZaRF+mDLFnj44fabP/35z75z\ncMwxMHOmn8ioUw0iIuHlXDSY2YnATKAY2A84xzm3dAfbfx74BnA4sCvwDDDbOVffp4hFduDll9u7\nCXfd5bsJ++zjuwnf/S6Ulvo5CyIikru+dBqGAyuBWuC/Q2x/ElAP/CvwFlAB3GFmRzvnnuzD+0sB\namlpYdSoUZG/7gcf+G5C25UOTz7p10g45hi4/HLfTSguVjchagOVT0mG8ilh5Vw0OOeWAcsAzHpf\nwsY5V9Vl6AozOxv4HKCiYZCoqKhg6dIeG1I5eeUV3024807fTXjrLRg1yncTZs3y3QT9/BtYUeZT\nkqd8Slixz2nIFhojgDfifm9JzuzZs/u87wcfwCOPtF/p8MQTvptw9NHwrW/5SYzqJsSrP/mU/KN8\nSlhJTISciT/F8ZsE3lsSkutVMK++2nluwltv+bkIZ57pTzuceaa6CUnSVU3ponxKWLEWDWZ2PvB9\nIHDOtcT53pLftm6FRx9tn5uQyfhuwlFHwTe/6ecmHHmkv8+DiIgkI7aGrpl9CbgN+Cfn3D1h9ikr\nKyMIgk6PkpISlixZ0mm7+vp6giDotn9lZSW1tbWdxjKZDEEQ0NLSuWaprq7utipac3MzQRDQ1NTU\naXzevHnMnDmz01hraytBENDQ0NBpvK6ujvLy8m6xTZ06ddAfx/r1MGdOhjFjAvbeu4Xjj4cf/xjG\nj4dzzqnmiitqePRRf3+HY46Bdevy8zggHfnQceg4dByFdxx1dXUf/m4sKioiCAKqqrpOJYxOvxZ3\nMrNt9HLJZXa7acDPgKnOud+HeF0t7pQytbW1fPWrF/LYY+3rJjQ2+ueOOsp3EqZM8X9XNyH/1dbW\ncuGFFyYdhkRE+UyXgVzcKedOg5kNN7OJZnZ4duiQ7Ndjs89fa2YLO2x/PrAQuBx43MxGZx97RHEA\nkt9eew1++Uu47roM++7rbyF9yy3wiU/AL37huw2PPQZz5vglnFUwFIZMJtKfQ5Iw5VPCyrnTYGYn\nA/cAXXdc6JyrMLOfAwc6507Lbn8Pfq2GrhY65yp6eA91GgrU1q3w+OPtcxMaG8E5f3VDWzfhmGNU\nHIiIDJS8WkbaOXcfO+hQOOfKu3x9ah/ikgLy+uuwfLkvEurrYcMG2HNPv17CpZf6Kx1Gj046ShER\n6S/de0JytnUr/OlP7esmPP647yZMmgQXX+zXTTj6aNhZ/7pERFJFP9YllJYW3024806/fsKGDfDR\nj/puwiWX+G5CUVHSUYqIyEDSGnqyXdu2tU9QPOYYfwvpL38ZnnkGvvY1eOABf1pi8WKYPr33gmF7\nlzRJ4VI+00X5lLDUaZAPbdjQuZvQ0gIjR/puwsUX+3s77Ldf3157xowZ0QYriVI+00X5lLBUNAxi\n27b5lRfb1k149FE/N+Hww+Gii/zchGOPjWZuQmlpaf9fRPKG8pkuyqeEpaJhkNmwwV/hcOedvqvw\n2muwxx6+m3DRRb6bsP/+SUcpIiL5SEVDym3b5u8K2bGbsG0bfPazUF7u10047jgYOjTpSEVEJN9p\nImQKvfEGLFrkJyjuv7+/0dP11/v5CLfeCmvXwpNPwnXXwcknx1MwdF3vXQqb8pkuyqeEpaIhBdrm\nJlx1FRx/POyzD0yb5jsM06fDvff60xK/+x388z/DmDHxx1hXVxf/m8qAUT7TRfmUsPp1w6qBomWk\ne/fmm3DXXe0LLK1fDyNGwBln+AmMZ50FBxyQdJQiIhK3vFpGWpLhHKxc2T434eGHfYfhsMPgK1/x\nhcJxx8EuuyQdqYiIpJWKhjz21luduwmvvgq77+67CT/+sZ/EOHZs0lGKiMhgoaIhjzjnJyi23SHy\n4Yf9fR4OO8yvxlhW5ucsqJsgIiJJ0ETIhL39Nvz2t1BR4ScoHnEEXH017L03LFgAL70ETz/tr344\n9dTCLRjKy8t730gKhvKZLsqnhKVOQ8ycgz//uf2Uw4MP+m7Cpz4F55/vTzmccALsumvSkUZLK86l\ni/KZLsqnhKWrJ2Lw9tuwYkX7PR3WrYNhw/zchClT/OPAA5OOUkRE0kBXTxQY5/wphbYrHR58ED74\nAMaPhy9+0c9NOPHE9HUTREQk3VQ0ROSdd+Duu9tPO6xd67sJp50GN9/suwkHH5x0lCIiIn2niZB9\n5Bw89RTMnesnKO69N3zhC3D//XDeef6mUBs2wB13wCWXqGBoaGhIOgSJkPKZLsqnhKWiIQcbN8L/\n/A987Wswbpy/6dPs2TB8uO8mvPgiNDXBTTfB5Mmw225JR5w/5s6dm3QIEiHlM12UTwlLpyd2wDl4\n9tn2uQkNDbBlC3z843DuuX5uwkknqTgIY9GiRUmHIBFSPtNF+ZSwVDTswFtv+W7Crrv6UxA33ujn\nJhx6aNKRFZ5hw4YlHYJE6CMf+UjSIUiE9PmUsFQ07MCee/ruwuGHg35GymC3ceNGfnjFFTx4xx0M\n37KFd4cO5fjPfY7vXH01I0aMSDo8EYmBioZelJQkHYFI8jZu3Mi5JSV8e9UqZm/bhgEOWL5gAef+\n8Y/87uGHVTiIDAKaCCmxmDlzZtIhSD/88Ior+PaqVZyVLRhmAgactW0bVatWccP3vpdwhNIf+nxK\nWCoaJBbjxo1LOgTphwfvuIMzt2378OuO2Txr2zYeXLo0/qAkMvp8Slg5Fw1mdqKZLTWzdWa2zcyC\nEPucYmaNZrbZzP5iZtP7Fq4UqksvvTTpEKSPnHMM37IF6zDWMZsGDNuyhXxckl7C0edTwupLp2E4\nsBK4BH9ac4fM7CDg98DdwETgZuBnZja5D+896OkHc//pe5gbM+PdoUN7/LA74N2hQzGzHrYQkbTI\nuWhwzi1zzv3AOfe/QJifEt8AXnTOzXLOPeecWwD8FqjK9b0Hq40bN1J92WWccfDBnDN2LGccfDDV\nl13Gxo0bkw6tYOh72D/Hf+5zLB+y/R8Xy4YM4YSg14ajiKRAHHMajgVWdBlbDui6hBDaZq2XLFjA\nXWvW8L/r1nHXmjWULFjAuSUlBfNLr6mpKbH3Tsv3MEnfufpqbpwwgTuHDMEBTfgOw51DhnDThAlc\nftVVCUco/ZHk51MKSxxFQxGwvsvYemAPM9N9HnvRddY6FOas9VmzZiX23mn5HiZpxIgR/O7hh3l0\nxgxKDzqIU3bbjdKDDuLRGTN0uWUKJPn5lMKiqyfyXNdZ6x0V0qz1+fPnJ/beafkeJm3EiBHMvvlm\n7lq9mkebmrhr9Wpm33yzCoYUSPLzKYUljqLhVWB0l7HRwDvOufd2tGNZWRlBEHR6lJSUsGTJkk7b\n1dfXE2znnGplZSW1tbWdxjKZDEEQ0NLS0mm8urqampqaTmPNzc0EQdCtdTdv3rxu1zW3trYSBEG3\nu8XV1dVRXl7eLbapU6f2ehxts9ZnALVd9s8AZwM7bd7caWJfPh4H+Eu6ksjHAw880Gnmfx3Q8Sja\nZv6HPQ4o/H9XURzHgQcemIrjgHTko7/H0XbJZaEfR5vBdBx1dXUf/m4sKioiCAKqqgZuyqD1Zya5\nmW0DznHO9fhfNTO7DpjinJvYYezXwEedc2U97DMJaGxsbGTSpEl9ji8Nzjj4YO5as2a7M04dMPmg\ng1ixenXcYRUUfQ9FZDDJZDIUFxcDFDvnMlG+dl/WaRhuZhPN7PDs0CHZr8dmn7/WzBZ22OUn2W1q\nzOyTZnYJcB5wY7+jHwQ0a73/9D0UEYlGX05PHAk8ATTi/6N2A75bPif7fBEwtm1j59wa4B+AM/Dr\nO1QBFzrnul5RIdvRddY6FOas9a6tuzil5XuYT5LMp0RP+ZSwcr5hlXPuPnZQbDjnup2Acc7dDxTn\n+l7SPmv9hu99jxuXLmXYli20Dh3K8UHA7666qmAmobW2tib23mn5HuaTJPMp0VM+Jax+zWkYKJrT\n0DPnnFbe6yd9D0UkzfJqToMkS7/s+k/fQxGRvlHRICIiIqGoaJBYdL2uWQqb8pkuyqeEpaJBYlFR\nUZF0CBIh5TNdlE8JS0WDxGL27NlJhyARUj7TRfmUsFQ0SCx0FUy6KJ/ponxKWCoaREREJBQVDSIi\nIhKKigaJRde7xUlhUz7TRfmUsFQ0SCwymUgXJZOEKZ/ponxKWFpGWkREJEW0jLSIiIgkTkWDiIiI\nhKKiQUREREJR0SCxCIIg6RAkQspnuiifEpaKBonFjBkzkg5BIqR8povyKWGpaJBYlJaWJh2CREj5\nTBflU8JS0SAiIiKhqGgQERGRUFQ0SCyWLFmSdAgSIeUzXZRPCUtFg8Sirq4u6RAkQspnuiifEpaK\nBonF4sWLkw5BIqR8povyKWGpaBAREZFQVDSIiIhIKH0qGsys0sxWm9kmM3vEzI7qZfsLzGylmb1r\nZi+bWa2Z7dW3kEVERCQJORcNZjYVuAGoBo4AngSWm9moHrY/HlgI/BT4FHAecDRwWx9jlgJUXl6e\ndAgSIeUzXZRPCasvnYYq4Fbn3C+cc03AxUArUNHD9scCq51zC5xzLznnHgJuxRcOMkhoxbl0UT7T\nRfmUsHIqGsxsKFAM3N025pxzwAqgpIfdHgbGmtmU7GuMBv4J+L++BCyFadq0aUmHIBFSPtNF+ZSw\ncu00jAJ2AtZ3GV8PFG1vh2xn4cvAYjN7H3gFeBPQHVJEREQKyIBfPWFmnwJuBmYDk4AzgYPxpyhE\nRESkQOQUQ2NQAAAUOklEQVRaNLQAW4HRXcZHA6/2sM93gQedczc65552zt0FXAJUZE9V9KisrIwg\nCDo9SkpKui15Wl9fv937wVdWVlJbW9tpLJPJEAQBLS0tncarq6upqanpNNbc3EwQBDQ1NXUanzdv\nHjNnzuw01traShAENDQ0dBqvq6vb7iSjqVOnDqrjaGhoSMVxQDry0d/jaGhoSMVxQDry0d/jaPuz\n0I+jzWA6jrq6ug9/NxYVFREEAVVVVd32iYr5KQk57GD2CPCoc+6b2a8NaAZ+5Jy7fjvb/xZ43zl3\nfoexEqABGOOc61ZsmNkkoLGxsZFJkyblFJ/kpyAIWLp0adJhSESUz3RRPtMlk8lQXFwMUOycy0T5\n2n05PXEjcJGZfcXMxgM/AYYBtwOY2bVmtrDD9ncA55rZxWZ2cPYSzJvxhUdP3QlJmUWLFiUdgkRI\n+UwX5VPC2jnXHZxzv8muyXAl/rTESuBM59zr2U2KgLEdtl9oZrsDlcAPgbfwV198t5+xSwEZNmxY\n0iFIhJTPdFE+JayciwYA59wtwC09PNftBIxzbgGwoC/vJSIiIvlB954QERGRUFQ0SCy6zhaWwqZ8\npovyKWGpaJBYjBs3LukQJELKZ7oonxJWzpdcxkGXXIqIiPRNvl1yKSIiIoOQigYREREJRUWDxKLr\nUqpS2JTPdFE+JSwVDRKLWbNmJR2CREj5TBflU8JS0SCxmD9/ftIhSISUz3RRPiUsFQ0SC13SlS7K\nZ7oonxKWigYREREJRUWDiIiIhKKiQWJRU1OTdAgSIeUzXZRPCUtFg8SitbU16RAkQspnuiifEpaW\nkRYREUkRLSMtIiIiiVPRICIiIqGoaJBYtLS0JB2CREj5TBflU8JS0SCxqKioSDoEiZDymS7Kp4Sl\nokFiMXv27KRDkAgpn+mifEpYKhokFroKJl2Uz3RRPiUsFQ0iIiISiooGERERCUVFg8SitrY26RAk\nQspnuiifEpaKBolFJhPpomSSMOUzXZRPCUvLSIuIiKRI3i0jbWaVZrbazDaZ2SNmdlQv2+9iZleb\n2Roz22xmL5rZV/sUsYiIiCRi51x3MLOpwA3A14DHgCpguZl9wjnX07Ji/wXsA5QDLwD7oVMjIiIi\nBSXnogFfJNzqnPsFgJldDPwDUAHM7bqxmZ0FnAgc4px7Kzvc3LdwRUREJCk5/W/fzIYCxcDdbWPO\nT4pYAZT0sNvngD8B/2Jma83sOTO73sx262PMUoCCIEg6BImQ8pkuyqeElWunYRSwE7C+y/h64JM9\n7HMIvtOwGTgn+xo/BvYCLszx/aVAzZgxI+kQJELKZ7oonxJWHPMKhgDbgPOdc39yzi0Dvg1MN7Nd\nd7RjWVkZQRB0epSUlLBkyZJO29XX12+3Uq6srOx2/XEmkyEIgm53dauurqampqbTWHNzM0EQ0NTU\n1Gl83rx5zJw5s9NYa2srQRDQ0NDQabyuro7y8vJusU2dOnVQHUdpaWkqjgPSkY/+HkdpaWkqjgPS\nkY/+HkdpaWkqjqPNYDqOurq6D383FhUVEQQBVVVV3faJSk6XXGZPT7QC5zrnlnYYvx0Y6Zz7/Hb2\nuR04zjn3iQ5j44FngE84517Yzj665FJERKQP8uaSS+fcFqAROL1tzMws+/VDPez2ILC/mQ3rMPZJ\nfPdhbU7RioiISGL6cnriRuAiM/tKtmPwE2AYcDuAmV1rZgs7bP9rYAPwczObYGYn4a+yqHXOvdev\n6KVgdG0JSmFTPtNF+ZSwci4anHO/Ab4DXAk8AXwWONM593p2kyJgbIft3wUmAx8FHgd+Cfwv8M1+\nRS4Fpa6uLukQJELKZ7oonxKWlpEWERFJkbyZ0yAiIiKDl4oGERERCUVFg4iIiISiokFisb0FSqRw\nKZ/ponxKWCoaJBZtK85JOiif6aJ8SlgqGiQW06ZNSzoEiZDymS7Kp4SlokFERERCUdEgIiIioaho\nkFh0vXubFDblM12UTwlLRYPEYu7cuUmHIBFSPtNF+ZSwVDRILBYtWpR0CBIh5TNdlE8JS0WDxGLY\nsGG9byQFQ/lMF+VTwlLRICIiIqGoaBAREZFQVDRILGbOnJl0CBIh5TNdlE8JS0WDxGLcuHFJhyAR\nUj7TRfmUsMw5l3QM3ZjZJKCxsbGRSZMmJR2OiIhIwchkMhQXFwMUO+cyUb62Og0iIiISiooGERER\nCUVFg8Siqakp6RAkQspnuiifEpaKBonFrFmzkg5BIqR8povyKWGpaJBYzJ8/P+kQJELKZ7oonxKW\nigaJhS7pShflM12UTwlLRYOIiIiEoqJBREREQulT0WBmlWa22sw2mdkjZnZUyP2ON7MtZhbpYhOS\n/2pqapIOQSKkfKaL8ilh5Vw0mNlU4AagGjgCeBJYbmajetlvJLAQWNGHOKXAtba2Jh2CREj5TBfl\nU8LKeRlpM3sEeNQ5983s1wb8DfiRc27uDvarA/4CbAPOds71uD60lpEWERHpm7xZRtrMhgLFwN1t\nY85XHSuAkh3sVw4cDMzpW5giIiKStJ1z3H4UsBOwvsv4euCT29vBzD4OXAOc4Jzb5hsTIiIiUmgG\n9OoJMxsC/Aqods690DY8kO8p+amlpSXpECRCyme6KJ8SVq5FQwuwFRjdZXw08Op2th8BHAnMz141\nsQX4PnC4mb1vZqfs6M3KysoIgqDTo6SkhCVLlnTarr6+niAIuu1fWVlJbW1tp7FMJkMQBN0+JNXV\n1d1mEDc3NxMEQbd12efNm8fMmTM7jbW2thIEAQ0NDZ3G6+rqKC8v7xbb1KlTB9VxVFRUpOI4IB35\n6O9xVFRUpOI4IB356O9xVFRUpOI42gym46irq/vwd2NRURFBEFBVVdVtn6hENRGyGT8R8vou2xow\noctLVAKnAucCa5xzm7bzHpoImTKZTEa5TBHlM12Uz3QZyImQuc5pALgRuN3MGoHHgCpgGHA7gJld\nC+zvnJuenST5bMedzew1YLNzblV/ApfCoh9I6aJ8povyKWHlXDQ4536TXZPhSvxpiZXAmc6517Ob\nFAFjowtRRERE8kFfOg04524Bbunhue4nYDo/PwddeikiIlJwdO8JiUXXCUVS2JTPdFE+JSwVDRKL\nTEa3G0kT5TNdlE8JK+erJ+KgqydERET6Jm+WkRYREZHBS0WDiIiIhKKiQUREREJR0SCx2N4yrVK4\nlM90UT4lLBUNEosZM2YkHYJESPlMF+VTwlLRILEoLS1NOgSJkPKZLsqnhKWiQUREREJR0SAiIiKh\nqGiQWHS9h70UNuUzXZRPCUtFg8Sirq4u6RAkQspnuiifEpaKBonF4sWLkw5BIqR8povyKWGpaBAR\nEZFQVDSIiIhIKCoaREREJBQVDRKL8vLypEOQCCmf6aJ8SlgqGiQWWnEuXZTPdFE+JSwVDRKLadOm\nJR2CREj5TBflU8JS0SAiIiKhqGgQERGRUFQ0SCwaGhqSDkEipHymi/IpYalokFjMnTs36RAkQspn\nuiifEpaKBonFokWLkg5BIqR8povyKWGpaJBYDBs2LOkQJELKZ7oonxJWn4oGM6s0s9VmtsnMHjGz\no3aw7efNrN7MXjOzt83sITPTRcEiIiIFJueiwcymAjcA1cARwJPAcjMb1cMuJwH1wBRgEnAPcIeZ\nTexTxCIiIpKIvnQaqoBbnXO/cM41ARcDrUDF9jZ2zlU5537onGt0zr3gnLsC+CvwuT5HLQVn5syZ\nSYcgEVI+00X5lLByKhrMbChQDNzdNuacc8AKoCTkaxgwAngjl/eWwjZu3LikQ5AIKZ/ponxKWLl2\nGkYBOwHru4yvB4pCvsZMYDjwmxzfWwrYpZdemnQIEiHlM12UTwkr1qsnzOx84PvAPznnWnrbvqys\njCAIOj1KSkpYsmRJp+3q6+sJgqDb/pWVldTW1nYay2QyBEFAS0vnt6+urqampqbTWHNzM0EQ0NTU\n1Gl83rx53dp5ra2tBEHQbZGUurq67d5BburUqToOHYeOQ8eh49Bx9Os46urqPvzdWFRURBAEVFVV\nddsnKubPLoTc2J+eaAXOdc4t7TB+OzDSOff5Hez7JeBnwHnOuWW9vM8koLGxsZFJkyaFjk9ERGSw\ny2QyFBcXAxQ75zJRvnZOnQbn3BagETi9bSw7R+F04KGe9jOzaUAt8KXeCgZJp67VthQ25TNdlE8J\nqy+nJ24ELjKzr5jZeOAnwDDgdgAzu9bMFrZtnD0lsRC4HHjczEZnH3v0O3opGLNmzUo6BImQ8pku\nyqeEtXOuOzjnfpNdk+FKYDSwEjjTOfd6dpMiYGyHXS7CT55ckH20WUgPl2lK+syfPz/pECRCyme6\nKJ8SVs5FA4Bz7hbglh6eK+/y9al9eQ9JF13SlS7KZ7oonxKW7j0hIiIioahoEBERkVBUNEgsul7D\nLIVN+UwX5VPCUtEgsWhtbU06BImQ8pkuyqeEldPiTnHR4k4iIiJ9kzeLO4mIiMjgpaJBREREQlHR\nILHoeoMXKWzKZ7oonxKWigaJRUWFFv9ME+UzXZRPCUtFg8Ri9uzZSYcgEVI+00X5lLBUNEgsdBVM\nuiif6aJ8SlgqGkRERCQUFQ0iIiISiooGiUVtbW3SIUiElM90UT4lLBUNEotMJtJFySRhyme6KJ8S\nlpaRFhERSREtIy0iIiKJU9EgIiIioahoEBERkVBUNEgsgiBIOgSJkPKZLsqnhKWiQWIxY8aMpEOQ\nCCmf6aJ8SlgqGiQWpaWlSYcgEVI+00X5lLBUNIiIiEgoKhpEREQkFBUNEoslS5YkHYJESPlMF+VT\nwupT0WBmlWa22sw2mdkjZnZUL9ufYmaNZrbZzP5iZtP7Fq4UqpqamqRDkAgpn+mifEpYORcNZjYV\nuAGoBo4AngSWm9moHrY/CPg9cDcwEbgZ+JmZTe5byFKI9tlnn6RDkAgpn+mifEpYfek0VAG3Oud+\n4ZxrAi4GWoGKHrb/BvCic26Wc+4559wC4LfZ1xEREZECkVPRYGZDgWJ81wAA5+94tQIo6WG3Y7PP\nd7R8B9vnnbq6urx5vVz2DbNtb9vs6Pmenov6+xU15TO35wZbPvvzmlHns7ftlM+Bfc1c9xvIz2i+\n5DPXTsMoYCdgfZfx9UBRD/sU9bD9Hma2a47vnwj9ksntucH2Q0n5TFah/pJR0bB9hZrPsNsXetGw\nc6zvFt5uAKtWrUo6DgDefvvtSO8335/Xy2XfMNv2ts2Onu/pue2NP/bYY5F+D/tD+VQ+B+o1o85n\nb9spnwP7mrnuN5Cf0VzGO/zu3K3XoHNk/uxCyI396YlW4Fzn3NIO47cDI51zn9/OPvcBjc65b3cY\n+ypwk3Nuzx7e53zgV6EDExERka4ucM79OsoXzKnT4JzbYmaNwOnAUgAzs+zXP+pht4eBKV3GSrPj\nPVkOXACsATbnEqOIiMggtxtwEP53aaRy6jQAmNkXgdvxV008hr8K4jxgvHPudTO7FtjfOTc9u/1B\nwFPALcB/4AuMfwfKnHNdJ0iKiIhInsp5ToNz7jfZNRmuBEYDK4EznXOvZzcpAsZ22H6Nmf0DcBNw\nGbAWuFAFg4iISGHJudMgIiIig5PuPSEiIiKhqGgQERGRUAqyaDCzfzSzJjN7zswuTDoe6R8z+28z\ne8PMfpN0LNI/ZnaAmd1jZs+Y2UozOy/pmKR/zGykmT1uZhkz+7OZ/XPSMUn/mdlHzGyNmc3Nab9C\nm9NgZjsBzwInA38HMsAxzrk3Ew1M+szMTgJGANOdc19MOh7pOzMrAvZ1zv3ZzEYDjcDHnXObEg5N\n+ih7Wf2uzrnNZvYR4BmgWD9zC5uZXQUcCvzNOTcr7H6F2Gk4GnjaOfeqc+7vwP/h132QAuWcux9f\nAEqBy34u/5z9+3qgBdgr2aikP5zXtl7OR7J/WlLxSP+Z2ceATwJ35rpvIRYN+wPrOny9DhiTUCwi\n0gMzKwaGOOfW9bqx5LXsKYqVQDNwvXPujaRjkn75IfCv9KH4i7VoMLMTzWypma0zs21mFmxnm0oz\nW21mm8zsETM7Ks4YJTzlM12izKeZ7QUsBC4a6LilZ1Hl1Dn3tnPucOBg4AIz2yeO+KWzKPKZ3ec5\n59zzbUO5xBB3p2E4fjGoS4BukynMbCpwA1ANHAE8CSzPLibV5mXggA5fj8mOSfyiyKfkj0jyaWa7\nAP8DXOOce3Sgg5YdivQzml3E70ngxIEKWHYoinweC3zJzF7Edxz+2cy+FzoC51wiD2AbEHQZewS4\nucPXhl9BclaHsZ2A54D9gN2BVcCeSR2HHv3LZ4fnTgH+K+nj0KP/+QTqgB8kfQx6RJNTYF9g9+zf\nR+JvC3BY0scz2B/9/ZmbfX46MDeX982bOQ3ZO2gWA3e3jTl/VCuAkg5jW4HLgXvxV0780GkWb94J\nm8/stncBi4EpZtZsZsfEGav0Lmw+zex44J+Ac8zsiexleofFHa/0LofP6IHAA2b2BHAf/pfSM3HG\nKr3L5Wduf+R874kBNArfRVjfZXw9fpbnh5xzvwd+H1Nc0je55HNyXEFJn4XKp3PuQfLr54r0LGxO\nH8e3uiW/hf6Z28Y5tzDXN8mbToOIiIjkt3wqGlqArfg7Z3Y0Gng1/nCkn5TPdFE+00c5TZdY8pk3\nRYNzbgt+9bjT28ayK5GdDjyUVFzSN8pnuiif6aOcpktc+Yz13KOZDQc+Rvt1oYeY2UTgDefc34Ab\ngdvNrBF4DKgChgG3xxmnhKN8povymT7KabrkRT5jvkTkZPxlIlu7PP6jwzaXAGuATcDDwJFJX9qi\nh/I5GB7KZ/oeymm6HvmQz4K7YZWIiIgkI2/mNIiIiEh+U9EgIiIioahoEBERkVBUNIiIiEgoKhpE\nREQkFBUNIiIiEoqKBhEREQlFRYOIiIiEoqJBREREQlHRICIiIqGoaBAREZFQVDSIiIhIKCoaRERE\nJJT/D/4qo+iCHdmnAAAAAElFTkSuQmCC\n", "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -498,6 +524,15 @@ "plt.ylim(0)\n", "plt.grid()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] } ], "metadata": { @@ -520,11 +555,16 @@ "version": "2.7.12" }, "latex_envs": { + "LaTeX_envs_menu_present": true, "bibliofile": "biblio.bib", "cite_by": "apalike", "current_citInitial": 1, "eqLabelWithNumbers": true, - "eqNumInitial": 0 + "eqNumInitial": 0, + "labels_anchors": false, + "latex_user_defs": false, + "report_style_numbering": false, + "user_envs_cfg": false } }, "nbformat": 4, From 5052845f461beda41a8ee451883353d5cec10d0b Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Tue, 6 Feb 2018 19:20:45 -0500 Subject: [PATCH 025/272] added saving of ROC curves and accumulated disruptions --- examples/performance_analysis.py | 5 ++++- plasma/utils/performance.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/performance_analysis.py b/examples/performance_analysis.py index 4729a286..31568581 100644 --- a/examples/performance_analysis.py +++ b/examples/performance_analysis.py @@ -34,7 +34,8 @@ #P_thresh_opt = 0.566#0.566#0.92# analyzer.compute_tradeoffs_and_print_from_training() linestyle="-" -analyzer.compute_tradeoffs_and_plot('test',save_figure=save_figure,plot_string='_test',linestyle=linestyle) +P_thresh_range,missed_range,fp_range = analyzer.compute_tradeoffs_and_plot('test',save_figure=save_figure,plot_string='_test',linestyle=linestyle) +np.savez('test_roc.npz',"P_thresh_range",P_thresh_range,"missed_range",missed_range,"fp_range",fp_range) analyzer.compute_tradeoffs_and_plot('train',save_figure=save_figure,plot_string='_train',linestyle=linestyle) analyzer.summarize_shot_prediction_stats_by_mode(P_thresh_opt,'test') @@ -55,6 +56,8 @@ alarms,disr_alarms,nondisr_alarms = analyzer.gather_first_alarms(P_thresh_opt,'test') analyzer.hist_alarms(disr_alarms,'disruptive alarms, P thresh = {}'.format(P_thresh_opt),save_figure=save_figure,linestyle=linestyle) +np.savez('disruptive_alarms_test.npz',"disr_alarms",disr_alarms,"P_thresh_opt",P_thresh_opt) + print('{} disruptive alarms'.format(len(disr_alarms))) print('{} seconds mean alarm time'.format(np.mean(disr_alarms[disr_alarms > 0]))) print('{} seconds median alarm time'.format(np.median(disr_alarms[disr_alarms > 0]))) diff --git a/plasma/utils/performance.py b/plasma/utils/performance.py index 43bd6b5a..35ec16a9 100644 --- a/plasma/utils/performance.py +++ b/plasma/utils/performance.py @@ -551,7 +551,7 @@ def compute_tradeoffs_and_print_from_training(self): def compute_tradeoffs_and_plot(self,mode,save_figure=True,plot_string='',linestyle="-"): correct_range, accuracy_range, fp_range,missed_range,early_alarm_range = self.get_metrics_vs_p_thresh(mode) - self.tradeoff_plot(accuracy_range,missed_range,fp_range,early_alarm_range,save_figure=save_figure,plot_string=plot_string,linestyle=linestyle) + return self.tradeoff_plot(accuracy_range,missed_range,fp_range,early_alarm_range,save_figure=save_figure,plot_string=plot_string,linestyle=linestyle) def get_prediction_type(self,TP,FP,FN,TN,early,late): if TP: @@ -834,7 +834,9 @@ def tradeoff_plot(self,accuracy_range,missed_range,fp_range,early_alarm_range,sa plt.ylim([0,1]) if save_figure: plt.savefig(title_str + '_roc.png',bbox_inches='tight',dpi=200) + np.savez(title_str + '_roc.npz',"P_thresh_range",P_thresh_range,"missed_range",missed_range,"fp_range",fp_range) print('ROC area ({}) is {}'.format(plot_string,self.roc_from_missed_fp(missed_range,fp_range))) + return P_thresh_range,missed_range,fp_range def get_pred_truth_disr_by_shot(self,shot): if shot in self.shot_list_test: From b6c0f3d28c857da88e30baccf79f305d739c4578 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Tue, 6 Feb 2018 19:22:31 -0500 Subject: [PATCH 026/272] recasting to shotlist --- plasma/utils/performance.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plasma/utils/performance.py b/plasma/utils/performance.py index 43bd6b5a..7ac711cf 100644 --- a/plasma/utils/performance.py +++ b/plasma/utils/performance.py @@ -12,7 +12,7 @@ from plasma.preprocessor.normalize import VarNormalizer as Normalizer from plasma.conf import conf -from plasma.primitives.shots import Shot +from plasma.primitives.shots import Shot,ShotList class PerformanceAnalyzer(): def __init__(self,results_dir=None,shots_dir=None,i = 0,T_min_warn = None,T_max_warn = None, verbose = False,pred_ttd=False,conf=None): @@ -342,8 +342,8 @@ def load_ith_file(self): self.pred_test = dat['y_prime_test'] self.truth_test = dat['y_gold_test'] self.disruptive_test = dat['disruptive_test'] - self.shot_list_test = dat['shot_list_test'][()] - self.shot_list_train = dat['shot_list_train'][()] + self.shot_list_test = ShotList(dat['shot_list_test'][()]) + self.shot_list_train = ShotList(dat['shot_list_train'][()]) self.saved_conf = dat['conf'][()] self.conf['data']['T_warning'] = self.saved_conf['data']['T_warning'] #all files must agree on T_warning due to output of truth vs. normalized shot ttd. for mode in ['test','train']: From 85d4b8b6fdfdfbbd7251b670d56bf72a1ae72c30 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Wed, 7 Feb 2018 04:33:55 -0500 Subject: [PATCH 027/272] backwards compatibility for normalizer (check if signal has is_positive attribute). Plotting options for performance analyzer --- plasma/preprocessor/normalize.py | 7 +- plasma/utils/performance.py | 146 ++++++++++++++++--------------- 2 files changed, 80 insertions(+), 73 deletions(-) diff --git a/plasma/preprocessor/normalize.py b/plasma/preprocessor/normalize.py index 06e9ed3d..032ea4fb 100644 --- a/plasma/preprocessor/normalize.py +++ b/plasma/preprocessor/normalize.py @@ -411,6 +411,7 @@ def get_individual_shot_file(prepath,shot_num,ext='.txt'): def apply_positivity(shot): for (i,sig) in enumerate(shot.signals): - if sig.is_strictly_positive: - #print ('Applying positivity constraint to {} signal'.format(sig.description)) - shot.signals_dict[sig]=np.clip(shot.signals_dict[sig],0,np.inf) + if hasattr(sig,"is_strictly_positive"): #backwards compatibility when this attribute didn't exist + if sig.is_strictly_positive: + #print ('Applying positivity constraint to {} signal'.format(sig.description)) + shot.signals_dict[sig]=np.clip(shot.signals_dict[sig],0,np.inf) diff --git a/plasma/utils/performance.py b/plasma/utils/performance.py index cd5a45f7..eb02b3b2 100644 --- a/plasma/utils/performance.py +++ b/plasma/utils/performance.py @@ -656,78 +656,84 @@ def plot_shot(self,shot,save_fig=True,normalize=True,truth=None,prediction=None, if(shot.previously_saved(self.shots_dir)): shot.restore(self.shots_dir) - t_disrupt = shot.t_disrupt - is_disruptive = shot.is_disruptive - if normalize: - self.normalizer.apply(shot) + if shot.signals_dict is not None: #make sure shot was saved with data + t_disrupt = shot.t_disrupt + is_disruptive = shot.is_disruptive + if normalize: + self.normalizer.apply(shot) - use_signals = self.saved_conf['paths']['use_signals'] - fontsize= 15 - lower_lim = 0 #len(pred) - plt.close() - colors = ["b","k"] - lss = ["-","--"] - f,axarr = plt.subplots(len(use_signals)+1,1,sharex=True,figsize=(10,15))#, squeeze=False) - plt.title(prediction_type) - assert(np.all(shot.ttd.flatten() == truth.flatten())) - xx = range(len(prediction)) #list(reversed(range(len(pred)))) - for i,sig in enumerate(use_signals): - ax = axarr[i] - num_channels = sig.num_channels - sig_arr = shot.signals_dict[sig] - if num_channels == 1: - # if j == 0: - ax.plot(xx,sig_arr[:,0],linewidth=2)#,linestyle=lss[j],color=colors[j]) - # else: - # ax.plot(xx,sig_arr[:,0],linewidth=2)#,linestyle=lss[j],color=colors[j],label = labels[sig]) - ax.plot([],linestyle="none",label = sig.description)#labels[sig]) - if np.min(sig_arr[:,0]) < 0: - ax.set_ylim([-6,6]) - ax.set_yticks([-5,0,5]) + use_signals = self.saved_conf['paths']['use_signals'] + fontsize= 15 + lower_lim = 0 #len(pred) + plt.close() + colors = ["b","k"] + lss = ["-","--"] + f,axarr = plt.subplots(len(use_signals)+1,1,sharex=True,figsize=(10,15))#, squeeze=False) + plt.title(prediction_type) + assert(np.all(shot.ttd.flatten() == truth.flatten())) + xx = range(len(prediction)) #list(reversed(range(len(pred)))) + for i,sig in enumerate(use_signals): + ax = axarr[i] + num_channels = sig.num_channels + sig_arr = shot.signals_dict[sig] + if num_channels == 1: + # if j == 0: + ax.plot(xx,sig_arr[:,0],linewidth=2)#,linestyle=lss[j],color=colors[j]) + # else: + # ax.plot(xx,sig_arr[:,0],linewidth=2)#,linestyle=lss[j],color=colors[j],label = labels[sig]) + ax.plot([],linestyle="none",label = sig.description)#labels[sig]) + if np.min(sig_arr[:,0]) < 0: + ax.set_ylim([-6,6]) + ax.set_yticks([-5,0,5]) + # ax.plot(xx,sig_arr[:,0],linewidth=2)#,linestyle=lss[j],color=colors[j],label = labels[sig]) + ax.plot([],linestyle="none",label = sig.description)#labels[sig]) + if np.min(sig_arr[:,0]) < 0: + ax.set_ylim([-6,6]) + ax.set_yticks([-5,0,5]) + else: + ax.set_ylim([0,8]) + ax.set_yticks([0,5]) + # ax.set_ylabel(labels[sig],size=fontsize) else: - ax.set_ylim([0,8]) - ax.set_yticks([0,5]) - # ax.set_ylabel(labels[sig],size=fontsize) - else: - ax.imshow(sig_arr[:,:].T, aspect='auto', label = sig.description,cmap="inferno" ) - ax.set_ylim([0,num_channels]) - ax.text(lower_lim+200, 45, sig.description, bbox={'facecolor': 'white', 'pad': 10},fontsize=fontsize-5) - ax.set_yticks([0,num_channels/2]) - ax.set_yticklabels(["0","0.5"]) - ax.set_ylabel("$\\rho$",size=fontsize) - ax.legend(loc="best",labelspacing=0.1,fontsize=fontsize,frameon=False) - ax.axvline(len(truth)-self.T_min_warn,color='r',linewidth=0.5) - plt.setp(ax.get_xticklabels(),visible=False) + ax.imshow(sig_arr[:,:].T, aspect='auto', label = sig.description,cmap="inferno" ) + ax.set_ylim([0,num_channels]) + ax.text(lower_lim+200, 45, sig.description, bbox={'facecolor': 'white', 'pad': 10},fontsize=fontsize-5) + ax.set_yticks([0,num_channels/2]) + ax.set_yticklabels(["0","0.5"]) + ax.set_ylabel("$\\rho$",size=fontsize) + ax.legend(loc="best",labelspacing=0.1,fontsize=fontsize,frameon=False) + ax.axvline(len(truth)-self.T_min_warn,color='r',linewidth=0.5) + plt.setp(ax.get_xticklabels(),visible=False) + plt.setp(ax.get_yticklabels(),fontsize=fontsize) + f.subplots_adjust(hspace=0) + #print(sig) + #print('min: {}, max: {}'.format(np.min(sig_arr), np.max(sig_arr))) + ax = axarr[-1] + # ax.semilogy((-truth+0.0001),label='ground truth') + # ax.plot(-prediction+0.0001,'g',label='neural net prediction') + # ax.axhline(-P_thresh_opt,color='k',label='trigger threshold') + # nn = np.min(pred) + ax.plot(xx,truth,'g',label='target',linewidth=2) + # ax.axhline(0.4,linestyle="--",color='k',label='threshold') + ax.plot(xx,prediction,'b',label='RNN output',linewidth=2) + ax.axhline(P_thresh_opt,linestyle="--",color='k',label='threshold') + ax.set_ylim([-2,2]) + ax.set_yticks([-1,0,1]) + # if len(truth)-T_max_warn >= 0: + # ax.axvline(len(truth)-T_max_warn,color='r')#,label='max warning time') + ax.axvline(len(truth)-self.T_min_warn,color='r',linewidth=0.5)#,label='min warning time') + ax.set_xlabel('T [ms]',size=fontsize) + # ax.axvline(2400) + ax.legend(loc = (0.5,0.7),fontsize=fontsize-5,labelspacing=0.1,frameon=False) plt.setp(ax.get_yticklabels(),fontsize=fontsize) - f.subplots_adjust(hspace=0) - #print(sig) - #print('min: {}, max: {}'.format(np.min(sig_arr), np.max(sig_arr))) - ax = axarr[-1] - # ax.semilogy((-truth+0.0001),label='ground truth') - # ax.plot(-prediction+0.0001,'g',label='neural net prediction') - # ax.axhline(-P_thresh_opt,color='k',label='trigger threshold') - # nn = np.min(pred) - ax.plot(xx,truth,'g',label='target',linewidth=2) - # ax.axhline(0.4,linestyle="--",color='k',label='threshold') - ax.plot(xx,prediction,'b',label='RNN output',linewidth=2) - ax.axhline(P_thresh_opt,linestyle="--",color='k',label='threshold') - ax.set_ylim([-2,2]) - ax.set_yticks([-1,0,1]) - # if len(truth)-T_max_warn >= 0: - # ax.axvline(len(truth)-T_max_warn,color='r')#,label='max warning time') - ax.axvline(len(truth)-self.T_min_warn,color='r',linewidth=0.5)#,label='min warning time') - ax.set_xlabel('T [ms]',size=fontsize) - # ax.axvline(2400) - ax.legend(loc = (0.5,0.7),fontsize=fontsize-5,labelspacing=0.1,frameon=False) - plt.setp(ax.get_yticklabels(),fontsize=fontsize) - plt.setp(ax.get_xticklabels(),fontsize=fontsize) - # plt.xlim(0,200) - plt.xlim([lower_lim,len(truth)]) - # plt.savefig("{}.png".format(num),dpi=200,bbox_inches="tight") - if save_fig: - plt.savefig('sig_fig_{}{}.png'.format(shot.number,extra_filename),bbox_inches='tight') - np.savez('sig_{}{}.npz'.format(shot.number,extra_filename),shot=shot,T_min_warn=self.T_min_warn,T_max_warn=self.T_max_warn,prediction=prediction,truth=truth,use_signals=use_signals,P_thresh=P_thresh_opt) - #plt.show() + plt.setp(ax.get_xticklabels(),fontsize=fontsize) + # plt.xlim(0,200) + plt.xlim([lower_lim,len(truth)]) + # plt.savefig("{}.png".format(num),dpi=200,bbox_inches="tight") + if save_fig: + plt.savefig('sig_fig_{}{}.png'.format(shot.number,extra_filename),bbox_inches='tight') + np.savez('sig_{}{}.npz'.format(shot.number,extra_filename),shot=shot,T_min_warn=self.T_min_warn,T_max_warn=self.T_max_warn,prediction=prediction,truth=truth,use_signals=use_signals,P_thresh=P_thresh_opt) + #plt.show() else: print("Shot hasn't been processed") @@ -834,7 +840,7 @@ def tradeoff_plot(self,accuracy_range,missed_range,fp_range,early_alarm_range,sa plt.ylim([0,1]) if save_figure: plt.savefig(title_str + '_roc.png',bbox_inches='tight',dpi=200) - np.savez(title_str + '_roc.npz',"P_thresh_range",P_thresh_range,"missed_range",missed_range,"fp_range",fp_range) + #np.savez(title_str + '_roc.npz',"P_thresh_range",P_thresh_range,"missed_range",missed_range,"fp_range",fp_range) print('ROC area ({}) is {}'.format(plot_string,self.roc_from_missed_fp(missed_range,fp_range))) return P_thresh_range,missed_range,fp_range From 8ce98dc672ee733804c9293475471727da173215 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Fri, 9 Feb 2018 06:42:49 -0500 Subject: [PATCH 028/272] fixed bug where all workers had the same numpy random seed (it gets reset to 0 in preprocessor). Thus need to explicitly give each worker a different random seed right before training. --- examples/mpi_learn.py | 3 +++ plasma/models/mpi_runner.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/mpi_learn.py b/examples/mpi_learn.py index a965a86b..7bc4ce8b 100644 --- a/examples/mpi_learn.py +++ b/examples/mpi_learn.py @@ -86,6 +86,9 @@ loader = Loader(conf,normalizer) print("...done") +#ensure training has a separate random seed for every worker +np.random.seed(task_index) +random.seed(task_index) if not only_predict: mpi_train(conf,shot_list_train,shot_list_validate,loader) diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 2fad6bc2..8dde8b55 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -20,6 +20,7 @@ import time import datetime import numpy as np +import random from functools import partial import socket @@ -170,7 +171,8 @@ def get_val(self): class MPIModel(): def __init__(self,model,optimizer,comm,batch_iterator,batch_size,num_replicas=None,warmup_steps=1000,lr=0.01,num_batches_minimum=100): - # random.seed(task_index) + random.seed(task_index) + np.random.seed(task_index) self.epoch = 0 self.num_so_far = 0 self.num_so_far_accum = 0 @@ -640,6 +642,7 @@ def mpi_make_predictions_and_evaluate(conf,shot_list,loader,custom_path=None): def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=None): + loader.set_inference_mode(False) conf['num_workers'] = comm.Get_size() From 480e303555fc37ef8955035e8805f8df57d610ef Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Fri, 9 Feb 2018 12:46:36 +0100 Subject: [PATCH 029/272] not saving from within preprocessor class --- plasma/utils/performance.py | 1 - 1 file changed, 1 deletion(-) diff --git a/plasma/utils/performance.py b/plasma/utils/performance.py index 35ec16a9..faa42478 100644 --- a/plasma/utils/performance.py +++ b/plasma/utils/performance.py @@ -834,7 +834,6 @@ def tradeoff_plot(self,accuracy_range,missed_range,fp_range,early_alarm_range,sa plt.ylim([0,1]) if save_figure: plt.savefig(title_str + '_roc.png',bbox_inches='tight',dpi=200) - np.savez(title_str + '_roc.npz',"P_thresh_range",P_thresh_range,"missed_range",missed_range,"fp_range",fp_range) print('ROC area ({}) is {}'.format(plot_string,self.roc_from_missed_fp(missed_range,fp_range))) return P_thresh_range,missed_range,fp_range From dea576eadfa6d2b47d566980f9dd00118a7e53fa Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Fri, 9 Feb 2018 06:49:23 -0500 Subject: [PATCH 030/272] conf update --- examples/conf.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/conf.yaml b/examples/conf.yaml index 1d48aa12..54a9880a 100644 --- a/examples/conf.yaml +++ b/examples/conf.yaml @@ -3,21 +3,21 @@ #will do stuff in fs_path / [username] / signal_data | shot_lists | processed shots, etc. fs_path: '/tigress' -target: 'maxhinge' #'maxhinge' #'maxhinge' #'binary' #'hinge' +target: 'hinge' #'maxhinge' #'maxhinge' #'binary' #'hinge' num_gpus: 4 paths: signal_prepath: '/signal_data/' #/signal_data/jet/ shot_list_dir: '/shot_lists/' tensorboard_save_path: '/Graph/' - data: jet_to_d3d_data #'d3d_to_jet_data' #'d3d_to_jet_data' # 'jet_to_d3d_data' #jet_data + data: jet_data #'d3d_to_jet_data' #'d3d_to_jet_data' # 'jet_to_d3d_data' #jet_data specific_signals: [] #['q95','li','ip','betan','energy','lm','pradcore','pradedge','pradtot','pin','torquein','tmamp1','tmamp2','tmfreq1','tmfreq2','pechin','energydt','ipdirect','etemp_profile','edens_profile'] #if left empty will use all valid signals defined on a machine. Only use if need a custom set executable: "mpi_learn.py" shallow_executable: "learn.py" data: - bleed_in: 5 #how many shots from the test sit to use in training? - bleed_in_repeat_fac: 10 + bleed_in: 0 #how many shots from the test sit to use in training? + bleed_in_repeat_fac: 1 #how many times to repeat shots in training and validation? bleed_in_remove_from_test: True bleed_in_equalize_sets: False signal_to_augment: None #'plasma current' #or None @@ -53,7 +53,7 @@ data: floatx: 'float32' model: - shallow: True + shallow: False shallow_model: num_samples: 1000000 #1000000 #the number of samples to use for training type: "xgboost" #"xgboost" #"xgboost" #"random_forest" "xgboost" From b080734f892fba9e1adde63b9b7ebf6b63c81f90 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Fri, 9 Feb 2018 07:31:32 -0500 Subject: [PATCH 031/272] implemented randomized cutting of first batch_size shots in the buffer. This means shots end at random times even at the beginning of training, which removes the bias that for the first couple of training iterations the RNN would only be exposed to the beginnings of various shots. --- plasma/models/loader.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/plasma/models/loader.py b/plasma/models/loader.py index d3751701..56ba0021 100644 --- a/plasma/models/loader.py +++ b/plasma/models/loader.py @@ -88,10 +88,15 @@ def training_batch_generator(self,shot_list): yield X[start:end],y[start:end],reset_states_now,num_so_far,num_total epoch += 1 - def fill_training_buffer(self,Xbuff,Ybuff,end_indices,shot): + def fill_training_buffer(self,Xbuff,Ybuff,end_indices,shot,is_first_fill=False): sig,res = self.get_signal_result_from_shot(shot) - sig_len = res.shape[0] length = self.conf['model']['length'] + if is_first_fill:#cut signal to random position + cut_idx = np.random.randint(res.shape[0]-length+1) + sig = sig[cut_idx:] + res = res[cut_idx:] + + sig_len = res.shape[0] sig_len = (sig_len // length)*length #make divisible by lenth assert(sig_len > 0) batch_idx = np.where(end_indices == 0)[0][0] @@ -157,8 +162,10 @@ def training_batch_generator_partial_reset(self,shot_list): num_total = len(shot_list) num_so_far = 0 returned = False + num_steps = 0 warmup_steps = self.conf['training']['batch_generator_warmup_steps'] - is_warmup_period = warmup_steps > 0 + is_warmup_period = num_steps < warmup_steps + is_first_fill = num_steps < batch_size while True: # the list of all shots shot_list.shuffle() @@ -174,11 +181,12 @@ def training_batch_generator_partial_reset(self,shot_list): X,Y = self.return_from_training_buffer(Xbuff,Ybuff,end_indices) yield X,Y,batches_to_reset,num_so_far,num_total,is_warmup_period returned = True - warmup_steps -= 1 - is_warmup_period = warmup_steps > 0 + num_steps += 1 + is_warmup_period = num_steps < warmup_steps + is_first_fill = num_steps < batch_size batches_to_reset[:] = False - Xbuff,Ybuff,batch_idx = self.fill_training_buffer(Xbuff,Ybuff,end_indices,shot) + Xbuff,Ybuff,batch_idx = self.fill_training_buffer(Xbuff,Ybuff,end_indices,shot,is_first_fill) batches_to_reset[batch_idx] = True if returned and not is_warmup_period: num_so_far += 1 From e06bcb160698fd3cb3cd2d370a3897216f68e337 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Fri, 9 Feb 2018 07:42:44 -0500 Subject: [PATCH 032/272] removed print --- plasma/models/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plasma/models/builder.py b/plasma/models/builder.py index da3274d7..e1913b85 100644 --- a/plasma/models/builder.py +++ b/plasma/models/builder.py @@ -183,7 +183,7 @@ def slicer_output_shape(input_shape,indices): pre_rnn = Dense(dense_size//4,activation='relu',kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn) pre_rnn_model = Model(inputs = pre_rnn_input,outputs=pre_rnn) - pre_rnn_model.summary() + #pre_rnn_model.summary() x_input = Input(batch_shape = batch_input_shape) x_in = TimeDistributed(pre_rnn_model) (x_input) From e21f431881ff302a60ee66ec0b24a36a56219120 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Fri, 9 Feb 2018 08:01:45 -0500 Subject: [PATCH 033/272] modified printout to add walltime --- plasma/models/mpi_runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 8dde8b55..6a6c34f9 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -173,6 +173,7 @@ class MPIModel(): def __init__(self,model,optimizer,comm,batch_iterator,batch_size,num_replicas=None,warmup_steps=1000,lr=0.01,num_batches_minimum=100): random.seed(task_index) np.random.seed(task_index) + self.start_time = time.time() self.epoch = 0 self.num_so_far = 0 self.num_so_far_accum = 0 @@ -468,7 +469,7 @@ def train_epoch(self): loss_averager.add_val(curr_loss) ave_loss = loss_averager.get_val() eta = self.estimate_remaining_time(t0 - t_start,self.num_so_far-self.epoch*num_total,num_total) - write_str = '\r[{}] step: {} [ETA: {:.2f}s] [{:.2f}/{}], loss: {:.5f} [{:.5f}] | '.format(self.task_index,step,eta,1.0*self.num_so_far,num_total,ave_loss,curr_loss) + write_str = '\r[{}] step: {} [ETA: {:.2f}s] [{:.2f}/{}], loss: {:.5f} [{:.5f}] | walltime: {:.4f} | '.format(self.task_index,step,eta,1.0*self.num_so_far,num_total,ave_loss,curr_loss,time.time()-self.start_time) print_unique(write_str + write_str_0) step += 1 else: From 9aded7f49feeecc24a79c2ec9932c07a4a0beec3 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Fri, 23 Feb 2018 15:50:55 -0500 Subject: [PATCH 034/272] Add a config parameter to control loss scaling. Default is 1.0 --- examples/conf.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/conf.yaml b/examples/conf.yaml index c59d00d2..b7f8244b 100644 --- a/examples/conf.yaml +++ b/examples/conf.yaml @@ -54,6 +54,7 @@ data: floatx: 'float32' model: + loss_scale_factor: 1.0 use_bidirectional: false use_batch_norm: false shallow: False From d44a2f62cdb0534cbc9194292b107aad0e5e869a Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Fri, 23 Feb 2018 15:51:56 -0500 Subject: [PATCH 035/272] =?UTF-8?q?=20Unscale=20gradients=20by=20DUMMY=5FL?= =?UTF-8?q?R=20factor=20before=20subtracting,=20unscale=20loss=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plasma/models/mpi_runner.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 6a6c34f9..d70d80df 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -130,9 +130,6 @@ def __init__(self,lr): def get_deltas(self,raw_deltas): - if K.floatx() == "float16": - raw_deltas[:] = map(lambda w: w.astype(np.float32),raw_deltas) - if self.iterations == 0: self.m_list = [np.zeros_like(g) for g in raw_deltas] self.v_list = [np.zeros_like(g) for g in raw_deltas] @@ -150,9 +147,6 @@ def get_deltas(self,raw_deltas): self.iterations += 1 - if K.floatx() == "float16": - deltas[:] = map(lambda w: w.astype(np.float16),deltas) - return deltas @@ -258,10 +252,17 @@ def train_on_batch_and_get_deltas(self,X_batch,Y_batch,verbose=False): weights_after_update = self.model.get_weights() self.model.set_weights(weights_before_update) + + #unscale before subtracting + weights_before_update = multiply_params(weights_before_update,1.0/self.DUMMY_LR) + weights_after_update = multiply_params(weights_after_update,1.0/self.DUMMY_LR) deltas = subtract_params(weights_after_update,weights_before_update) - deltas = multiply_params(deltas,1.0/self.DUMMY_LR) - + + #unscale loss + if conf['model']['loss_scale_factor'] != 1.0: + deltas = multiply_params(deltas,1.0/conf['model']['loss_scale_factor']) + return deltas,loss From 210ede80982a2b324de0dd39dde9c21754be3f11 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Fri, 23 Feb 2018 15:52:39 -0500 Subject: [PATCH 036/272] Add loss scaling option to targets --- plasma/models/targets.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/plasma/models/targets.py b/plasma/models/targets.py index 0823f459..7f214702 100644 --- a/plasma/models/targets.py +++ b/plasma/models/targets.py @@ -5,6 +5,8 @@ from plasma.utils.evaluation import mae_np,mse_np,binary_crossentropy_np,hinge_np,squared_hinge_np import keras.backend as K +import plasma.conf + #Requirement: larger value must mean disruption more likely. class Target(object): activation = 'linear' @@ -12,7 +14,7 @@ class Target(object): @abc.abstractmethod def loss_np(y_true,y_pred): - return mse_np(y_true,y_pred) + return conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) @abc.abstractmethod def remapper(ttd,T_warning): @@ -29,7 +31,7 @@ class BinaryTarget(Target): @staticmethod def loss_np(y_true,y_pred): - return binary_crossentropy_np(y_true,y_pred) + return conf['model']['loss_scale_factor']*binary_crossentropy_np(y_true,y_pred) @staticmethod def remapper(ttd,T_warning,as_array_of_shots=True): @@ -51,7 +53,7 @@ class TTDTarget(Target): @staticmethod def loss_np(y_true,y_pred): - return mse_np(y_true,y_pred) + return conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) @staticmethod def remapper(ttd,T_warning): @@ -85,7 +87,6 @@ def remapper(ttd,T_warning): def threshold_range(T_warning): return np.logspace(-6,np.log10(T_warning),100) - class TTDLinearTarget(Target): activation = 'linear' @@ -93,7 +94,7 @@ class TTDLinearTarget(Target): @staticmethod def loss_np(y_true,y_pred): - return mse_np(y_true,y_pred) + return conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) @staticmethod @@ -128,7 +129,7 @@ def loss(y_true, y_pred): weight_mask = K.cast(K.greater(weight_mask,0.0),K.floatx()) #positive label! weight_mask = fac*weight_mask + (1 - weight_mask) #return weight_mask*squared_hinge(y_true,y_pred1) - return overall_fac*weight_mask*hinge(y_true,y_pred1) + return conf['model']['loss_scale_factor']*overall_fac*weight_mask*hinge(y_true,y_pred1) @staticmethod def loss_np(y_true, y_pred): @@ -144,7 +145,7 @@ def loss_np(y_true, y_pred): weight_mask = np.greater(y_true,0.0).astype(np.float32) #positive label! weight_mask = fac*weight_mask + (1 - weight_mask) #return np.mean(weight_mask*np.square(np.maximum(1. - y_true * y_pred, 0.)))#, axis=-1) only during training, here we want to completely sum up over all instances - return np.mean(overall_fac*weight_mask*np.maximum(1. - y_true * y_pred, 0.))#, axis=-1) only during training, here we want to completely sum up over all instances + return conf['model']['loss_scale_factor']*np.mean(overall_fac*weight_mask*np.maximum(1. - y_true * y_pred, 0.))#, axis=-1) only during training, here we want to completely sum up over all instances # def _loss_tensor_old(y_true, y_pred): @@ -174,7 +175,7 @@ class HingeTarget(Target): @staticmethod def loss_np(y_true, y_pred): - return hinge_np(y_true,y_pred) + return conf['model']['loss_scale_factor']*hinge_np(y_true,y_pred) #return squared_hinge_np(y_true,y_pred) @staticmethod @@ -188,4 +189,3 @@ def remapper(ttd,T_warning,as_array_of_shots=True): @staticmethod def threshold_range(T_warning): return np.concatenate((np.linspace(-2,-1.06,100),np.linspace(-1.06,-0.96,100),np.linspace(-0.96,2,50))) - From 151738c743d1bb1e404598a25bae323296d531d0 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Fri, 23 Feb 2018 15:53:40 -0500 Subject: [PATCH 037/272] Use scratch gpfs on Tigergpu - SSDs --- examples/slurm.cmd | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/examples/slurm.cmd b/examples/slurm.cmd index 16af0d4f..1e22ad99 100644 --- a/examples/slurm.cmd +++ b/examples/slurm.cmd @@ -1,5 +1,5 @@ #!/bin/bash -#SBATCH -t 01:30:00 +#SBATCH -t 01:00:00 #SBATCH -N 3 #SBATCH --ntasks-per-node=4 #SBATCH --ntasks-per-socket=2 @@ -7,21 +7,19 @@ #SBATCH -c 4 #SBATCH --mem-per-cpu=0 -export PYTHONHASHSEED=0 -module load anaconda -source activate pppl +module load anaconda/4.4.0 +source activate PPPL module load cudatoolkit/8.0 module load cudnn/cuda-8.0/6.0 module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 -module load intel/17.0/64/17.0.4.196 intel-mkl/2017.3/4/64 +module load intel/17.0/64/17.0.4.196 #remove checkpoints for a benchmark run -rm /tigress/$USER/model_checkpoints/* -rm /tigress/$USER/results/* -rm /tigress/$USER/csv_logs/* -rm /tigress/$USER/Graph/* -rm /tigress/$USER/normalization/* +rm /scratch/gpfs/$USER/model_checkpoints/* +rm /scratch/gpfs/$USER/results/* +rm /scratch/gpfs/$USER/csv_logs/* +rm /scratch/gpfs/$USER/Graph/* +rm /scratch/gpfs/$USER/normalization/* export OMPI_MCA_btl="tcp,self,sm" - srun python mpi_learn.py From ac7df4e3e8a5cb234c696484c75e36fd30ee6320 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Tue, 6 Mar 2018 19:20:16 -0500 Subject: [PATCH 038/272] make normalizer resilient for when conf does not contain norm_stat_range --- plasma/preprocessor/normalize.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/plasma/preprocessor/normalize.py b/plasma/preprocessor/normalize.py index f9f23ac0..794e7837 100644 --- a/plasma/preprocessor/normalize.py +++ b/plasma/preprocessor/normalize.py @@ -43,7 +43,9 @@ def __init__(self,conf): self.remapper = conf['data']['target'].remapper self.machines = set() self.inference_mode = False - self.bound = self.conf['data']['norm_stat_range'] + self.bound = np.Inf + if 'norm_stat_range' in self.conf['data']: + self.bound = self.conf['data']['norm_stat_range'] @abc.abstractmethod def __str__(self): @@ -193,7 +195,9 @@ def __init__(self,conf): Normalizer.__init__(self,conf) self.means = dict() self.stds = dict() - self.bound = self.conf['data']['norm_stat_range'] + self.bound = np.Inf + if 'norm_stat_range' in self.conf['data']: + self.bound = self.conf['data']['norm_stat_range'] def __str__(self): s = '' @@ -335,7 +339,9 @@ def __init__(self,conf): Normalizer.__init__(self,conf) self.minimums = None self.maximums = None - self.bound = self.conf['data']['norm_stat_range'] + self.bound = np.Inf + if 'norm_stat_range' in self.conf['data']: + self.bound = self.conf['data']['norm_stat_range'] def __str__(self): From 735e1c26a2e9269b14e35dfc31db1baac5d08c44 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Tue, 6 Mar 2018 19:28:06 -0500 Subject: [PATCH 039/272] remove bidirectional RNN since it violates causality. Add resilience for when batch_normalization is not defined in the conf --- plasma/models/builder.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/plasma/models/builder.py b/plasma/models/builder.py index e1913b85..f4e466b6 100644 --- a/plasma/models/builder.py +++ b/plasma/models/builder.py @@ -72,12 +72,13 @@ def get_0D_1D_indices(self): def build_model(self,predict,custom_batch_size=None): conf = self.conf model_conf = conf['model'] - use_bidirectional = model_conf['use_bidirectional'] rnn_size = model_conf['rnn_size'] rnn_type = model_conf['rnn_type'] regularization = model_conf['regularization'] dense_regularization = model_conf['dense_regularization'] - use_batch_norm = model_conf['use_batch_norm'] + use_batch_norm = False + if 'use_batch_norm' in model_conf: + use_batch_norm = model_conf['use_batch_norm'] dropout_prob = model_conf['dropout_prob'] length = model_conf['length'] @@ -187,12 +188,6 @@ def slicer_output_shape(input_shape,indices): x_input = Input(batch_shape = batch_input_shape) x_in = TimeDistributed(pre_rnn_model) (x_input) - if use_bidirectional: - for _ in range(model_conf['rnn_layers']): - x_in = Bidirectional(rnn_model(rnn_size, return_sequences=return_sequences, - stateful=stateful,kernel_regularizer=l2(regularization),recurrent_regularizer=l2(regularization), - bias_regularizer=l2(regularization),dropout=dropout_prob,recurrent_dropout=dropout_prob)) (x_in) - x_in = Dropout(dropout_prob) (x_in) else: for _ in range(model_conf['rnn_layers']): x_in = rnn_model(rnn_size, return_sequences=return_sequences,#batch_input_shape=batch_input_shape, From 76a566ecfade9cebcbfcc3c103c8c011b336856e Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Tue, 6 Mar 2018 19:32:19 -0500 Subject: [PATCH 040/272] remove bidirectional RNN since it violates causality. Add resilience for when batch_normalization is not defined in the conf --- plasma/models/builder.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/plasma/models/builder.py b/plasma/models/builder.py index f4e466b6..8d63aeb7 100644 --- a/plasma/models/builder.py +++ b/plasma/models/builder.py @@ -188,12 +188,11 @@ def slicer_output_shape(input_shape,indices): x_input = Input(batch_shape = batch_input_shape) x_in = TimeDistributed(pre_rnn_model) (x_input) - else: - for _ in range(model_conf['rnn_layers']): - x_in = rnn_model(rnn_size, return_sequences=return_sequences,#batch_input_shape=batch_input_shape, - stateful=stateful,kernel_regularizer=l2(regularization),recurrent_regularizer=l2(regularization), - bias_regularizer=l2(regularization),dropout=dropout_prob,recurrent_dropout=dropout_prob) (x_in) - x_in = Dropout(dropout_prob) (x_in) + for _ in range(model_conf['rnn_layers']): + x_in = rnn_model(rnn_size, return_sequences=return_sequences,#batch_input_shape=batch_input_shape, + stateful=stateful,kernel_regularizer=l2(regularization),recurrent_regularizer=l2(regularization), + bias_regularizer=l2(regularization),dropout=dropout_prob,recurrent_dropout=dropout_prob) (x_in) + x_in = Dropout(dropout_prob) (x_in) if return_sequences: #x_out = TimeDistributed(Dense(100,activation='tanh')) (x_in) x_out = TimeDistributed(Dense(1,activation=output_activation)) (x_in) From e42ba53176466995156c84ca08ee6f6fe7cdfd16 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Sun, 11 Mar 2018 05:12:03 -0400 Subject: [PATCH 041/272] After 2.0.6, keras Model does not have stop_training attribute. Only added in the callback --- plasma/models/mpi_runner.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index d70d80df..3f5bdf3b 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -185,6 +185,7 @@ def __init__(self,model,optimizer,comm,batch_iterator,batch_size,num_replicas=No self.num_workers = comm.Get_size() self.task_index = comm.Get_rank() self.history = cbks.History() + self.model.stop_training = False if num_replicas is None or num_replicas < 1 or num_replicas > self.num_workers: self.num_replicas = self.num_workers else: @@ -736,7 +737,6 @@ def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=Non epoch_logs['train_loss'] = ave_loss best_so_far = cmp_fn(epoch_logs[conf['callbacks']['monitor']],best_so_far) - stop_training = False if task_index == 0: print('=========Summary======== for epoch{}'.format(step)) print('Training Loss numpy: {:.3e}'.format(ave_loss)) @@ -747,8 +747,6 @@ def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=Non print('Training ROC: {:.4f}'.format(roc_area_train)) callbacks.on_epoch_end(int(round(e)), epoch_logs) - if hasattr(mpi_model.model,'stop_training'): - stop_training = mpi_model.model.stop_training if best_so_far != epoch_logs[conf['callbacks']['monitor']]: #only save model weights if quantity we are tracking is improving print("Not saving model weights") specific_builder.delete_model_weights(train_model,int(round(e))) @@ -759,7 +757,7 @@ def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=Non val_steps = 1 tensorboard.on_epoch_end(val_generator,val_steps,int(round(e)),epoch_logs) - stop_training = comm.bcast(stop_training,root=0) + stop_training = comm.bcast(mpi_model.model.stop_training,root=0) if stop_training: print("Stopping training due to early stopping") break From 7484c1192a8331b165a4d87d946dcb907187e437 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Sun, 11 Mar 2018 05:14:01 -0400 Subject: [PATCH 042/272] Relax Keras version requirement now --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 59e70b94..ab3be99f 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ download_url = "https://github.com/PPPLDeepLearning/plasma-python", #license = "Apache Software License v2", test_suite = "tests", - install_requires = ['keras==2.0.6','pathos','matplotlib==2.0.2','hyperopt','mpi4py','xgboost'], + install_requires = ['keras>2.0.8','pathos','matplotlib==2.0.2','hyperopt','mpi4py','xgboost'], tests_require = [], classifiers = ["Development Status :: 3 - Alpha", "Environment :: Console", From 5aa60ed6707b5103265056de33345dd9f0bb5423 Mon Sep 17 00:00:00 2001 From: Alexey Svyatkovskiy Date: Mon, 12 Mar 2018 19:41:36 +0300 Subject: [PATCH 043/272] Import error in targets --- plasma/models/targets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plasma/models/targets.py b/plasma/models/targets.py index 7f214702..0ce7a25c 100644 --- a/plasma/models/targets.py +++ b/plasma/models/targets.py @@ -5,7 +5,7 @@ from plasma.utils.evaluation import mae_np,mse_np,binary_crossentropy_np,hinge_np,squared_hinge_np import keras.backend as K -import plasma.conf +import plasma.conf as conf #Requirement: larger value must mean disruption more likely. class Target(object): From 541d98e24689ef5a1686db2c8515e20755de3ab2 Mon Sep 17 00:00:00 2001 From: Alexey Svyatkovskiy Date: Mon, 12 Mar 2018 19:59:00 +0300 Subject: [PATCH 044/272] Bug fix: proper plasma.conf import --- plasma/models/targets.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/plasma/models/targets.py b/plasma/models/targets.py index 0ce7a25c..c311a89f 100644 --- a/plasma/models/targets.py +++ b/plasma/models/targets.py @@ -5,7 +5,7 @@ from plasma.utils.evaluation import mae_np,mse_np,binary_crossentropy_np,hinge_np,squared_hinge_np import keras.backend as K -import plasma.conf as conf +import plasma.conf #Requirement: larger value must mean disruption more likely. class Target(object): @@ -14,7 +14,7 @@ class Target(object): @abc.abstractmethod def loss_np(y_true,y_pred): - return conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) + return plasma.conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) @abc.abstractmethod def remapper(ttd,T_warning): @@ -31,7 +31,7 @@ class BinaryTarget(Target): @staticmethod def loss_np(y_true,y_pred): - return conf['model']['loss_scale_factor']*binary_crossentropy_np(y_true,y_pred) + return plasma.conf['model']['loss_scale_factor']*binary_crossentropy_np(y_true,y_pred) @staticmethod def remapper(ttd,T_warning,as_array_of_shots=True): @@ -53,7 +53,7 @@ class TTDTarget(Target): @staticmethod def loss_np(y_true,y_pred): - return conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) + return plasma.conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) @staticmethod def remapper(ttd,T_warning): @@ -94,7 +94,7 @@ class TTDLinearTarget(Target): @staticmethod def loss_np(y_true,y_pred): - return conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) + return plasma.conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) @staticmethod @@ -129,7 +129,7 @@ def loss(y_true, y_pred): weight_mask = K.cast(K.greater(weight_mask,0.0),K.floatx()) #positive label! weight_mask = fac*weight_mask + (1 - weight_mask) #return weight_mask*squared_hinge(y_true,y_pred1) - return conf['model']['loss_scale_factor']*overall_fac*weight_mask*hinge(y_true,y_pred1) + return plasma.conf['model']['loss_scale_factor']*overall_fac*weight_mask*hinge(y_true,y_pred1) @staticmethod def loss_np(y_true, y_pred): @@ -145,7 +145,7 @@ def loss_np(y_true, y_pred): weight_mask = np.greater(y_true,0.0).astype(np.float32) #positive label! weight_mask = fac*weight_mask + (1 - weight_mask) #return np.mean(weight_mask*np.square(np.maximum(1. - y_true * y_pred, 0.)))#, axis=-1) only during training, here we want to completely sum up over all instances - return conf['model']['loss_scale_factor']*np.mean(overall_fac*weight_mask*np.maximum(1. - y_true * y_pred, 0.))#, axis=-1) only during training, here we want to completely sum up over all instances + return plasma.conf['model']['loss_scale_factor']*np.mean(overall_fac*weight_mask*np.maximum(1. - y_true * y_pred, 0.))#, axis=-1) only during training, here we want to completely sum up over all instances # def _loss_tensor_old(y_true, y_pred): @@ -175,7 +175,7 @@ class HingeTarget(Target): @staticmethod def loss_np(y_true, y_pred): - return conf['model']['loss_scale_factor']*hinge_np(y_true,y_pred) + return plasma.conf['model']['loss_scale_factor']*hinge_np(y_true,y_pred) #return squared_hinge_np(y_true,y_pred) @staticmethod From 48e314d13201efb1af7cb7fb13d5f251406c8829 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Mon, 12 Mar 2018 15:29:02 -0400 Subject: [PATCH 045/272] Defer import causing a cyclic dependency until later. Only needed in Python 2 --- plasma/conf_parser.py | 14 +++++++------- plasma/models/targets.py | 23 ++++++++++++++--------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index 1dea38b3..f5190d35 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -1,4 +1,3 @@ -import plasma.models.targets as t from plasma.primitives.shots import ShotListFiles from data.signals import * @@ -10,6 +9,7 @@ def parameters(input_file): """Parse yaml file of configuration parameters.""" + import plasma.models with open(input_file, 'r') as yaml_file: params = yaml.load(yaml_file) @@ -43,18 +43,18 @@ def parameters(input_file): #ensure shallow model has +1 -1 target. if params['model']['shallow'] or params['target'] == 'hinge': - params['data']['target'] = t.HingeTarget + params['data']['target'] = plasma.models.targets.HingeTarget elif params['target'] == 'maxhinge': t.MaxHingeTarget.fac = params['data']['positive_example_penalty'] - params['data']['target'] = t.MaxHingeTarget + params['data']['target'] = plasma.models.targets.MaxHingeTarget elif params['target'] == 'binary': - params['data']['target'] = t.BinaryTarget + params['data']['target'] = plasma.models.targets.BinaryTarget elif params['target'] == 'ttd': - params['data']['target'] = t.TTDTarget + params['data']['target'] = plasma.models.targets.TTDTarget elif params['target'] == 'ttdinv': - params['data']['target'] = t.TTDInvTarget + params['data']['target'] = plasma.models.targets.TTDInvTarget elif params['target'] == 'ttdlinear': - params['data']['target'] = t.TTDLinearTarget + params['data']['target'] = plasma.models.targets.TTDLinearTarget else: print('Unkown type of target. Exiting') exit(1) diff --git a/plasma/models/targets.py b/plasma/models/targets.py index c311a89f..c409fa68 100644 --- a/plasma/models/targets.py +++ b/plasma/models/targets.py @@ -5,8 +5,6 @@ from plasma.utils.evaluation import mae_np,mse_np,binary_crossentropy_np,hinge_np,squared_hinge_np import keras.backend as K -import plasma.conf - #Requirement: larger value must mean disruption more likely. class Target(object): activation = 'linear' @@ -14,7 +12,8 @@ class Target(object): @abc.abstractmethod def loss_np(y_true,y_pred): - return plasma.conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) + import plasma.conf + return plasma.conf.conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) @abc.abstractmethod def remapper(ttd,T_warning): @@ -31,7 +30,8 @@ class BinaryTarget(Target): @staticmethod def loss_np(y_true,y_pred): - return plasma.conf['model']['loss_scale_factor']*binary_crossentropy_np(y_true,y_pred) + import plasma.conf + return plasma.conf.conf['model']['loss_scale_factor']*binary_crossentropy_np(y_true,y_pred) @staticmethod def remapper(ttd,T_warning,as_array_of_shots=True): @@ -53,7 +53,8 @@ class TTDTarget(Target): @staticmethod def loss_np(y_true,y_pred): - return plasma.conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) + import plasma.conf + return plasma.conf.conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) @staticmethod def remapper(ttd,T_warning): @@ -94,7 +95,8 @@ class TTDLinearTarget(Target): @staticmethod def loss_np(y_true,y_pred): - return plasma.conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) + import plasma.conf + return plasma.conf.conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) @staticmethod @@ -118,6 +120,7 @@ class MaxHingeTarget(Target): @staticmethod def loss(y_true, y_pred): + import plasma.conf fac = MaxHingeTarget.fac #overall_fac = np.prod(np.array(K.shape(y_pred)[1:]).astype(np.float32)) overall_fac = K.prod(K.cast(K.shape(y_pred)[1:],K.floatx())) @@ -129,10 +132,11 @@ def loss(y_true, y_pred): weight_mask = K.cast(K.greater(weight_mask,0.0),K.floatx()) #positive label! weight_mask = fac*weight_mask + (1 - weight_mask) #return weight_mask*squared_hinge(y_true,y_pred1) - return plasma.conf['model']['loss_scale_factor']*overall_fac*weight_mask*hinge(y_true,y_pred1) + return plasma.conf.conf['model']['loss_scale_factor']*overall_fac*weight_mask*hinge(y_true,y_pred1) @staticmethod def loss_np(y_true, y_pred): + import plasma.conf fac = MaxHingeTarget.fac #print(y_pred.shape) overall_fac = np.prod(np.array(y_pred.shape).astype(np.float32)) @@ -145,7 +149,7 @@ def loss_np(y_true, y_pred): weight_mask = np.greater(y_true,0.0).astype(np.float32) #positive label! weight_mask = fac*weight_mask + (1 - weight_mask) #return np.mean(weight_mask*np.square(np.maximum(1. - y_true * y_pred, 0.)))#, axis=-1) only during training, here we want to completely sum up over all instances - return plasma.conf['model']['loss_scale_factor']*np.mean(overall_fac*weight_mask*np.maximum(1. - y_true * y_pred, 0.))#, axis=-1) only during training, here we want to completely sum up over all instances + return plasma.conf.conf['model']['loss_scale_factor']*np.mean(overall_fac*weight_mask*np.maximum(1. - y_true * y_pred, 0.))#, axis=-1) only during training, here we want to completely sum up over all instances # def _loss_tensor_old(y_true, y_pred): @@ -175,7 +179,8 @@ class HingeTarget(Target): @staticmethod def loss_np(y_true, y_pred): - return plasma.conf['model']['loss_scale_factor']*hinge_np(y_true,y_pred) + import plasma.conf + return plasma.conf.conf['model']['loss_scale_factor']*hinge_np(y_true,y_pred) #return squared_hinge_np(y_true,y_pred) @staticmethod From d4270cd705181c24b17913a19582038d428b8d0f Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Mon, 12 Mar 2018 15:46:35 -0400 Subject: [PATCH 046/272] Defer import causing a cyclic dependency until later. Only needed in Python 2 --- plasma/conf_parser.py | 14 +++++++------- plasma/models/targets.py | 28 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index f5190d35..b97b0ca7 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -9,7 +9,7 @@ def parameters(input_file): """Parse yaml file of configuration parameters.""" - import plasma.models + from plasma.models.targets import HingeTarget, MaxHingeTarget, BinaryTarget, TTDTarget, TTDInvTarget, TTDLinearTarget with open(input_file, 'r') as yaml_file: params = yaml.load(yaml_file) @@ -43,18 +43,18 @@ def parameters(input_file): #ensure shallow model has +1 -1 target. if params['model']['shallow'] or params['target'] == 'hinge': - params['data']['target'] = plasma.models.targets.HingeTarget + params['data']['target'] = HingeTarget elif params['target'] == 'maxhinge': t.MaxHingeTarget.fac = params['data']['positive_example_penalty'] - params['data']['target'] = plasma.models.targets.MaxHingeTarget + params['data']['target'] = MaxHingeTarget elif params['target'] == 'binary': - params['data']['target'] = plasma.models.targets.BinaryTarget + params['data']['target'] = BinaryTarget elif params['target'] == 'ttd': - params['data']['target'] = plasma.models.targets.TTDTarget + params['data']['target'] = TTDTarget elif params['target'] == 'ttdinv': - params['data']['target'] = plasma.models.targets.TTDInvTarget + params['data']['target'] = TTDInvTarget elif params['target'] == 'ttdlinear': - params['data']['target'] = plasma.models.targets.TTDLinearTarget + params['data']['target'] = TTDLinearTarget else: print('Unkown type of target. Exiting') exit(1) diff --git a/plasma/models/targets.py b/plasma/models/targets.py index c409fa68..7aa3df6c 100644 --- a/plasma/models/targets.py +++ b/plasma/models/targets.py @@ -12,8 +12,8 @@ class Target(object): @abc.abstractmethod def loss_np(y_true,y_pred): - import plasma.conf - return plasma.conf.conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) + from plasma.conf import conf + return conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) @abc.abstractmethod def remapper(ttd,T_warning): @@ -30,8 +30,8 @@ class BinaryTarget(Target): @staticmethod def loss_np(y_true,y_pred): - import plasma.conf - return plasma.conf.conf['model']['loss_scale_factor']*binary_crossentropy_np(y_true,y_pred) + from plasma.conf import conf + return conf['model']['loss_scale_factor']*binary_crossentropy_np(y_true,y_pred) @staticmethod def remapper(ttd,T_warning,as_array_of_shots=True): @@ -53,8 +53,8 @@ class TTDTarget(Target): @staticmethod def loss_np(y_true,y_pred): - import plasma.conf - return plasma.conf.conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) + from plasma.conf import conf + return conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) @staticmethod def remapper(ttd,T_warning): @@ -95,8 +95,8 @@ class TTDLinearTarget(Target): @staticmethod def loss_np(y_true,y_pred): - import plasma.conf - return plasma.conf.conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) + from plasma.conf import conf + return conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) @staticmethod @@ -120,7 +120,7 @@ class MaxHingeTarget(Target): @staticmethod def loss(y_true, y_pred): - import plasma.conf + from plasma.conf import conf fac = MaxHingeTarget.fac #overall_fac = np.prod(np.array(K.shape(y_pred)[1:]).astype(np.float32)) overall_fac = K.prod(K.cast(K.shape(y_pred)[1:],K.floatx())) @@ -132,11 +132,11 @@ def loss(y_true, y_pred): weight_mask = K.cast(K.greater(weight_mask,0.0),K.floatx()) #positive label! weight_mask = fac*weight_mask + (1 - weight_mask) #return weight_mask*squared_hinge(y_true,y_pred1) - return plasma.conf.conf['model']['loss_scale_factor']*overall_fac*weight_mask*hinge(y_true,y_pred1) + return conf['model']['loss_scale_factor']*overall_fac*weight_mask*hinge(y_true,y_pred1) @staticmethod def loss_np(y_true, y_pred): - import plasma.conf + from plasma.conf import conf fac = MaxHingeTarget.fac #print(y_pred.shape) overall_fac = np.prod(np.array(y_pred.shape).astype(np.float32)) @@ -149,7 +149,7 @@ def loss_np(y_true, y_pred): weight_mask = np.greater(y_true,0.0).astype(np.float32) #positive label! weight_mask = fac*weight_mask + (1 - weight_mask) #return np.mean(weight_mask*np.square(np.maximum(1. - y_true * y_pred, 0.)))#, axis=-1) only during training, here we want to completely sum up over all instances - return plasma.conf.conf['model']['loss_scale_factor']*np.mean(overall_fac*weight_mask*np.maximum(1. - y_true * y_pred, 0.))#, axis=-1) only during training, here we want to completely sum up over all instances + return conf['model']['loss_scale_factor']*np.mean(overall_fac*weight_mask*np.maximum(1. - y_true * y_pred, 0.))#, axis=-1) only during training, here we want to completely sum up over all instances # def _loss_tensor_old(y_true, y_pred): @@ -179,8 +179,8 @@ class HingeTarget(Target): @staticmethod def loss_np(y_true, y_pred): - import plasma.conf - return plasma.conf.conf['model']['loss_scale_factor']*hinge_np(y_true,y_pred) + from plasma.conf import conf + return conf['model']['loss_scale_factor']*hinge_np(y_true,y_pred) #return squared_hinge_np(y_true,y_pred) @staticmethod From 66cf3f8ac69483e019da13ef2be8f210be44c737 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Mon, 12 Mar 2018 15:50:03 -0400 Subject: [PATCH 047/272] Fix t.MaxHingeTarget.fac -> MaxHingeTarget.fac --- plasma/conf_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index b97b0ca7..97ec81ae 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -45,7 +45,7 @@ def parameters(input_file): if params['model']['shallow'] or params['target'] == 'hinge': params['data']['target'] = HingeTarget elif params['target'] == 'maxhinge': - t.MaxHingeTarget.fac = params['data']['positive_example_penalty'] + MaxHingeTarget.fac = params['data']['positive_example_penalty'] params['data']['target'] = MaxHingeTarget elif params['target'] == 'binary': params['data']['target'] = BinaryTarget From 1d5cf00dc17d60276adaa27a103b87ddfa33ea8e Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Mon, 19 Mar 2018 14:12:21 -0400 Subject: [PATCH 048/272] supporting logging of val and test roc values for various T_min_warn times --- examples/mpi_learn.py | 2 +- plasma/models/mpi_runner.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/examples/mpi_learn.py b/examples/mpi_learn.py index 7bc4ce8b..9b38cc07 100644 --- a/examples/mpi_learn.py +++ b/examples/mpi_learn.py @@ -90,7 +90,7 @@ np.random.seed(task_index) random.seed(task_index) if not only_predict: - mpi_train(conf,shot_list_train,shot_list_validate,loader) + mpi_train(conf,shot_list_train,shot_list_validate,loader,shot_list_test=shot_list_test) #load last model for testing loader.set_inference_mode(True) diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 6a6c34f9..dbdcfbd4 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -23,6 +23,7 @@ import random from functools import partial +from copy import deepcopy import socket sys.setrecursionlimit(10000) import getpass @@ -642,7 +643,7 @@ def mpi_make_predictions_and_evaluate(conf,shot_list,loader,custom_path=None): return y_prime,y_gold,disruptive,roc_area,loss -def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=None): +def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=None,shot_list_test=None): loader.set_inference_mode(False) conf['num_workers'] = comm.Get_size() @@ -730,6 +731,17 @@ def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=Non mpi_model.batch_iterator_func.__exit__() mpi_model.num_so_far_accum = mpi_model.num_so_far_indiv mpi_model.set_batch_iterator_func() + if 'monitor_test' in conf['callbacks'].keys() and conf['callbacks']['monitor_test']: + conf_curr = deepcopy(conf) + T_min_warn_orig = conf['data']['T_min_warn'] + for T_min_curr in conf_curr['callbacks']['monitor_times']: + conf_curr['data']['T_min_warn'] = T_min_curr + assert(conf['data']['T_min_warn'] == T_min_warn_orig) + if shot_list_test is not None: + _,_,_,roc_area_t,_ = mpi_make_predictions_and_evaluate(conf_curr,shot_list_test,loader) + epoch_logs['test_roc_{}'.format(T_min_curr)] = roc_area_t + _,_,_,roc_area_v,_ = mpi_make_predictions_and_evaluate(conf_curr,shot_list_validate,loader) + epoch_logs['val_roc_{}'.format(T_min_curr)] = roc_area_v epoch_logs['val_roc'] = roc_area epoch_logs['val_loss'] = loss epoch_logs['train_loss'] = ave_loss From 34f66b67549a44384d89a23fa9c4e596974000df Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Mon, 19 Mar 2018 15:52:09 -0400 Subject: [PATCH 049/272] modified performance analyzer to handle case where shot is shorter than T_min_warn --- plasma/utils/performance.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plasma/utils/performance.py b/plasma/utils/performance.py index 7bbb9538..6abac58e 100644 --- a/plasma/utils/performance.py +++ b/plasma/utils/performance.py @@ -188,8 +188,11 @@ def get_threshold_arrays(self,preds,truths,disruptives): d_early_thresholds.append(-np.inf) else: d_early_thresholds.append(np.max(pred[early_indices])) - - d_correct_thresholds.append(np.max(pred[correct_indices])) + + if np.sum(correct_indices) == 0: + d_correct_thresholds.append(-np.inf) + else: + d_correct_thresholds.append(np.max(pred[correct_indices])) else: nd_thresholds.append(np.max(pred)) return np.array(d_early_thresholds), np.array(d_correct_thresholds),np.array(d_late_thresholds), np.array(nd_thresholds) From 8aa5c1ae7cd087cd0ca54cf2b643dbb0158d8916 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Tue, 20 Mar 2018 20:29:51 -0400 Subject: [PATCH 050/272] implemented more general hash that allows deep copies of the conf to maintain the hash --- plasma/models/builder.py | 14 ++++++++++---- plasma/models/mpi_runner.py | 1 + plasma/utils/downloading.py | 22 ++++++++++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/plasma/models/builder.py b/plasma/models/builder.py index 8d63aeb7..82ef1144 100644 --- a/plasma/models/builder.py +++ b/plasma/models/builder.py @@ -19,7 +19,7 @@ import os,sys import numpy as np from copy import deepcopy -from plasma.utils.downloading import makedirs_process_safe +from plasma.utils.downloading import makedirs_process_safe,general_object_hash import hashlib @@ -31,18 +31,22 @@ def on_batch_end(self, batch, logs=None): self.losses.append(logs.get('loss')) + class ModelBuilder(object): def __init__(self,conf): self.conf = conf def get_unique_id(self): - num_epochs = self.conf['training']['num_epochs'] + #num_epochs = self.conf['training']['num_epochs'] this_conf = deepcopy(self.conf) - #don't make hash dependent on number of epochs. + #don't make hash dependent on number of epochs or T_min_warn as those can be modified this_conf['training']['num_epochs'] = 0 - unique_id = int(hashlib.md5((dill.dumps(this_conf).decode('unicode_escape')).encode('utf-8')).hexdigest(),16) + this_conf['data']['T_min_warn'] = 30 + #unique_id = int(hashlib.md5((dill.dumps(this_conf).decode('unicode_escape')).encode('utf-8')).hexdigest(),16) + unique_id = general_object_hash(this_conf) return unique_id + def get_0D_1D_indices(self): #make sure all 1D indices are contiguous in the end! use_signals = self.conf['paths']['use_signals'] @@ -270,6 +274,8 @@ def get_all_saved_files(self): self.ensure_save_directory() unique_id = self.get_unique_id() filenames = os.listdir(self.conf['paths']['model_save_path']) + print("All saved files with id {} and path {}".format(unique_id,self.conf['paths']['model_save_path'])) + print(filenames) epochs = [] for file in filenames: curr_id,epoch = self.extract_id_and_epoch_from_filename(file) diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index dbdcfbd4..d4a4ea51 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -731,6 +731,7 @@ def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=Non mpi_model.batch_iterator_func.__exit__() mpi_model.num_so_far_accum = mpi_model.num_so_far_indiv mpi_model.set_batch_iterator_func() + if 'monitor_test' in conf['callbacks'].keys() and conf['callbacks']['monitor_test']: conf_curr = deepcopy(conf) T_min_warn_orig = conf['data']['T_min_warn'] diff --git a/plasma/utils/downloading.py b/plasma/utils/downloading.py index 8980ebde..7cf2d48e 100644 --- a/plasma/utils/downloading.py +++ b/plasma/utils/downloading.py @@ -26,6 +26,8 @@ import os import errno +import dill,hashlib + # import gadata # from plasma.primitives.shots import ShotList @@ -34,6 +36,26 @@ #print("Importing numpy version"+np.__version__) +def general_object_hash(o): + """Makes a hash from a dictionary, list, tuple or set to any level, that contains + only other hashable types (including any lists, tuples, sets, and + dictionaries). Relies on dill for serialization""" + + if isinstance(o, (set, tuple, list)): + return tuple([make_hash(e) for e in o]) + + elif not isinstance(o, dict): + return myhash(o) + + new_o = deepcopy(o) + for k, v in new_o.items(): + new_o[k] = make_hash(v) + + return myhash(tuple(frozenset(sorted(new_o.items())))) + +def myhash(x): + return int(hashlib.md5((dill.dumps(x).decode('unicode_escape')).encode('utf-8')).hexdigest(),16) + def get_missing_value_array(): return np.array([-1.0]) From 375326a95488b73c67c461f9cee45fc1fb547636 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Tue, 20 Mar 2018 20:31:21 -0400 Subject: [PATCH 051/272] implemented more general hash that allows deep copies of the conf to maintain the hash --- plasma/utils/downloading.py | 1 + 1 file changed, 1 insertion(+) diff --git a/plasma/utils/downloading.py b/plasma/utils/downloading.py index 7cf2d48e..ad5e8a38 100644 --- a/plasma/utils/downloading.py +++ b/plasma/utils/downloading.py @@ -27,6 +27,7 @@ import errno import dill,hashlib +from copy import deepcopy # import gadata From 68eb2870853464bcafea1404ee05fa7e28a3df78 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Tue, 20 Mar 2018 21:22:58 -0400 Subject: [PATCH 052/272] whitespace --- plasma/models/builder.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/plasma/models/builder.py b/plasma/models/builder.py index 82ef1144..2cbf7eeb 100644 --- a/plasma/models/builder.py +++ b/plasma/models/builder.py @@ -274,8 +274,6 @@ def get_all_saved_files(self): self.ensure_save_directory() unique_id = self.get_unique_id() filenames = os.listdir(self.conf['paths']['model_save_path']) - print("All saved files with id {} and path {}".format(unique_id,self.conf['paths']['model_save_path'])) - print(filenames) epochs = [] for file in filenames: curr_id,epoch = self.extract_id_and_epoch_from_filename(file) From cfc36fe5c5ba4baa658b8fa2332c587180a1db1c Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Tue, 20 Mar 2018 21:23:28 -0400 Subject: [PATCH 053/272] option for just a single profile --- plasma/conf_parser.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index 1dea38b3..a13ca7e4 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -97,6 +97,14 @@ def parameters(input_file): params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = jet_signals_1D + elif params['paths']['data'] == 'jet_data_temp_profile': + params['paths']['shot_files'] = [jet_carbon_wall] + params['paths']['shot_files_test'] = [jet_iterlike_wall] + params['paths']['use_signals_dict'] = {'etemp_profile' : etemp_profile} + elif params['paths']['data'] == 'jet_data_dens_profile': + params['paths']['shot_files'] = [jet_carbon_wall] + params['paths']['shot_files_test'] = [jet_iterlike_wall] + params['paths']['use_signals_dict'] = {'edens_profile' : edens_profile} elif params['paths']['data'] == 'jet_carbon_data': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [] @@ -154,6 +162,14 @@ def parameters(input_file): params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = fully_defined_signals_0D + elif params['paths']['data'] == 'd3d_data_temp_profile': #jet data but with fully defined signals + params['paths']['shot_files'] = [d3d_full] + params['paths']['shot_files_test'] = [] + params['paths']['use_signals_dict'] = {'etemp_profile' : etemp_profile}#fully_defined_signals_0D + elif params['paths']['data'] == 'd3d_data_dens_profile': #jet data but with fully defined signals + params['paths']['shot_files'] = [d3d_full] + params['paths']['shot_files_test'] = [] + params['paths']['use_signals_dict'] = {'edens_profile' : edens_profile}#fully_defined_signals_0D #cross-machine elif params['paths']['data'] == 'jet_to_d3d_data': From a6f58a10be65d1bbce0c848026ac9b686cb0b246 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Tue, 20 Mar 2018 21:35:25 -0400 Subject: [PATCH 054/272] small function naming bug --- plasma/utils/downloading.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plasma/utils/downloading.py b/plasma/utils/downloading.py index ad5e8a38..65a9bfa0 100644 --- a/plasma/utils/downloading.py +++ b/plasma/utils/downloading.py @@ -43,14 +43,14 @@ def general_object_hash(o): dictionaries). Relies on dill for serialization""" if isinstance(o, (set, tuple, list)): - return tuple([make_hash(e) for e in o]) + return tuple([general_object_hash(e) for e in o]) elif not isinstance(o, dict): return myhash(o) new_o = deepcopy(o) for k, v in new_o.items(): - new_o[k] = make_hash(v) + new_o[k] = general_object_hash(v) return myhash(tuple(frozenset(sorted(new_o.items())))) From 6d25d3712c0cc0e4eaeb7636982412bbac884197 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Wed, 21 Mar 2018 15:37:38 -0400 Subject: [PATCH 055/272] printing roc values to log --- plasma/models/mpi_runner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index d4a4ea51..d96adda1 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -740,9 +740,11 @@ def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=Non assert(conf['data']['T_min_warn'] == T_min_warn_orig) if shot_list_test is not None: _,_,_,roc_area_t,_ = mpi_make_predictions_and_evaluate(conf_curr,shot_list_test,loader) - epoch_logs['test_roc_{}'.format(T_min_curr)] = roc_area_t + print_unique('epoch {}, test_roc_{} = {}'.format(int(round(e)),T_min_curr,roc_area_t)) + #epoch_logs['test_roc_{}'.format(T_min_curr)] = roc_area_t _,_,_,roc_area_v,_ = mpi_make_predictions_and_evaluate(conf_curr,shot_list_validate,loader) - epoch_logs['val_roc_{}'.format(T_min_curr)] = roc_area_v + print_unique('epoch {}, val_roc_{} = {}'.format(int(round(e)),T_min_curr,roc_area_v)) + #epoch_logs['val_roc_{}'.format(T_min_curr)] = roc_area_v epoch_logs['val_roc'] = roc_area epoch_logs['val_loss'] = loss epoch_logs['train_loss'] = ave_loss From 8bf886af0071ffd6dab35c72d37e09b563d03431 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Wed, 21 Mar 2018 22:01:20 -0400 Subject: [PATCH 056/272] adding support for pytorch model --- examples/learn.py | 4 +- plasma/models/loader.py | 129 +++++++++- plasma/models/torch_runner.py | 464 ++++++++++++++++++++++++++++++++++ 3 files changed, 589 insertions(+), 8 deletions(-) create mode 100644 plasma/models/torch_runner.py diff --git a/examples/learn.py b/examples/learn.py index 14eae8fb..a1b9a94a 100644 --- a/examples/learn.py +++ b/examples/learn.py @@ -33,7 +33,9 @@ from plasma.preprocessor.preprocess import Preprocessor, guarantee_preprocessed from plasma.models.loader import Loader -if conf['model']['shallow']: +if conf['model']['torch']: + from plasma.models.torch_runner import train, make_predictions_and_evaluate_gpu +elif conf['model']['shallow']: from plasma.models.shallow_runner import train, make_predictions_and_evaluate_gpu else: from plasma.models.runner import train, make_predictions_and_evaluate_gpu diff --git a/plasma/models/loader.py b/plasma/models/loader.py index 56ba0021..9c8e9d2b 100644 --- a/plasma/models/loader.py +++ b/plasma/models/loader.py @@ -133,7 +133,128 @@ def resize_buffer(self,buff,new_length): return new_buff + def inference_batch_generator_full_shot(self,shot_list): + """ + The method implements a training batch generator as a Python generator with a while-loop. + It iterates indefinitely over the data set and returns one mini-batch of data at a time. + + NOTE: Can be inefficient during distributed training because one process loading data will + cause all other processes to stall. + + Argument list: + - shot_list: + + Returns: + - One mini-batch of data and label as a Numpy array: X[start:end],y[start:end] + - reset_states_now: boolean flag indicating when to reset state during stateful RNN training + - num_so_far,num_total: number of samples generated so far and the total dataset size as per shot_list + """ + batch_size = self.conf['training']['pred_batch_size'] + sig,res = self.get_signal_result_from_shot(shot_list.shots[0]) + Xbuff = np.empty((batch_size,) + sig.shape,dtype=self.conf['data']['floatx']) + Ybuff = np.empty((batch_size,) + res.shape,dtype=self.conf['data']['floatx']) + Maskbuff = np.empty((batch_size,) + res.shape,dtype=self.conf['data']['floatx']) + disr = np.empty(batch_size,dtype=bool) + # epoch = 0 + num_total = len(shot_list) + num_so_far = 0 + returned = False + num_steps = 0 + batch_idx = 0 + # warmup_steps = self.conf['training']['batch_generator_warmup_steps'] + # is_warmup_period = num_steps < warmup_steps + # is_first_fill = num_steps < batch_size + while True: + # the list of all shots + # shot_list.shuffle() + for i in range(num_total): + shot = shot_list.shots[i] + sig,res = self.get_signal_result_from_shot(shot) + sig_len = res.shape[0] + if sig_len > Xbuff.shape[1]: #resize buffer if needed + old_len = Xbuff.shape[1] + Xbuff = self.resize_buffer(Xbuff,sig_len) + Ybuff = self.resize_buffer(Ybuff,sig_len) + Maskbuff = self.resize_buffer(Maskbuff,sig_len) + Maskbuff[:,old_len:,:] = 0.0 + + Xbuff[batch_idx,:,:] = sig + Ybuff[batch_idx,:,:] = res + Maskbuff[batch_idx,:sig_len,:] = 1.0 + Maskbuff[batch_idx,sig_len:,:] = 0.0 + disr[batch_idx] = shot.is_disruptive_shot() + batch_idx += 1 + if batch_idx == batch_size: + num_so_far += batch_size + yield 1.0*Xbuff,1.0*Ybuff,1.0*Maskbuff,disr & True,num_so_far,num_total + batch_idx = 0 + + + + + def training_batch_generator_full_shot_partial_reset(self,shot_list): + """ + The method implements a training batch generator as a Python generator with a while-loop. + It iterates indefinitely over the data set and returns one mini-batch of data at a time. + NOTE: Can be inefficient during distributed training because one process loading data will + cause all other processes to stall. + + Argument list: + - shot_list: + + Returns: + - One mini-batch of data and label as a Numpy array: X[start:end],y[start:end] + - reset_states_now: boolean flag indicating when to reset state during stateful RNN training + - num_so_far,num_total: number of samples generated so far and the total dataset size as per shot_list + """ + batch_size = self.conf['training']['batch_size'] + sig,res = self.get_signal_result_from_shot(shot_list.shots[0]) + Xbuff = np.empty((batch_size,) + sig.shape,dtype=self.conf['data']['floatx']) + Ybuff = np.empty((batch_size,) + res.shape,dtype=self.conf['data']['floatx']) + Maskbuff = np.empty((batch_size,) + res.shape,dtype=self.conf['data']['floatx']) + # epoch = 0 + num_total = len(shot_list) + num_so_far = 0 + returned = False + num_steps = 0 + batch_idx = 0 + # warmup_steps = self.conf['training']['batch_generator_warmup_steps'] + # is_warmup_period = num_steps < warmup_steps + # is_first_fill = num_steps < batch_size + while True: + # the list of all shots + shot_list.shuffle() + for i in range(num_total): + shot = self.sample_shot_from_list_given_index(shot_list,i) + sig,res = self.get_signal_result_from_shot(shot) + sig_len = res.shape[0] + if sig_len > Xbuff.shape[1]: #resize buffer if needed + old_len = Xbuff.shape[1] + Xbuff = self.resize_buffer(Xbuff,sig_len) + Ybuff = self.resize_buffer(Ybuff,sig_len) + Maskbuff = self.resize_buffer(Maskbuff,sig_len) + Maskbuff[:,old_len:,:] = 0.0 + + Xbuff[batch_idx,:,:] = sig + Ybuff[batch_idx,:,:] = res + Maskbuff[batch_idx,:sig_len,:] = 1.0 + Maskbuff[batch_idx,sig_len:,:] = 0.0 + batch_idx += 1 + if batch_idx == batch_size: + num_so_far += batch_size + yield 1.0*Xbuff,1.0*Ybuff,1.0*Maskbuff,num_so_far,num_total + batch_idx = 0 + + def sample_shot_from_list_given_index(self,shot_list,i): + if self.conf['training']['ranking_difficulty_fac'] == 1.0: + if self.conf['data']['equalize_classes']: + shot = shot_list.sample_equal_classes() + else: + shot = shot_list.shots[i] + else: #draw the shot weighted + shot = shot_list.sample_weighted() + return shot def training_batch_generator_partial_reset(self,shot_list): """ @@ -170,13 +291,7 @@ def training_batch_generator_partial_reset(self,shot_list): # the list of all shots shot_list.shuffle() for i in range(len(shot_list)): - if self.conf['training']['ranking_difficulty_fac'] == 1.0: - if self.conf['data']['equalize_classes']: - shot = shot_list.sample_equal_classes() - else: - shot = shot_list.shots[i] - else: #draw the shot weighted - shot = shot_list.sample_weighted() + shot = self.sample_shot_from_list_given_index(shot_list,i) while not np.any(end_indices == 0): X,Y = self.return_from_training_buffer(Xbuff,Ybuff,end_indices) yield X,Y,batches_to_reset,num_so_far,num_total,is_warmup_period diff --git a/plasma/models/torch_runner.py b/plasma/models/torch_runner.py new file mode 100644 index 00000000..08629fbe --- /dev/null +++ b/plasma/models/torch_runner.py @@ -0,0 +1,464 @@ +from __future__ import print_function +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt + +import numpy as np +import sys +if sys.version_info[0] < 3: + from itertools import imap + +#leading to import errors: +#from hyperopt import hp, STATUS_OK +#from hyperas.distributions import conditional + +import time +import datetime +import os +from functools import partial +import pathos.multiprocessing as mp +from xgboost import XGBClassifier +from sklearn.neural_network import MLPClassifier + +from plasma.conf import conf +from plasma.models.loader import Loader, ProcessGenerator +from plasma.utils.performance import PerformanceAnalyzer +from plasma.utils.evaluation import * +from plasma.utils.state_reset import reset_states +from plasma.utils.downloading import makedirs_process_safe + +from keras.utils.generic_utils import Progbar + +import hashlib + +import torch +import torch.nn as nn +from torch.autograd import Variable +import torch.optim as opt +from torch.nn.utils import weight_norm + +model_filename = 'torch_model.pt' + +class FTCN(nn.Module): + def __init__(self,n_scalars,n_profiles,profile_size,layer_sizes_spatial, + kernel_size_spatial,linear_size,output_size, + num_channels_tcn,kernel_size_temporal,dropout=0.1): + super(FTCN, self).__init__() + self.lin = InputBlock(n_scalars, n_profiles,profile_size, layer_sizes_spatial, kernel_size_spatial, linear_size, dropout) + self.input_layer = TimeDistributed(lin,batch_first=True) + self.tcn = TCN(linear_size, output_size, num_channels_tcn , kernel_size_temporal, dropout) + self.model = nn.Sequential(self.input_layer,self.tcn) + + def forward(self,x): + return self.model(x) + + +class InputBlock(nn.Module): + def __init__(self, n_scalars, n_profiles,profile_size, layer_sizes, kernel_size, linear_size, dropout=0.2): + super(InputBlock, self).__init__() + self.pooling_size = 2 + self.n_scalars = n_scalars + self.n_profiles = n_profiles + self.profile_size = profile_size + self.conv_output_size = profile_size + if self.n_profiles == 0: + self.net = None + self.conv_output_size = 0 + else: + self.layers = [] + for (i,layer_size) in enumerate(layer_sizes): + if i == 0: + input_size = n_profiles + else: + input_size = layer_sizes[i-1] + self.layers.append(weight_norm(nn.Conv1d(input_size, layer_size, kernel_size))) + self.layers.append(nn.ReLU()) + self.conv_output_size = calculate_conv_output_size(self.conv_output_size,0,1,1,kernel_size) + self.layers.append(nn.MaxPool1d(kernel_size=self.pooling_size)) + self.conv_output_size = calculate_conv_output_size(self.conv_output_size,0,1,self.pooling_size,self.pooling_size) + self.layers.append(nn.Dropout2d(dropout)) + self.net = nn.Sequential(*self.layers) + self.conv_output_size = self.conv_output_size*layer_sizes[-1] + self.linear_layers = [] + + print("Final feature size = {}".format(self.n_scalars + self.conv_output_size)) + self.linear_layers.append(nn.Linear(self.conv_output_size+self.n_scalars,linear_size)) + self.linear_layers.append(nn.ReLU()) + self.linear_layers.append(nn.Linear(linear_size,linear_size)) + self.linear_layers.append(nn.ReLU()) + print("Final output size = {}".format(linear_size)) + self.linear_net = nn.Sequential(*self.linear_layers) + +# def init_weights(self): +# self.conv1.weight.data.normal_(0, 0.01) +# self.conv2.weight.data.normal_(0, 0.01) +# if self.downsample is not None: +# self.downsample.weight.data.normal_(0, 0.01) + + def forward(self, x): + if self.n_profiles == 0: + full_features = x#x_scalars + else: + if self.n_scalars == 0: + x_profiles = x + else: + x_scalars = x[:,:n_scalars] + x_profiles = x[:,n_scalars:] + x_profiles = x_profiles.contiguous().view(x.size(0),self.n_profiles,self.profile_size) + profile_features = self.net(x_profiles).view(x.size(0),-1) + if self.n_scalars == 0: + full_features = profile_features + else: + full_features = torch.cat([x_scalars,profile_features],dim=1) + + out = self.linear_net(full_features) +# out = self.net(x) +# res = x if self.downsample is None else self.downsample(x) + return out + + +def calculate_conv_output_size(L_in,padding,dilation,stride,kernel_size): + return int(np.floor((L_in + 2*padding - dilation*(kernel_size-1) - 1)*1.0/stride + 1)) + + +class Chomp1d(nn.Module): + def __init__(self, chomp_size): + super(Chomp1d, self).__init__() + self.chomp_size = chomp_size + + def forward(self, x): + return x[:, :, :-self.chomp_size].contiguous() + + +class TemporalBlock(nn.Module): + def __init__(self, n_inputs, n_outputs, kernel_size, stride, dilation, padding, dropout=0.2): + super(TemporalBlock, self).__init__() + self.conv1 = weight_norm(nn.Conv1d(n_inputs, n_outputs, kernel_size, + stride=stride, padding=padding, dilation=dilation)) + self.chomp1 = Chomp1d(padding) + self.relu1 = nn.ReLU() + self.dropout1 = nn.Dropout2d(dropout) + + self.conv2 = weight_norm(nn.Conv1d(n_outputs, n_outputs, kernel_size, + stride=stride, padding=padding, dilation=dilation)) + self.chomp2 = Chomp1d(padding) + self.relu2 = nn.ReLU() + self.dropout2 = nn.Dropout2d(dropout) + + self.net = nn.Sequential(self.conv1, self.chomp1, self.relu1, self.dropout1, + self.conv2, self.chomp2, self.relu2, self.dropout2) + self.downsample = nn.Conv1d(n_inputs, n_outputs, 1) if n_inputs != n_outputs else None + self.relu = nn.ReLU() + self.init_weights() + + def init_weights(self): + self.conv1.weight.data.normal_(0, 0.01) + self.conv2.weight.data.normal_(0, 0.01) + if self.downsample is not None: + self.downsample.weight.data.normal_(0, 0.01) + + def forward(self, x): + out = self.net(x) + res = x if self.downsample is None else self.downsample(x) + return self.relu(out + res) + +#dimensions are batch,channels,length +class TemporalConvNet(nn.Module): + def __init__(self, num_inputs, num_channels, kernel_size=2, dropout=0.2): + super(TemporalConvNet, self).__init__() + layers = [] + num_levels = len(num_channels) + for i in range(num_levels): + dilation_size = 2 ** i + in_channels = num_inputs if i == 0 else num_channels[i-1] + out_channels = num_channels[i] + layers += [TemporalBlock(in_channels, out_channels, kernel_size, stride=1, dilation=dilation_size, + padding=(kernel_size-1) * dilation_size, dropout=dropout)] + + self.network = nn.Sequential(*layers) + + def forward(self, x): + return self.network(x) + + +class TCN(nn.Module): + def __init__(self, input_size, output_size, num_channels, kernel_size, dropout): + super(TCN, self).__init__() + self.tcn = TemporalConvNet(input_size, num_channels, kernel_size, dropout=dropout) + self.linear = nn.Linear(num_channels[-1], output_size) +# self.sig = nn.Sigmoid() + + def forward(self, x): + # x needs to have dimension (N, C, L) in order to be passed into CNN + output = self.tcn(x.transpose(1, 2)).transpose(1, 2) + output = self.linear(output)#.transpose(1,2)).transpose(1,2) + return output +# return self.sig(output) + + + + +# def train(model,data_gen,lr=0.001,iters = 100): +# log_step = int(round(iters*0.1)) +# optimizer = opt.Adam(model.parameters(),lr = lr) +# model.train() +# total_loss = 0 +# count = 0 +# loss_fn = nn.MSELoss(size_average=False) +# for i in range(iters): +# x_,y_,mask_ = data_gen() +# # print(y) +# x, y, mask = Variable(torch.from_numpy(x_).float()), Variable(torch.from_numpy(y_).float()),Variable(torch.from_numpy(mask_).byte()) +# # print(y) +# optimizer.zero_grad() +# # output = model(x.unsqueeze(0)).squeeze(0) +# output = model(x)#.unsqueeze(0)).squeeze(0) +# output_masked = torch.masked_select(output,mask) +# y_masked = torch.masked_select(y,mask) +# # print(y.shape,output.shape) +# loss = loss_fn(output_masked,y_masked) +# total_loss += loss.data[0] +# count += output.size(0) + +# # if args.clip > 0: +# # torch.nn.utils.clip_grad_norm(model.parameters(), args.clip) +# loss.backward() +# optimizer.step() +# if i > 0 and i % log_step == 0: +# cur_loss = total_loss / count +# print("Epoch {:2d} | lr {:.5f} | loss {:.5f}".format(0,lr, cur_loss)) +# total_loss = 0.0 +# count = 0 + + + + + + + + +class TimeDistributed(nn.Module): + def __init__(self, module, batch_first=False): + super(TimeDistributed, self).__init__() + self.module = module + self.batch_first = batch_first + + def forward(self, x): + + if len(x.size()) <= 2: + return self.module(x) + + # Squash samples and timesteps into a single axis + x_reshape = x.contiguous().view(-1, x.size(-1)) # (samples * timesteps, input_size) + + y = self.module(x_reshape) + + # We have to reshape Y + if self.batch_first: + y = y.contiguous().view(x.size(0), -1, y.size(-1)) # (samples, timesteps, output_size) + else: + y = y.view(-1, x.size(1), y.size(-1)) # (timesteps, samples, output_size) + + return y + +import keras.callbacks as cbks + + + + +def build_torch_model(conf): + dropout = conf['model']['dropout_prob'] +# dim = 10 + + # lin = nn.Linear(input_size,intermediate_dim) + n_scalars, n_profile, profile_size = get_signal_dimensions(conf) + dim = n_scalars+n_profiles*profile_size + input_size = dim + output_size = 1 + # intermediate_dim = 15 + + layer_sizes_spatial = [40,20,20] + kernel_size_spatial = 3 + linear_size = 10 + + num_channels_tcn = [3]*5 + kernel_size_temporal = 3 + model = FTCN(n_scalars,n_profiles,profile_size,layer_sizes_spatial, + kernel_size_spatial,linear_size,output_size,num_channels_tcn, + kernel_size_temporal,dropout) + + return model + +def get_signal_dimensions(conf): + #make sure all 1D indices are contiguous in the end! + use_signals = conf['paths']['use_signals'] + n_scalars = 0 + n_profiles = 0 + profile_size = 0 + is_1D_region = use_signals[0].num_channels > 1#do we have any 1D indices? + for sig in use_signals: + num_channels = sig.num_channels + if num_channels > 1: + profile_size = num_channels + num_1D += 1 + is_1D_region = True + else: + assert(not is_1D_region), "make sure all use_signals are ordered such that 1D signals come last!" + assert(num_channels == 1) + num_0D += 1 + is_1D_region = False + return n_scalars,n_profiles,profile_size + +def train_epoch(model,data_gen,loss_fn): + loss = 0 + total_loss = 0 + num_so_far = 0 + x_,y_,mask_,num_so_far_start,num_total = next(data_gen) + num_so_far = num_so_far_start + step = 0 + while True: + # print(y) + x, y, mask = Variable(torch.from_numpy(x_).float()), Variable(torch.from_numpy(y_).float()),Variable(torch.from_numpy(mask_).byte()) + # print(y) + optimizer.zero_grad() + # output = model(x.unsqueeze(0)).squeeze(0) + output = model(x)#.unsqueeze(0)).squeeze(0) + output_masked = torch.masked_select(output,mask) + y_masked = torch.masked_select(y,mask) + # print(y.shape,output.shape) + loss = loss_fn(output_masked,y_masked) + total_loss += loss.data[0] + # count += output.size(0) + + # if args.clip > 0: + # torch.nn.utils.clip_grad_norm(model.parameters(), args.clip) + loss.backward() + optimizer.step() + step += 1 + if num_so_far-num_so_far_start >= num_total: + break + x_,y_,mask_,num_so_far_start,num_total = next(data_gen) + return step,loss,total_loss,num_so_far,1.0*num_so_far/num_total + + +def train(conf,shot_list_train,shot_list_validate,loader): + + np.random.seed(1) + + data_gen = ProcessGenerator(partial(loader.training_batch_generator_full_shot_partial_reset,shot_list=shot_list_train)) + + print('validate: {} shots, {} disruptive'.format(len(shot_list_validate),shot_list_validate.num_disruptive())) + print('training: {} shots, {} disruptive'.format(len(shot_list_train),shot_list_train.num_disruptive())) + + loader.set_inference_mode(False) + + train_model = build_torch_model(conf) + + #load the latest epoch we did. Returns -1 if none exist yet + # e = specific_builder.load_model_weights(train_model) + + num_epochs = conf['training']['num_epochs'] + lr_decay = conf['model']['lr_decay'] + batch_size = conf['training']['batch_size'] + lr = conf['model']['lr'] + clipnorm = conf['model']['clipnorm'] + e = 0 + # warmup_steps = conf['model']['warmup_steps'] + # num_batches_minimum = conf['training']['num_batches_minimum'] + + # if 'adam' in conf['model']['optimizer']: + # optimizer = MPIAdam(lr=lr) + # elif conf['model']['optimizer'] == 'sgd' or conf['model']['optimizer'] == 'tf_sgd': + # optimizer = MPISGD(lr=lr) + # elif 'momentum_sgd' in conf['model']['optimizer']: + # optimizer = MPIMomentumSGD(lr=lr) + # else: + # print("Optimizer not implemented yet") + # exit(1) + + print('{} epochs left to go'.format(num_epochs - 1 - e)) + + + if conf['callbacks']['mode'] == 'max': + best_so_far = -np.inf + cmp_fn = max + else: + best_so_far = np.inf + cmp_fn = min + optimizer = opt.Adam(model.parameters(),lr = lr) + model.train() + not_updated = 0 + total_loss = 0 + count = 0 + loss_fn = nn.MSELoss(size_average=False) + model_path = conf['paths']['model_save_path'] + model_filename #save_prepath + model_filename + makedirs_process_safe(conf['paths']['model_save_path']) + while e < num_epochs-1: + print_unique('\nEpoch {}/{}'.format(e,num_epochs)) + (step,ave_loss,curr_loss,num_so_far,effective_epochs) = train_epoch(model,data_gen,loss_fn) + e = effective_epochs + loader.verbose=False #True during the first iteration + # if task_index == 0: + # specific_builder.save_model_weights(train_model,int(round(e))) + model.save_state_dict(model_path) + _,_,_,roc_area,loss = mpi_make_predictions_and_evaluate(conf,shot_list_validate,loader) + + best_so_far = cmp_fn(roc_area,best_so_far) + + stop_training = False + print('=========Summary======== for epoch{}'.format(step)) + print('Training Loss numpy: {:.3e}'.format(ave_loss)) + print('Validation Loss: {:.3e}'.format(loss)) + print('Validation ROC: {:.4f}'.format(roc_area)) + + if best_so_far != epoch_logs[conf['callbacks']['monitor']]: #only save model weights if quantity we are tracking is improving + print("No improvement, still saving model") + not_updated += 1 + else: + print("Saving model") + model.save_state_dict(model_path) + # specific_builder.delete_model_weights(train_model,int(round(e))) + if not_updated > patience: + print("Stopping training due to early stopping") + break + +def make_predictions(conf,shot_list,loader,custom_path=None): + generator = loader.inference_batch_generator_full_shot(shot_list) + inference_model = build_torch_model(conf) + + if custom_path == None: + model_path = conf['paths']['model_save_path'] + model_filename#save_prepath + model_filename + else: + model_path = custom_path + inference_model.load_state_dict(model_path) + #shot_list = shot_list.random_sublist(10) + + y_prime = [] + y_gold = [] + disruptive = [] + num_shots = len(shot_list) + + pbar = Progbar(num_shots) + while True: + x_,y_,mask_,disr_,num_so_far,num_total = next(generator) + x, y, mask = Variable(torch.from_numpy(x_).float()), Variable(torch.from_numpy(y_).float()),Variable(torch.from_numpy(mask_).byte()) + output = model(x) + for batch_idx in range(x.shape[0]) + y_prime[batch_idx] += [output[batch_idx,:,:]] + y_gold += [y_[batch_idx,:,:]] + disruptive += [disr[batch_idx]] + pbar.add(1.0) + if len(disruptive) >= num_shots: + y_prime = y_prime[:num_shots] + y_gold = y_gold[:num_shots] + disruptive = disruptive[:num_shots] + break + return y_prime,y_gold,disruptive + +def make_predictions_and_evaluate_gpu(conf,shot_list,loader,custom_path = None): + y_prime,y_gold,disruptive = make_predictions(conf,shot_list,loader,custom_path) + analyzer = PerformanceAnalyzer(conf=conf) + roc_area = analyzer.get_roc_area(y_prime,y_gold,disruptive) + loss = get_loss_from_list(y_prime,y_gold,conf['data']['target']) + return y_prime,y_gold,disruptive,roc_area,loss + From ad5de2f262f227ff32c45649de31a86aa883dd39 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Thu, 22 Mar 2018 02:01:56 -0400 Subject: [PATCH 057/272] added support for torch fully convolutional model. Temporal convolutions and spatial convolutions. TODO is to add multi GPU support and customizability via conf. Parameters currently hardcoded. --- plasma/models/builder.py | 3 +- plasma/models/loader.py | 51 ++++++++---- plasma/models/torch_runner.py | 146 +++++++++++++++++++--------------- 3 files changed, 120 insertions(+), 80 deletions(-) diff --git a/plasma/models/builder.py b/plasma/models/builder.py index 2cbf7eeb..28459936 100644 --- a/plasma/models/builder.py +++ b/plasma/models/builder.py @@ -273,7 +273,8 @@ def extract_id_and_epoch_from_filename(self,filename): def get_all_saved_files(self): self.ensure_save_directory() unique_id = self.get_unique_id() - filenames = os.listdir(self.conf['paths']['model_save_path']) + path = self.conf['paths']['model_save_path'] + filenames = [name for name in os.listdir(path) if os.path.isfile(os.path.join(path, name))] epochs = [] for file in filenames: curr_id,epoch = self.extract_id_and_epoch_from_filename(file) diff --git a/plasma/models/loader.py b/plasma/models/loader.py index 9c8e9d2b..248642e0 100644 --- a/plasma/models/loader.py +++ b/plasma/models/loader.py @@ -123,11 +123,13 @@ def shift_buffer(self,buff,length): buff[:,:-length,:] = buff[:,length:,:] - def resize_buffer(self,buff,new_length): + def resize_buffer(self,buff,new_length,dtype=None): + if dtype == None: + dtype = self.conf['data']['floatx'] old_length = buff.shape[1] batch_size = buff.shape[0] num_signals = buff.shape[2] - new_buff = np.empty((batch_size,new_length,num_signals),dtype=self.conf['data']['floatx']) + new_buff = np.zeros((batch_size,new_length,num_signals),dtype=dtype) new_buff[:,:old_length,:] = buff #print("Resizing buffer to new length {}".format(new_length)) return new_buff @@ -149,18 +151,20 @@ def inference_batch_generator_full_shot(self,shot_list): - reset_states_now: boolean flag indicating when to reset state during stateful RNN training - num_so_far,num_total: number of samples generated so far and the total dataset size as per shot_list """ - batch_size = self.conf['training']['pred_batch_size'] + batch_size = self.conf['model']['pred_batch_size'] sig,res = self.get_signal_result_from_shot(shot_list.shots[0]) - Xbuff = np.empty((batch_size,) + sig.shape,dtype=self.conf['data']['floatx']) - Ybuff = np.empty((batch_size,) + res.shape,dtype=self.conf['data']['floatx']) - Maskbuff = np.empty((batch_size,) + res.shape,dtype=self.conf['data']['floatx']) - disr = np.empty(batch_size,dtype=bool) + Xbuff = np.zeros((batch_size,) + sig.shape,dtype=self.conf['data']['floatx']) + Ybuff = np.zeros((batch_size,) + res.shape,dtype=self.conf['data']['floatx']) + Maskbuff = np.zeros((batch_size,) + res.shape,dtype=self.conf['data']['floatx']) + disr = np.zeros(batch_size,dtype=bool) + lengths = np.zeros(batch_size,dtype=int) # epoch = 0 num_total = len(shot_list) num_so_far = 0 returned = False num_steps = 0 batch_idx = 0 + np.seterr(all='raise') # warmup_steps = self.conf['training']['batch_generator_warmup_steps'] # is_warmup_period = num_steps < warmup_steps # is_first_fill = num_steps < batch_size @@ -178,15 +182,29 @@ def inference_batch_generator_full_shot(self,shot_list): Maskbuff = self.resize_buffer(Maskbuff,sig_len) Maskbuff[:,old_len:,:] = 0.0 - Xbuff[batch_idx,:,:] = sig - Ybuff[batch_idx,:,:] = res + Xbuff[batch_idx,:,:] = 0.0 + Ybuff[batch_idx,:,:] = 0.0 + Maskbuff[batch_idx,:,:] = 0.0 + Xbuff[batch_idx,:sig_len,:] = sig + Ybuff[batch_idx,:sig_len,:] = res Maskbuff[batch_idx,:sig_len,:] = 1.0 - Maskbuff[batch_idx,sig_len:,:] = 0.0 disr[batch_idx] = shot.is_disruptive_shot() + lengths[batch_idx] = res.shape[0] batch_idx += 1 if batch_idx == batch_size: num_so_far += batch_size - yield 1.0*Xbuff,1.0*Ybuff,1.0*Maskbuff,disr & True,num_so_far,num_total + x1 = 1.0*Xbuff + try: + x2 = 1.0*Ybuff + except: + print(Ybuff[:100]) + print(Ybuff[-100:]) + print(Ybuff) + x3 = 1.0*Maskbuff + x4 = disr & True + x5 = 1*lengths + + yield x1,x2,x3,x4,x5,num_so_far,num_total batch_idx = 0 @@ -236,10 +254,13 @@ def training_batch_generator_full_shot_partial_reset(self,shot_list): Maskbuff = self.resize_buffer(Maskbuff,sig_len) Maskbuff[:,old_len:,:] = 0.0 - Xbuff[batch_idx,:,:] = sig - Ybuff[batch_idx,:,:] = res + Xbuff[batch_idx,:,:] = 0.0 + Ybuff[batch_idx,:,:] = 0.0 + Maskbuff[batch_idx,:,:] = 0.0 + + Xbuff[batch_idx,:sig_len,:] = sig + Ybuff[batch_idx,:sig_len,:] = res Maskbuff[batch_idx,:sig_len,:] = 1.0 - Maskbuff[batch_idx,sig_len:,:] = 0.0 batch_idx += 1 if batch_idx == batch_size: num_so_far += batch_size @@ -735,8 +756,10 @@ def __init__(self,generator): def fill_batch_queue(self): print("Starting process to fetch data") + count = 0 while True: self.queue.put(next(self.generator),True) + count += 1 def __next__(self): return self.queue.get(True) diff --git a/plasma/models/torch_runner.py b/plasma/models/torch_runner.py index 08629fbe..3d7885ba 100644 --- a/plasma/models/torch_runner.py +++ b/plasma/models/torch_runner.py @@ -45,7 +45,7 @@ def __init__(self,n_scalars,n_profiles,profile_size,layer_sizes_spatial, num_channels_tcn,kernel_size_temporal,dropout=0.1): super(FTCN, self).__init__() self.lin = InputBlock(n_scalars, n_profiles,profile_size, layer_sizes_spatial, kernel_size_spatial, linear_size, dropout) - self.input_layer = TimeDistributed(lin,batch_first=True) + self.input_layer = TimeDistributed(self.lin,batch_first=True) self.tcn = TCN(linear_size, output_size, num_channels_tcn , kernel_size_temporal, dropout) self.model = nn.Sequential(self.input_layer,self.tcn) @@ -102,8 +102,8 @@ def forward(self, x): if self.n_scalars == 0: x_profiles = x else: - x_scalars = x[:,:n_scalars] - x_profiles = x[:,n_scalars:] + x_scalars = x[:,:self.n_scalars] + x_profiles = x[:,self.n_scalars:] x_profiles = x_profiles.contiguous().view(x.size(0),self.n_profiles,self.profile_size) profile_features = self.net(x_profiles).view(x.size(0),-1) if self.n_scalars == 0: @@ -271,18 +271,18 @@ def build_torch_model(conf): # dim = 10 # lin = nn.Linear(input_size,intermediate_dim) - n_scalars, n_profile, profile_size = get_signal_dimensions(conf) + n_scalars, n_profiles, profile_size = get_signal_dimensions(conf) dim = n_scalars+n_profiles*profile_size input_size = dim output_size = 1 # intermediate_dim = 15 - layer_sizes_spatial = [40,20,20] + layer_sizes_spatial = [6,3,3]#[40,20,20] kernel_size_spatial = 3 - linear_size = 10 + linear_size = 5 - num_channels_tcn = [3]*5 - kernel_size_temporal = 3 + num_channels_tcn = [10,5,3,3]#[3]*5 + kernel_size_temporal = 3 #3 model = FTCN(n_scalars,n_profiles,profile_size,layer_sizes_spatial, kernel_size_spatial,linear_size,output_size,num_channels_tcn, kernel_size_temporal,dropout) @@ -300,16 +300,68 @@ def get_signal_dimensions(conf): num_channels = sig.num_channels if num_channels > 1: profile_size = num_channels - num_1D += 1 + n_profiles += 1 is_1D_region = True else: assert(not is_1D_region), "make sure all use_signals are ordered such that 1D signals come last!" assert(num_channels == 1) - num_0D += 1 + n_scalars += 1 is_1D_region = False return n_scalars,n_profiles,profile_size -def train_epoch(model,data_gen,loss_fn): +def apply_model_to_np(model,x): + # return model(Variable(torch.from_numpy(x).float()).unsqueeze(0)).squeeze(0).data.numpy() + return model(Variable(torch.from_numpy(x).float())).data.numpy() + + + +def make_predictions(conf,shot_list,loader,custom_path=None): + generator = loader.inference_batch_generator_full_shot(shot_list) + inference_model = build_torch_model(conf) + + if custom_path == None: + model_path = get_model_path(conf) + else: + model_path = custom_path + inference_model.load_state_dict(torch.load(model_path)) + #shot_list = shot_list.random_sublist(10) + + y_prime = [] + y_gold = [] + disruptive = [] + num_shots = len(shot_list) + + pbar = Progbar(num_shots) + while True: + x,y,mask,disr,lengths,num_so_far,num_total = next(generator) + #x, y, mask = Variable(torch.from_numpy(x_).float()), Variable(torch.from_numpy(y_).float()),Variable(torch.from_numpy(mask_).byte()) + output = apply_model_to_np(inference_model,x) + for batch_idx in range(x.shape[0]): + curr_length = lengths[batch_idx] + y_prime += [output[batch_idx,:curr_length,0]] + y_gold += [y[batch_idx,:curr_length,0]] + disruptive += [disr[batch_idx]] + pbar.add(1.0) + if len(disruptive) >= num_shots: + y_prime = y_prime[:num_shots] + y_gold = y_gold[:num_shots] + disruptive = disruptive[:num_shots] + break + return y_prime,y_gold,disruptive + +def make_predictions_and_evaluate_gpu(conf,shot_list,loader,custom_path = None): + y_prime,y_gold,disruptive = make_predictions(conf,shot_list,loader,custom_path) + analyzer = PerformanceAnalyzer(conf=conf) + roc_area = analyzer.get_roc_area(y_prime,y_gold,disruptive) + loss = get_loss_from_list(y_prime,y_gold,conf['data']['target']) + return y_prime,y_gold,disruptive,roc_area,loss + + +def get_model_path(conf): + return conf['paths']['model_save_path'] + 'torch/' + model_filename #save_prepath + model_filename + + +def train_epoch(model,data_gen,optimizer,loss_fn): loss = 0 total_loss = 0 num_so_far = 0 @@ -335,17 +387,19 @@ def train_epoch(model,data_gen,loss_fn): loss.backward() optimizer.step() step += 1 + print("[{}] [{}/{}] loss: {:.3f}, ave_loss: {:.3f}".format(step,num_so_far-num_so_far_start,num_total,loss.data[0],total_loss/step)) if num_so_far-num_so_far_start >= num_total: break - x_,y_,mask_,num_so_far_start,num_total = next(data_gen) - return step,loss,total_loss,num_so_far,1.0*num_so_far/num_total + x_,y_,mask_,num_so_far,num_total = next(data_gen) + return step,loss.data[0],total_loss,num_so_far,1.0*num_so_far/num_total def train(conf,shot_list_train,shot_list_validate,loader): np.random.seed(1) - data_gen = ProcessGenerator(partial(loader.training_batch_generator_full_shot_partial_reset,shot_list=shot_list_train)) + #data_gen = ProcessGenerator(partial(loader.training_batch_generator_full_shot_partial_reset,shot_list=shot_list_train)()) + data_gen = partial(loader.training_batch_generator_full_shot_partial_reset,shot_list=shot_list_train)() print('validate: {} shots, {} disruptive'.format(len(shot_list_validate),shot_list_validate.num_disruptive())) print('training: {} shots, {} disruptive'.format(len(shot_list_train),shot_list_train.num_disruptive())) @@ -358,6 +412,7 @@ def train(conf,shot_list_train,shot_list_validate,loader): # e = specific_builder.load_model_weights(train_model) num_epochs = conf['training']['num_epochs'] + patience = conf['callbacks']['patience'] lr_decay = conf['model']['lr_decay'] batch_size = conf['training']['batch_size'] lr = conf['model']['lr'] @@ -385,23 +440,25 @@ def train(conf,shot_list_train,shot_list_validate,loader): else: best_so_far = np.inf cmp_fn = min - optimizer = opt.Adam(model.parameters(),lr = lr) - model.train() + optimizer = opt.Adam(train_model.parameters(),lr = lr) + scheduler = opt.lr_scheduler.ExponentialLR(optimizer,lr_decay) + train_model.train() not_updated = 0 total_loss = 0 count = 0 - loss_fn = nn.MSELoss(size_average=False) - model_path = conf['paths']['model_save_path'] + model_filename #save_prepath + model_filename - makedirs_process_safe(conf['paths']['model_save_path']) + loss_fn = nn.MSELoss(size_average=True) + model_path = get_model_path(conf) + makedirs_process_safe(os.path.dirname(model_path)) while e < num_epochs-1: - print_unique('\nEpoch {}/{}'.format(e,num_epochs)) - (step,ave_loss,curr_loss,num_so_far,effective_epochs) = train_epoch(model,data_gen,loss_fn) + scheduler.step() + print('\nEpoch {}/{}'.format(e,num_epochs)) + (step,ave_loss,curr_loss,num_so_far,effective_epochs) = train_epoch(train_model,data_gen,optimizer,loss_fn) e = effective_epochs loader.verbose=False #True during the first iteration # if task_index == 0: # specific_builder.save_model_weights(train_model,int(round(e))) - model.save_state_dict(model_path) - _,_,_,roc_area,loss = mpi_make_predictions_and_evaluate(conf,shot_list_validate,loader) + torch.save(train_model.state_dict(),model_path) + _,_,_,roc_area,loss = make_predictions_and_evaluate_gpu(conf,shot_list_validate,loader) best_so_far = cmp_fn(roc_area,best_so_far) @@ -411,54 +468,13 @@ def train(conf,shot_list_train,shot_list_validate,loader): print('Validation Loss: {:.3e}'.format(loss)) print('Validation ROC: {:.4f}'.format(roc_area)) - if best_so_far != epoch_logs[conf['callbacks']['monitor']]: #only save model weights if quantity we are tracking is improving + if best_so_far != roc_area: #only save model weights if quantity we are tracking is improving print("No improvement, still saving model") not_updated += 1 else: print("Saving model") - model.save_state_dict(model_path) # specific_builder.delete_model_weights(train_model,int(round(e))) if not_updated > patience: print("Stopping training due to early stopping") break -def make_predictions(conf,shot_list,loader,custom_path=None): - generator = loader.inference_batch_generator_full_shot(shot_list) - inference_model = build_torch_model(conf) - - if custom_path == None: - model_path = conf['paths']['model_save_path'] + model_filename#save_prepath + model_filename - else: - model_path = custom_path - inference_model.load_state_dict(model_path) - #shot_list = shot_list.random_sublist(10) - - y_prime = [] - y_gold = [] - disruptive = [] - num_shots = len(shot_list) - - pbar = Progbar(num_shots) - while True: - x_,y_,mask_,disr_,num_so_far,num_total = next(generator) - x, y, mask = Variable(torch.from_numpy(x_).float()), Variable(torch.from_numpy(y_).float()),Variable(torch.from_numpy(mask_).byte()) - output = model(x) - for batch_idx in range(x.shape[0]) - y_prime[batch_idx] += [output[batch_idx,:,:]] - y_gold += [y_[batch_idx,:,:]] - disruptive += [disr[batch_idx]] - pbar.add(1.0) - if len(disruptive) >= num_shots: - y_prime = y_prime[:num_shots] - y_gold = y_gold[:num_shots] - disruptive = disruptive[:num_shots] - break - return y_prime,y_gold,disruptive - -def make_predictions_and_evaluate_gpu(conf,shot_list,loader,custom_path = None): - y_prime,y_gold,disruptive = make_predictions(conf,shot_list,loader,custom_path) - analyzer = PerformanceAnalyzer(conf=conf) - roc_area = analyzer.get_roc_area(y_prime,y_gold,disruptive) - loss = get_loss_from_list(y_prime,y_gold,conf['data']['target']) - return y_prime,y_gold,disruptive,roc_area,loss - From bbfa51bb2054dbf28829fb2e71069e71b18f7c7a Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Sat, 24 Mar 2018 17:26:54 -0400 Subject: [PATCH 058/272] fix bugs related to T_min warn becoming too large. performance computation was wrong when T_max_warn was smaller. Shots can't be cut if they are shorter than T_min_warn --- examples/tune_hyperparams.py | 10 ++++++---- plasma/preprocessor/normalize.py | 3 +++ plasma/utils/performance.py | 4 ++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/examples/tune_hyperparams.py b/examples/tune_hyperparams.py index 56b8eb69..0e60c39c 100644 --- a/examples/tune_hyperparams.py +++ b/examples/tune_hyperparams.py @@ -8,9 +8,9 @@ tunables = [] shallow = False num_nodes = 1 -num_trials = 20 +num_trials = 30 -t_warn = CategoricalHyperparam(['data','T_warning'],[0.256,1.024,10.024]) +t_warn = CategoricalHyperparam(['data','T_warning'],[0.256,1.024,4.096,10.024]) cut_ends = CategoricalHyperparam(['data','cut_shot_ends'],[False,True]) #for shallow if shallow: @@ -35,19 +35,21 @@ fac = CategoricalHyperparam(['data','positive_example_penalty'],[1.0,4.0,16.0]) target = CategoricalHyperparam(['target'],['maxhinge','hinge','ttdinv','ttd']) #target = CategoricalHyperparam(['target'],['hinge','ttdinv','ttd']) - batch_size = CategoricalHyperparam(['training','batch_size'],[128,256]) + batch_size = CategoricalHyperparam(['training','batch_size'],[64,128]) dropout_prob = CategoricalHyperparam(['model','dropout_prob'],[0.01,0.05,0.1]) - conv_filters = CategoricalHyperparam(['model','num_conv_filters'],[128,256]) + conv_filters = CategoricalHyperparam(['model','num_conv_filters'],[64,128,256]) conv_layers = IntegerHyperparam(['model','num_conv_layers'],2,4) rnn_layers = IntegerHyperparam(['model','rnn_layers'],1,3) rnn_size = CategoricalHyperparam(['model','rnn_size'],[128,256]) dense_size = CategoricalHyperparam(['model','dense_size'],[128,256]) extra_dense_input = CategoricalHyperparam(['model','extra_dense_input'],[False,True]) equalize_classes = CategoricalHyperparam(['data','equalize_classes'],[False,True]) + t_min_warn = CategoricalHyperparam(['data','T_min_warn'],[30,70,200,500,1000]) #rnn_length = CategoricalHyperparam(['model','length'],[32,128]) #tunables = [lr,lr_decay,fac,target,batch_size,dropout_prob] tunables = [lr,lr_decay,fac,target,batch_size,equalize_classes,dropout_prob] tunables += [conv_filters,conv_layers,rnn_layers,rnn_size,dense_size,extra_dense_input] + tunables += [t_min_warn] tunables += [cut_ends,t_warn] diff --git a/plasma/preprocessor/normalize.py b/plasma/preprocessor/normalize.py index 794e7837..dd6cd05f 100644 --- a/plasma/preprocessor/normalize.py +++ b/plasma/preprocessor/normalize.py @@ -141,6 +141,9 @@ def train_on_files(self,shot_files,use_shots,all_machines): def cut_end_of_shot(self,shot): cut_shot_ends = self.conf['data']['cut_shot_ends'] if not self.inference_mode and cut_shot_ends: #only cut shots during training + if shot.ttd.shape[0] <= T_min_warn: + print("not cutting shot since T_min_warn is larger than length of shot") + return T_min_warn = self.conf['data']['T_min_warn'] for key in shot.signals_dict: shot.signals_dict[key] = shot.signals_dict[key][:-T_min_warn,:] diff --git a/plasma/utils/performance.py b/plasma/utils/performance.py index 6abac58e..f09a06e3 100644 --- a/plasma/utils/performance.py +++ b/plasma/utils/performance.py @@ -25,6 +25,9 @@ def __init__(self,results_dir=None,shots_dir=None,i = 0,T_min_warn = None,T_max_ self.T_min_warn = T_min_warn_def if T_max_warn == None: self.T_max_warn = T_max_warn_def + if self.T_max_warn < self.T_min_warn: + print("T max warn is too small: need to increase artificially.") #computation of statistics is only correct if T_max_warn is larger than T_min_warn + self.T_max_warn = self.T_min_warn + 1 self.verbose = verbose self.results_dir = results_dir self.shots_dir = shots_dir @@ -293,6 +296,7 @@ def create_acceptable_region(self,truth,mode): else: print('Error Invalid Mode for acceptable region') exit(1) + assert(self.T_max_warn > self.T_min_warn) acceptable = np.zeros_like(truth,dtype=bool) if acceptable_timesteps > 0: From 932220257ae4c5a0c7b0eaf784fb6410c9bd20eb Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Sat, 24 Mar 2018 17:28:15 -0400 Subject: [PATCH 059/272] added functionality to compute multiple ROC values for different T_min values at once. Add option to use ProcessGenerator or not --- plasma/models/mpi_runner.py | 63 ++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index d96adda1..14d4a744 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -171,9 +171,10 @@ def get_val(self): class MPIModel(): - def __init__(self,model,optimizer,comm,batch_iterator,batch_size,num_replicas=None,warmup_steps=1000,lr=0.01,num_batches_minimum=100): + def __init__(self,model,optimizer,comm,batch_iterator,batch_size,num_replicas=None,warmup_steps=1000,lr=0.01,num_batches_minimum=100,conf=None): random.seed(task_index) np.random.seed(task_index) + self.conf = conf self.start_time = time.time() self.epoch = 0 self.num_so_far = 0 @@ -200,10 +201,14 @@ def __init__(self,model,optimizer,comm,batch_iterator,batch_size,num_replicas=No def set_batch_iterator_func(self): - self.batch_iterator_func = ProcessGenerator(self.batch_iterator()) + if self.conf is not None and 'use_process_generator' in conf['training'] and conf['training']['use_process_generator']: + self.batch_iterator_func = ProcessGenerator(self.batch_iterator()) + else: + self.batch_iterator_func = self.batch_iterator() def close(self): - self.batch_iterator_func.__exit__() + if hasattr(self.batch_iterator_func,'__exit__'): + self.batch_iterator_func.__exit__() def set_lr(self,lr): self.lr = lr @@ -642,6 +647,24 @@ def mpi_make_predictions_and_evaluate(conf,shot_list,loader,custom_path=None): loss = get_loss_from_list(y_prime,y_gold,conf['data']['target']) return y_prime,y_gold,disruptive,roc_area,loss +def mpi_make_predictions_and_evaluate_multiple_times(conf,shot_list,loader,times,custom_path=None): + y_prime,y_gold,disruptive = mpi_make_predictions(conf,shot_list,loader,custom_path) + areas = [] + losses = [] + for T_min_curr in times: + #if 'monitor_test' in conf['callbacks'].keys() and conf['callbacks']['monitor_test']: + conf_curr = deepcopy(conf) + T_min_warn_orig = conf['data']['T_min_warn'] + conf_curr['data']['T_min_warn'] = T_min_curr + assert(conf['data']['T_min_warn'] == T_min_warn_orig) + analyzer = PerformanceAnalyzer(conf=conf_curr) + roc_area = analyzer.get_roc_area(y_prime,y_gold,disruptive) + #shot_list.set_weights(analyzer.get_shot_difficulty(y_prime,y_gold,disruptive)) + loss = get_loss_from_list(y_prime,y_gold,conf['data']['target']) + areas.append(roc_area) + losses.append(loss) + return areas,losses + def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=None,shot_list_test=None): @@ -680,7 +703,7 @@ def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=Non #{}batch_generator = partial(loader.training_batch_generator_process,shot_list=shot_list_train) print("warmup {}".format(warmup_steps)) - mpi_model = MPIModel(train_model,optimizer,comm,batch_generator,batch_size,lr=lr,warmup_steps = warmup_steps,num_batches_minimum=num_batches_minimum) + mpi_model = MPIModel(train_model,optimizer,comm,batch_generator,batch_size,lr=lr,warmup_steps = warmup_steps,num_batches_minimum=num_batches_minimum,conf=conf) mpi_model.compile(conf['model']['optimizer'],clipnorm,conf['data']['target'].loss) tensorboard = None @@ -709,6 +732,7 @@ def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=Non cmp_fn = min while e < num_epochs-1: + print_unique("begin epoch {} 0".format(e)) if task_index == 0: callbacks.on_epoch_begin(int(round(e))) mpi_model.set_lr(lr*lr_decay**e) @@ -733,18 +757,15 @@ def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=Non mpi_model.set_batch_iterator_func() if 'monitor_test' in conf['callbacks'].keys() and conf['callbacks']['monitor_test']: - conf_curr = deepcopy(conf) - T_min_warn_orig = conf['data']['T_min_warn'] - for T_min_curr in conf_curr['callbacks']['monitor_times']: - conf_curr['data']['T_min_warn'] = T_min_curr - assert(conf['data']['T_min_warn'] == T_min_warn_orig) - if shot_list_test is not None: - _,_,_,roc_area_t,_ = mpi_make_predictions_and_evaluate(conf_curr,shot_list_test,loader) - print_unique('epoch {}, test_roc_{} = {}'.format(int(round(e)),T_min_curr,roc_area_t)) - #epoch_logs['test_roc_{}'.format(T_min_curr)] = roc_area_t - _,_,_,roc_area_v,_ = mpi_make_predictions_and_evaluate(conf_curr,shot_list_validate,loader) - print_unique('epoch {}, val_roc_{} = {}'.format(int(round(e)),T_min_curr,roc_area_v)) - #epoch_logs['val_roc_{}'.format(T_min_curr)] = roc_area_v + times = conf['callbacks']['monitor_times'] + roc_areas,losses = mpi_make_predictions_and_evaluate_multiple_times(conf,shot_list_validate,loader,times) + for roc,t in zip(roc_areas,times): + print_unique('epoch {}, val_roc_{} = {}'.format(int(round(e)),t,roc)) + if shot_list_test is not None: + roc_areas,losses = mpi_make_predictions_and_evaluate_multiple_times(conf,shot_list_test,loader,times) + for roc,t in zip(roc_areas,times): + print_unique('epoch {}, test_roc_{} = {}'.format(int(round(e)),t,roc)) + epoch_logs['val_roc'] = roc_area epoch_logs['val_loss'] = loss epoch_logs['train_loss'] = ave_loss @@ -764,8 +785,12 @@ def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=Non if hasattr(mpi_model.model,'stop_training'): stop_training = mpi_model.model.stop_training if best_so_far != epoch_logs[conf['callbacks']['monitor']]: #only save model weights if quantity we are tracking is improving - print("Not saving model weights") - specific_builder.delete_model_weights(train_model,int(round(e))) + if 'monitor_test' in conf['callbacks'].keys() and conf['callbacks']['monitor_test']: + + print("No improvement, saving model weights anyways") + else: + print("Not saving model weights") + specific_builder.delete_model_weights(train_model,int(round(e))) #tensorboard if backend != 'theano': @@ -773,7 +798,9 @@ def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=Non val_steps = 1 tensorboard.on_epoch_end(val_generator,val_steps,int(round(e)),epoch_logs) + print_unique("end epoch {} 0".format(e)) stop_training = comm.bcast(stop_training,root=0) + print_unique("end epoch {} 1".format(e)) if stop_training: print("Stopping training due to early stopping") break From 4054c035717551cbd859bdb5ca4318f344813daf Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Sat, 24 Mar 2018 17:31:37 -0400 Subject: [PATCH 060/272] fix small error in variable ordering --- plasma/preprocessor/normalize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plasma/preprocessor/normalize.py b/plasma/preprocessor/normalize.py index dd6cd05f..c0b679bf 100644 --- a/plasma/preprocessor/normalize.py +++ b/plasma/preprocessor/normalize.py @@ -141,10 +141,10 @@ def train_on_files(self,shot_files,use_shots,all_machines): def cut_end_of_shot(self,shot): cut_shot_ends = self.conf['data']['cut_shot_ends'] if not self.inference_mode and cut_shot_ends: #only cut shots during training + T_min_warn = self.conf['data']['T_min_warn'] if shot.ttd.shape[0] <= T_min_warn: print("not cutting shot since T_min_warn is larger than length of shot") return - T_min_warn = self.conf['data']['T_min_warn'] for key in shot.signals_dict: shot.signals_dict[key] = shot.signals_dict[key][:-T_min_warn,:] shot.ttd = shot.ttd[:-T_min_warn] From 4dc0edb338cf19262bddb2c05e5c2f1cb40bf9d6 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Sat, 24 Mar 2018 20:32:41 -0400 Subject: [PATCH 061/272] ensure shot is only cut if it remains longer than RNN length after cutting --- plasma/preprocessor/normalize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plasma/preprocessor/normalize.py b/plasma/preprocessor/normalize.py index c0b679bf..4818c7aa 100644 --- a/plasma/preprocessor/normalize.py +++ b/plasma/preprocessor/normalize.py @@ -142,8 +142,8 @@ def cut_end_of_shot(self,shot): cut_shot_ends = self.conf['data']['cut_shot_ends'] if not self.inference_mode and cut_shot_ends: #only cut shots during training T_min_warn = self.conf['data']['T_min_warn'] - if shot.ttd.shape[0] <= T_min_warn: - print("not cutting shot since T_min_warn is larger than length of shot") + if shot.ttd.shape[0] - T_min_warn <= max(self.conf['model']['length'],0): + print("not cutting shot since length of shot after cutting by T_min_warn would be shorter than RNN length") return for key in shot.signals_dict: shot.signals_dict[key] = shot.signals_dict[key][:-T_min_warn,:] From c2e2ac68edbf3c366f96c38ee50efe0227e3b72c Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Sat, 24 Mar 2018 21:24:37 -0400 Subject: [PATCH 062/272] changed hash computation (removing enconding and decoding since there was a bug where it couldn't decode the dumps due to truncated unicode escape character \xXX --- plasma/utils/downloading.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plasma/utils/downloading.py b/plasma/utils/downloading.py index 65a9bfa0..91f4e968 100644 --- a/plasma/utils/downloading.py +++ b/plasma/utils/downloading.py @@ -56,6 +56,8 @@ def general_object_hash(o): def myhash(x): return int(hashlib.md5((dill.dumps(x).decode('unicode_escape')).encode('utf-8')).hexdigest(),16) + #return int(hashlib.md5((dill.dumps(x))).hexdigest(),16) + #return int(hashlib.md5((dill.dumps(x))))#.decode('unicode_escape')).encode('utf-8')).hexdigest(),16) def get_missing_value_array(): From a083b949580eb2e02dad51c7a19498b908a1f353 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Thu, 5 Apr 2018 18:47:58 -0700 Subject: [PATCH 063/272] added support for multiple times to disruption ROC evaluation for shallow models --- examples/learn.py | 2 +- plasma/models/runner.py | 2 +- plasma/models/shallow_runner.py | 36 ++++++++++++++++++++++++++++++++- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/examples/learn.py b/examples/learn.py index a1b9a94a..29a7e1c2 100644 --- a/examples/learn.py +++ b/examples/learn.py @@ -93,7 +93,7 @@ ##################################################### #train(conf,shot_list_train,loader) if not only_predict: - p = old_mp.Process(target = train,args=(conf,shot_list_train,shot_list_validate,loader)) + p = old_mp.Process(target = train,args=(conf,shot_list_train,shot_list_validate,loader,shot_list_test)) p.start() p.join() diff --git a/plasma/models/runner.py b/plasma/models/runner.py index 11f3ff3d..0a4506b3 100644 --- a/plasma/models/runner.py +++ b/plasma/models/runner.py @@ -24,7 +24,7 @@ backend = conf['model']['backend'] -def train(conf,shot_list_train,shot_list_validate,loader): +def train(conf,shot_list_train,shot_list_validate,loader,shot_list_test=None): loader.set_inference_mode(False) np.random.seed(1) diff --git a/plasma/models/shallow_runner.py b/plasma/models/shallow_runner.py index b9ccf883..04c5db8a 100644 --- a/plasma/models/shallow_runner.py +++ b/plasma/models/shallow_runner.py @@ -15,6 +15,7 @@ import time import datetime import os +from copy import deepcopy from functools import partial import pathos.multiprocessing as mp from xgboost import XGBClassifier @@ -284,7 +285,7 @@ def build_callbacks(conf): return cbks.CallbackList(callbacks) -def train(conf,shot_list_train,shot_list_validate,loader): +def train(conf,shot_list_train,shot_list_validate,loader,shot_list_test=None): np.random.seed(1) @@ -367,6 +368,20 @@ def train(conf,shot_list_train,shot_list_validate,loader): Y_predv = model.predict(Xv) print("Validate") print(classification_report(Yv,Y_predv)) + + + if 'monitor_test' in conf['callbacks'].keys() and conf['callbacks']['monitor_test']: + times = conf['callbacks']['monitor_times'] + roc_areas,losses = make_predictions_and_evaluate_multiple_times(conf,shot_list_validate,loader,times) + for roc,t in zip(roc_areas,times): + print('val_roc_{} = {}'.format(t,roc)) + if shot_list_test is not None: + roc_areas,losses = make_predictions_and_evaluate_multiple_times(conf,shot_list_test,loader,times) + for roc,t in zip(roc_areas,times): + print('test_roc_{} = {}'.format(t,roc)) + + + #print(confusion_matrix(Y,Y_pred)) _,_,_,roc_area,loss = make_predictions_and_evaluate_gpu(conf,shot_list_validate,loader) # _,_,_,roc_area_train,loss_train = make_predictions_and_evaluate_gpu(conf,shot_list_train,loader) @@ -378,6 +393,8 @@ def train(conf,shot_list_train,shot_list_validate,loader): epoch_logs['val_loss'] = loss # epoch_logs['train_roc'] = roc_area_train # epoch_logs['train_loss'] = loss_train + + callbacks.on_epoch_end(0, epoch_logs) @@ -432,3 +449,20 @@ def make_predictions_and_evaluate_gpu(conf,shot_list,loader,custom_path = None): loss = get_loss_from_list(y_prime,y_gold,conf['data']['target']) return y_prime,y_gold,disruptive,roc_area,loss +def make_predictions_and_evaluate_multiple_times(conf,shot_list,loader,times,custom_path=None): + y_prime,y_gold,disruptive = make_predictions(conf,shot_list,loader,custom_path) + areas = [] + losses = [] + for T_min_curr in times: + #if 'monitor_test' in conf['callbacks'].keys() and conf['callbacks']['monitor_test']: + conf_curr = deepcopy(conf) + T_min_warn_orig = conf['data']['T_min_warn'] + conf_curr['data']['T_min_warn'] = T_min_curr + assert(conf['data']['T_min_warn'] == T_min_warn_orig) + analyzer = PerformanceAnalyzer(conf=conf_curr) + roc_area = analyzer.get_roc_area(y_prime,y_gold,disruptive) + #shot_list.set_weights(analyzer.get_shot_difficulty(y_prime,y_gold,disruptive)) + loss = get_loss_from_list(y_prime,y_gold,conf['data']['target']) + areas.append(roc_area) + losses.append(loss) + return areas,losses From 638306bed08eff51b7208cfe684f96f2299ef675 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Thu, 5 Apr 2018 21:50:00 -0400 Subject: [PATCH 064/272] added support for torch model --- examples/learn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/learn.py b/examples/learn.py index a1b9a94a..e4d74256 100644 --- a/examples/learn.py +++ b/examples/learn.py @@ -33,7 +33,7 @@ from plasma.preprocessor.preprocess import Preprocessor, guarantee_preprocessed from plasma.models.loader import Loader -if conf['model']['torch']: +if 'torch' in conf['model'].keys() and conf['model']['torch']: from plasma.models.torch_runner import train, make_predictions_and_evaluate_gpu elif conf['model']['shallow']: from plasma.models.shallow_runner import train, make_predictions_and_evaluate_gpu From 46f2df9e11057d053821775a5bf3d68b875728e7 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Thu, 5 Apr 2018 22:14:09 -0400 Subject: [PATCH 065/272] add conf with options for not using process generator, using torch model, and printing validation and test rocs during training --- examples/conf.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/examples/conf.yaml b/examples/conf.yaml index c59d00d2..398d3d12 100644 --- a/examples/conf.yaml +++ b/examples/conf.yaml @@ -10,7 +10,7 @@ paths: signal_prepath: '/signal_data/' #/signal_data/jet/ shot_list_dir: '/shot_lists/' tensorboard_save_path: '/Graph/' - data: jet_data #'d3d_to_jet_data' #'d3d_to_jet_data' # 'jet_to_d3d_data' #jet_data + data: d3d_data_0D #'d3d_to_jet_data' #'d3d_to_jet_data' # 'jet_to_d3d_data' #jet_data specific_signals: [] #['q95','li','ip','betan','energy','lm','pradcore','pradedge','pradtot','pin','torquein','tmamp1','tmamp2','tmfreq1','tmfreq2','pechin','energydt','ipdirect','etemp_profile','edens_profile'] #if left empty will use all valid signals defined on a machine. Only use if need a custom set executable: "mpi_learn.py" shallow_executable: "learn.py" @@ -54,9 +54,9 @@ data: floatx: 'float32' model: - use_bidirectional: false use_batch_norm: false - shallow: False + torch: False + shallow: True shallow_model: num_samples: 1000000 #1000000 #the number of samples to use for training type: "xgboost" #"xgboost" #"xgboost" #"random_forest" "xgboost" @@ -120,7 +120,8 @@ training: data_parallel: False hyperparam_tuning: False batch_generator_warmup_steps: 0 - num_batches_minimum: 200 #minimum number of batches per epoch + use_process_generator: False + num_batches_minimum: 20 #minimum number of batches per epoch ranking_difficulty_fac: 1.0 #how much to upweight incorrectly classified shots during training callbacks: list: ['earlystop'] @@ -129,6 +130,8 @@ callbacks: monitor: 'val_roc' patience: 5 write_grads: False + monitor_test: True + monitor_times: [30,70,200,500,1000] env: name: 'frnn' type: 'anaconda' From 2bff1ef9f46dfb0ce9a13c0cd10fa9686dc25928 Mon Sep 17 00:00:00 2001 From: ASvyatkovskiy Date: Mon, 6 Aug 2018 01:14:32 -0400 Subject: [PATCH 066/272] Do not broadcast weights after all-reduce --- plasma/models/mpi_runner.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 3f5bdf3b..90753403 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -338,11 +338,7 @@ def set_new_weights(self,deltas,num_replicas=None): self.optimizer.set_lr(effective_lr) global_deltas = self.optimizer.get_deltas(global_deltas) - if self.comm.rank == 0: - new_weights = self.get_new_weights(global_deltas) - else: - new_weights = None - new_weights = self.comm.bcast(new_weights,root=0) + new_weights = self.get_new_weights(global_deltas) self.model.set_weights(new_weights) def build_callbacks(self,conf,callbacks_list): From ea5b3e62aef6f3f19ccc4bb3e31b71c311513285 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Mon, 6 Aug 2018 11:25:11 +0200 Subject: [PATCH 067/272] Update mpi_runner.py Ensure all ranks have equal weights to begin with by broadcasting weights after compilation of the model. --- plasma/models/mpi_runner.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 90753403..35181d68 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -226,7 +226,16 @@ def compile(self,optimizer,clipnorm,loss='mse'): else: print("Optimizer not implemented yet") exit(1) - self.model.compile(optimizer=optimizer_class,loss=loss) + self.model.compile(optimizer=optimizer_class,loss=loss) + self.ensure_equal_weights() + + def ensure_equal_weights(self): + if task_index == 0: + new_weights = self.model.get_weights() + else: + new_weights = None + nw = comm.bcast(new_weights,root=0) + self.model.set_weights(nw) From 68ac282350568fe6e162c80bda388f769ef8215a Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Mon, 12 Nov 2018 03:12:05 -0500 Subject: [PATCH 068/272] added later JET campaign shot lists --- data/shot_lists/jet/ILW_clear_full.txt | 10425 +++++++++++++++++++++++ data/shot_lists/jet/ILW_clear_late.txt | 7201 ++++++++++++++++ data/shot_lists/jet/ILW_unint_full.txt | 626 ++ data/shot_lists/jet/ILW_unint_late.txt | 380 + 4 files changed, 18632 insertions(+) create mode 100644 data/shot_lists/jet/ILW_clear_full.txt create mode 100644 data/shot_lists/jet/ILW_clear_late.txt create mode 100644 data/shot_lists/jet/ILW_unint_full.txt create mode 100644 data/shot_lists/jet/ILW_unint_late.txt diff --git a/data/shot_lists/jet/ILW_clear_full.txt b/data/shot_lists/jet/ILW_clear_full.txt new file mode 100644 index 00000000..b6b95ad3 --- /dev/null +++ b/data/shot_lists/jet/ILW_clear_full.txt @@ -0,0 +1,10425 @@ +80128 -1.000000 +80129 -1.000000 +80130 -1.000000 +80131 -1.000000 +80132 -1.000000 +80133 -1.000000 +80134 -1.000000 +80135 -1.000000 +80136 -1.000000 +80137 -1.000000 +80138 -1.000000 +80139 -1.000000 +80140 -1.000000 +80141 -1.000000 +80142 -1.000000 +80143 -1.000000 +80144 -1.000000 +80145 -1.000000 +80146 -1.000000 +80147 -1.000000 +80148 -1.000000 +80149 -1.000000 +80150 -1.000000 +80151 -1.000000 +80152 -1.000000 +80153 -1.000000 +80154 -1.000000 +80155 -1.000000 +80156 -1.000000 +80157 -1.000000 +80158 -1.000000 +80159 -1.000000 +80160 -1.000000 +80161 -1.000000 +80162 -1.000000 +80163 -1.000000 +80164 -1.000000 +80165 -1.000000 +80166 -1.000000 +80167 -1.000000 +80168 -1.000000 +80169 -1.000000 +80170 -1.000000 +80171 -1.000000 +80172 -1.000000 +80173 -1.000000 +80174 -1.000000 +80175 -1.000000 +80176 -1.000000 +80177 -1.000000 +80178 -1.000000 +80179 -1.000000 +80180 -1.000000 +80182 -1.000000 +80183 -1.000000 +80184 -1.000000 +80185 -1.000000 +80186 -1.000000 +80187 -1.000000 +80188 -1.000000 +80189 -1.000000 +80190 -1.000000 +80191 -1.000000 +80192 -1.000000 +80193 -1.000000 +80194 -1.000000 +80196 -1.000000 +80197 -1.000000 +80198 -1.000000 +80199 -1.000000 +80200 -1.000000 +80201 -1.000000 +80202 -1.000000 +80203 -1.000000 +80204 -1.000000 +80205 -1.000000 +80206 -1.000000 +80207 -1.000000 +80208 -1.000000 +80209 -1.000000 +80210 -1.000000 +80211 -1.000000 +80212 -1.000000 +80213 -1.000000 +80214 -1.000000 +80215 -1.000000 +80216 -1.000000 +80217 -1.000000 +80218 -1.000000 +80219 -1.000000 +80220 -1.000000 +80221 -1.000000 +80222 -1.000000 +80223 -1.000000 +80224 -1.000000 +80225 -1.000000 +80226 -1.000000 +80227 -1.000000 +80228 -1.000000 +80229 -1.000000 +80230 -1.000000 +80231 -1.000000 +80232 -1.000000 +80233 -1.000000 +80234 -1.000000 +80235 -1.000000 +80236 -1.000000 +80237 -1.000000 +80238 -1.000000 +80239 -1.000000 +80240 -1.000000 +80241 -1.000000 +80242 -1.000000 +80243 -1.000000 +80244 -1.000000 +80245 -1.000000 +80246 -1.000000 +80247 -1.000000 +80248 -1.000000 +80249 -1.000000 +80250 -1.000000 +80251 -1.000000 +80252 -1.000000 +80253 -1.000000 +80254 -1.000000 +80255 -1.000000 +80256 -1.000000 +80257 -1.000000 +80258 -1.000000 +80259 -1.000000 +80260 -1.000000 +80261 -1.000000 +80262 -1.000000 +80263 -1.000000 +80264 -1.000000 +80265 -1.000000 +80266 -1.000000 +80267 -1.000000 +80268 -1.000000 +80269 -1.000000 +80270 -1.000000 +80271 -1.000000 +80272 -1.000000 +80273 -1.000000 +80274 -1.000000 +80275 -1.000000 +80276 -1.000000 +80277 -1.000000 +80278 -1.000000 +80279 -1.000000 +80280 -1.000000 +80281 -1.000000 +80282 -1.000000 +80283 -1.000000 +80284 -1.000000 +80285 -1.000000 +80286 -1.000000 +80287 -1.000000 +80288 -1.000000 +80289 -1.000000 +80290 -1.000000 +80291 -1.000000 +80292 -1.000000 +80293 -1.000000 +80294 -1.000000 +80295 -1.000000 +80296 -1.000000 +80297 -1.000000 +80298 -1.000000 +80299 -1.000000 +80300 -1.000000 +80301 -1.000000 +80302 -1.000000 +80303 -1.000000 +80304 -1.000000 +80305 -1.000000 +80306 -1.000000 +80307 -1.000000 +80308 -1.000000 +80309 -1.000000 +80310 -1.000000 +80311 -1.000000 +80312 -1.000000 +80313 -1.000000 +80314 -1.000000 +80315 -1.000000 +80316 -1.000000 +80317 -1.000000 +80318 -1.000000 +80319 -1.000000 +80320 -1.000000 +80321 -1.000000 +80322 -1.000000 +80323 -1.000000 +80324 -1.000000 +80325 -1.000000 +80326 -1.000000 +80327 -1.000000 +80328 -1.000000 +80329 -1.000000 +80330 -1.000000 +80332 -1.000000 +80333 -1.000000 +80334 -1.000000 +80335 -1.000000 +80336 -1.000000 +80337 -1.000000 +80338 -1.000000 +80339 -1.000000 +80340 -1.000000 +80341 -1.000000 +80342 -1.000000 +80343 -1.000000 +80344 -1.000000 +80345 -1.000000 +80346 -1.000000 +80347 -1.000000 +80348 -1.000000 +80349 -1.000000 +80350 -1.000000 +80351 -1.000000 +80352 -1.000000 +80353 -1.000000 +80354 -1.000000 +80355 -1.000000 +80356 -1.000000 +80357 -1.000000 +80358 -1.000000 +80359 -1.000000 +80360 -1.000000 +80361 -1.000000 +80362 -1.000000 +80363 -1.000000 +80364 -1.000000 +80365 -1.000000 +80366 -1.000000 +80367 -1.000000 +80368 -1.000000 +80369 -1.000000 +80370 -1.000000 +80371 -1.000000 +80372 -1.000000 +80373 -1.000000 +80374 -1.000000 +80375 -1.000000 +80376 -1.000000 +80377 -1.000000 +80378 -1.000000 +80379 -1.000000 +80380 -1.000000 +80381 -1.000000 +80382 -1.000000 +80383 -1.000000 +80384 -1.000000 +80385 -1.000000 +80386 -1.000000 +80387 -1.000000 +80388 -1.000000 +80389 -1.000000 +80390 -1.000000 +80391 -1.000000 +80392 -1.000000 +80393 -1.000000 +80394 -1.000000 +80395 -1.000000 +80396 -1.000000 +80397 -1.000000 +80398 -1.000000 +80399 -1.000000 +80400 -1.000000 +80401 -1.000000 +80402 -1.000000 +80403 -1.000000 +80404 -1.000000 +80405 -1.000000 +80407 -1.000000 +80408 -1.000000 +80409 -1.000000 +80410 -1.000000 +80411 -1.000000 +80412 -1.000000 +80413 -1.000000 +80414 -1.000000 +80415 -1.000000 +80416 -1.000000 +80417 -1.000000 +80418 -1.000000 +80419 -1.000000 +80420 -1.000000 +80421 -1.000000 +80422 -1.000000 +80423 -1.000000 +80424 -1.000000 +80425 -1.000000 +80426 -1.000000 +80427 -1.000000 +80428 -1.000000 +80429 -1.000000 +80430 -1.000000 +80431 -1.000000 +80432 -1.000000 +80433 -1.000000 +80434 -1.000000 +80435 -1.000000 +80436 -1.000000 +80437 -1.000000 +80438 -1.000000 +80439 -1.000000 +80440 -1.000000 +80441 -1.000000 +80442 -1.000000 +80443 -1.000000 +80444 -1.000000 +80445 -1.000000 +80446 -1.000000 +80447 -1.000000 +80448 -1.000000 +80449 -1.000000 +80450 -1.000000 +80451 -1.000000 +80452 -1.000000 +80453 -1.000000 +80454 -1.000000 +80455 -1.000000 +80456 -1.000000 +80457 -1.000000 +80458 -1.000000 +80459 -1.000000 +80460 -1.000000 +80461 -1.000000 +80462 -1.000000 +80463 -1.000000 +80464 -1.000000 +80465 -1.000000 +80466 -1.000000 +80467 -1.000000 +80468 -1.000000 +80469 -1.000000 +80470 -1.000000 +80471 -1.000000 +80472 -1.000000 +80473 -1.000000 +80474 -1.000000 +80475 -1.000000 +80476 -1.000000 +80477 -1.000000 +80478 -1.000000 +80479 -1.000000 +80480 -1.000000 +80481 -1.000000 +80482 -1.000000 +80483 -1.000000 +80484 -1.000000 +80485 -1.000000 +80486 -1.000000 +80488 -1.000000 +80489 -1.000000 +80490 -1.000000 +80491 -1.000000 +80492 -1.000000 +80493 -1.000000 +80494 -1.000000 +80495 -1.000000 +80496 -1.000000 +80497 -1.000000 +80498 -1.000000 +80499 -1.000000 +80500 -1.000000 +80501 -1.000000 +80502 -1.000000 +80503 -1.000000 +80504 -1.000000 +80505 -1.000000 +80506 -1.000000 +80507 -1.000000 +80508 -1.000000 +80509 -1.000000 +80510 -1.000000 +80513 -1.000000 +80514 -1.000000 +80515 -1.000000 +80516 -1.000000 +80517 -1.000000 +80518 -1.000000 +80519 -1.000000 +80520 -1.000000 +80521 -1.000000 +80522 -1.000000 +80523 -1.000000 +80524 -1.000000 +80525 -1.000000 +80526 -1.000000 +80527 -1.000000 +80528 -1.000000 +80529 -1.000000 +80530 -1.000000 +80531 -1.000000 +80532 -1.000000 +80533 -1.000000 +80534 -1.000000 +80535 -1.000000 +80536 -1.000000 +80537 -1.000000 +80538 -1.000000 +80539 -1.000000 +80540 -1.000000 +80541 -1.000000 +80542 -1.000000 +80543 -1.000000 +80544 -1.000000 +80545 -1.000000 +80546 -1.000000 +80547 -1.000000 +80548 -1.000000 +80549 -1.000000 +80550 -1.000000 +80551 -1.000000 +80552 -1.000000 +80553 -1.000000 +80554 -1.000000 +80555 -1.000000 +80556 -1.000000 +80557 -1.000000 +80558 -1.000000 +80559 -1.000000 +80560 -1.000000 +80561 -1.000000 +80562 -1.000000 +80563 -1.000000 +80564 -1.000000 +80565 -1.000000 +80566 -1.000000 +80567 -1.000000 +80568 -1.000000 +80569 -1.000000 +80570 -1.000000 +80571 -1.000000 +80572 -1.000000 +80573 -1.000000 +80574 -1.000000 +80575 -1.000000 +80576 -1.000000 +80577 -1.000000 +80578 -1.000000 +80579 -1.000000 +80580 -1.000000 +80581 -1.000000 +80582 -1.000000 +80583 -1.000000 +80584 -1.000000 +80585 -1.000000 +80586 -1.000000 +80587 -1.000000 +80588 -1.000000 +80589 -1.000000 +80590 -1.000000 +80591 -1.000000 +80592 -1.000000 +80593 -1.000000 +80594 -1.000000 +80595 -1.000000 +80596 -1.000000 +80597 -1.000000 +80598 -1.000000 +80599 -1.000000 +80600 -1.000000 +80601 -1.000000 +80602 -1.000000 +80603 -1.000000 +80604 -1.000000 +80606 -1.000000 +80607 -1.000000 +80608 -1.000000 +80610 -1.000000 +80611 -1.000000 +80612 -1.000000 +80613 -1.000000 +80614 -1.000000 +80615 -1.000000 +80616 -1.000000 +80617 -1.000000 +80619 -1.000000 +80620 -1.000000 +80621 -1.000000 +80622 -1.000000 +80623 -1.000000 +80624 -1.000000 +80625 -1.000000 +80626 -1.000000 +80627 -1.000000 +80628 -1.000000 +80629 -1.000000 +80630 -1.000000 +80631 -1.000000 +80632 -1.000000 +80633 -1.000000 +80634 -1.000000 +80635 -1.000000 +80636 -1.000000 +80637 -1.000000 +80638 -1.000000 +80639 -1.000000 +80640 -1.000000 +80641 -1.000000 +80642 -1.000000 +80643 -1.000000 +80644 -1.000000 +80645 -1.000000 +80646 -1.000000 +80647 -1.000000 +80648 -1.000000 +80649 -1.000000 +80650 -1.000000 +80651 -1.000000 +80653 -1.000000 +80654 -1.000000 +80655 -1.000000 +80656 -1.000000 +80657 -1.000000 +80658 -1.000000 +80659 -1.000000 +80660 -1.000000 +80661 -1.000000 +80662 -1.000000 +80663 -1.000000 +80664 -1.000000 +80665 -1.000000 +80666 -1.000000 +80667 -1.000000 +80668 -1.000000 +80669 -1.000000 +80670 -1.000000 +80671 -1.000000 +80672 -1.000000 +80673 -1.000000 +80674 -1.000000 +80675 -1.000000 +80676 -1.000000 +80677 -1.000000 +80678 -1.000000 +80679 -1.000000 +80680 -1.000000 +80681 -1.000000 +80682 -1.000000 +80683 -1.000000 +80684 -1.000000 +80685 -1.000000 +80686 -1.000000 +80687 -1.000000 +80688 -1.000000 +80689 -1.000000 +80690 -1.000000 +80691 -1.000000 +80692 -1.000000 +80693 -1.000000 +80694 -1.000000 +80695 -1.000000 +80696 -1.000000 +80698 -1.000000 +80699 -1.000000 +80700 -1.000000 +80701 -1.000000 +80702 -1.000000 +80703 -1.000000 +80704 -1.000000 +80705 -1.000000 +80707 -1.000000 +80709 -1.000000 +80710 -1.000000 +80711 -1.000000 +80712 -1.000000 +80713 -1.000000 +80714 -1.000000 +80715 -1.000000 +80716 -1.000000 +80717 -1.000000 +80718 -1.000000 +80719 -1.000000 +80720 -1.000000 +80721 -1.000000 +80722 -1.000000 +80723 -1.000000 +80724 -1.000000 +80725 -1.000000 +80726 -1.000000 +80727 -1.000000 +80728 -1.000000 +80729 -1.000000 +80730 -1.000000 +80731 -1.000000 +80732 -1.000000 +80733 -1.000000 +80734 -1.000000 +80735 -1.000000 +80736 -1.000000 +80737 -1.000000 +80738 -1.000000 +80739 -1.000000 +80740 -1.000000 +80741 -1.000000 +80742 -1.000000 +80743 -1.000000 +80744 -1.000000 +80745 -1.000000 +80746 -1.000000 +80747 -1.000000 +80748 -1.000000 +80750 -1.000000 +80751 -1.000000 +80752 -1.000000 +80753 -1.000000 +80754 -1.000000 +80755 -1.000000 +80756 -1.000000 +80757 -1.000000 +80758 -1.000000 +80759 -1.000000 +80760 -1.000000 +80761 -1.000000 +80762 -1.000000 +80765 -1.000000 +80766 -1.000000 +80767 -1.000000 +80768 -1.000000 +80769 -1.000000 +80770 -1.000000 +80771 -1.000000 +80775 -1.000000 +80776 -1.000000 +80777 -1.000000 +80778 -1.000000 +80779 -1.000000 +80780 -1.000000 +80781 -1.000000 +80782 -1.000000 +80783 -1.000000 +80784 -1.000000 +80785 -1.000000 +80786 -1.000000 +80787 -1.000000 +80788 -1.000000 +80789 -1.000000 +80790 -1.000000 +80791 -1.000000 +80792 -1.000000 +80793 -1.000000 +80794 -1.000000 +80795 -1.000000 +80796 -1.000000 +80797 -1.000000 +80798 -1.000000 +80799 -1.000000 +80800 -1.000000 +80801 -1.000000 +80802 -1.000000 +80803 -1.000000 +80804 -1.000000 +80805 -1.000000 +80806 -1.000000 +80807 -1.000000 +80808 -1.000000 +80809 -1.000000 +80810 -1.000000 +80811 -1.000000 +80812 -1.000000 +80813 -1.000000 +80814 -1.000000 +80815 -1.000000 +80816 -1.000000 +80817 -1.000000 +80820 -1.000000 +80828 -1.000000 +80829 -1.000000 +80830 -1.000000 +80831 -1.000000 +80832 -1.000000 +80833 -1.000000 +80834 -1.000000 +80835 -1.000000 +80836 -1.000000 +80837 -1.000000 +80838 -1.000000 +80839 -1.000000 +80840 -1.000000 +80841 -1.000000 +80842 -1.000000 +80843 -1.000000 +80844 -1.000000 +80845 -1.000000 +80846 -1.000000 +80849 -1.000000 +80850 -1.000000 +80851 -1.000000 +80852 -1.000000 +80854 -1.000000 +80855 -1.000000 +80856 -1.000000 +80857 -1.000000 +80858 -1.000000 +80859 -1.000000 +80860 -1.000000 +80861 -1.000000 +80862 -1.000000 +80863 -1.000000 +80864 -1.000000 +80865 -1.000000 +80866 -1.000000 +80867 -1.000000 +80868 -1.000000 +80869 -1.000000 +80870 -1.000000 +80871 -1.000000 +80872 -1.000000 +80873 -1.000000 +80875 -1.000000 +80876 -1.000000 +80877 -1.000000 +80878 -1.000000 +80879 -1.000000 +80880 -1.000000 +80881 -1.000000 +80882 -1.000000 +80883 -1.000000 +80884 -1.000000 +80885 -1.000000 +80886 -1.000000 +80887 -1.000000 +80888 -1.000000 +80889 -1.000000 +80890 -1.000000 +80891 -1.000000 +80892 -1.000000 +80893 -1.000000 +80894 -1.000000 +80895 -1.000000 +80896 -1.000000 +80897 -1.000000 +80898 -1.000000 +80899 -1.000000 +80900 -1.000000 +80901 -1.000000 +80902 -1.000000 +80903 -1.000000 +80904 -1.000000 +80905 -1.000000 +80906 -1.000000 +80907 -1.000000 +80908 -1.000000 +80909 -1.000000 +80910 -1.000000 +80911 -1.000000 +80912 -1.000000 +80913 -1.000000 +80914 -1.000000 +80915 -1.000000 +80916 -1.000000 +80917 -1.000000 +80918 -1.000000 +80919 -1.000000 +80920 -1.000000 +80921 -1.000000 +80922 -1.000000 +80923 -1.000000 +80924 -1.000000 +80925 -1.000000 +80926 -1.000000 +80927 -1.000000 +80928 -1.000000 +80929 -1.000000 +80930 -1.000000 +80931 -1.000000 +80932 -1.000000 +80933 -1.000000 +80934 -1.000000 +80935 -1.000000 +80936 -1.000000 +80937 -1.000000 +80938 -1.000000 +80939 -1.000000 +80940 -1.000000 +80941 -1.000000 +80942 -1.000000 +80943 -1.000000 +80944 -1.000000 +80945 -1.000000 +80946 -1.000000 +80948 -1.000000 +80949 -1.000000 +80950 -1.000000 +80951 -1.000000 +80952 -1.000000 +80953 -1.000000 +80954 -1.000000 +80955 -1.000000 +80956 -1.000000 +80957 -1.000000 +80958 -1.000000 +80959 -1.000000 +80960 -1.000000 +80961 -1.000000 +80963 -1.000000 +80964 -1.000000 +80965 -1.000000 +80966 -1.000000 +80967 -1.000000 +80968 -1.000000 +80969 -1.000000 +80970 -1.000000 +80971 -1.000000 +80973 -1.000000 +80975 -1.000000 +80977 -1.000000 +80978 -1.000000 +80979 -1.000000 +80980 -1.000000 +80981 -1.000000 +80982 -1.000000 +80983 -1.000000 +80984 -1.000000 +80985 -1.000000 +80986 -1.000000 +80987 -1.000000 +80988 -1.000000 +80989 -1.000000 +80990 -1.000000 +80991 -1.000000 +80992 -1.000000 +80993 -1.000000 +80994 -1.000000 +80995 -1.000000 +80996 -1.000000 +80998 -1.000000 +80999 -1.000000 +81000 -1.000000 +81001 -1.000000 +81002 -1.000000 +81003 -1.000000 +81004 -1.000000 +81005 -1.000000 +81006 -1.000000 +81007 -1.000000 +81009 -1.000000 +81010 -1.000000 +81011 -1.000000 +81012 -1.000000 +81013 -1.000000 +81014 -1.000000 +81015 -1.000000 +81016 -1.000000 +81018 -1.000000 +81019 -1.000000 +81020 -1.000000 +81021 -1.000000 +81022 -1.000000 +81023 -1.000000 +81025 -1.000000 +81026 -1.000000 +81027 -1.000000 +81028 -1.000000 +81029 -1.000000 +81030 -1.000000 +81031 -1.000000 +81032 -1.000000 +81033 -1.000000 +81034 -1.000000 +81035 -1.000000 +81036 -1.000000 +81037 -1.000000 +81038 -1.000000 +81039 -1.000000 +81040 -1.000000 +81041 -1.000000 +81042 -1.000000 +81043 -1.000000 +81044 -1.000000 +81045 -1.000000 +81046 -1.000000 +81047 -1.000000 +81048 -1.000000 +81049 -1.000000 +81050 -1.000000 +81051 -1.000000 +81052 -1.000000 +81053 -1.000000 +81054 -1.000000 +81055 -1.000000 +81056 -1.000000 +81057 -1.000000 +81058 -1.000000 +81059 -1.000000 +81060 -1.000000 +81061 -1.000000 +81062 -1.000000 +81063 -1.000000 +81064 -1.000000 +81065 -1.000000 +81066 -1.000000 +81067 -1.000000 +81068 -1.000000 +81069 -1.000000 +81070 -1.000000 +81071 -1.000000 +81072 -1.000000 +81073 -1.000000 +81074 -1.000000 +81075 -1.000000 +81076 -1.000000 +81077 -1.000000 +81078 -1.000000 +81079 -1.000000 +81080 -1.000000 +81081 -1.000000 +81082 -1.000000 +81083 -1.000000 +81084 -1.000000 +81085 -1.000000 +81086 -1.000000 +81087 -1.000000 +81088 -1.000000 +81089 -1.000000 +81090 -1.000000 +81091 -1.000000 +81092 -1.000000 +81093 -1.000000 +81094 -1.000000 +81095 -1.000000 +81096 -1.000000 +81097 -1.000000 +81098 -1.000000 +81099 -1.000000 +81100 -1.000000 +81101 -1.000000 +81102 -1.000000 +81103 -1.000000 +81104 -1.000000 +81105 -1.000000 +81106 -1.000000 +81107 -1.000000 +81108 -1.000000 +81109 -1.000000 +81110 -1.000000 +81111 -1.000000 +81112 -1.000000 +81113 -1.000000 +81114 -1.000000 +81115 -1.000000 +81116 -1.000000 +81117 -1.000000 +81118 -1.000000 +81119 -1.000000 +81120 -1.000000 +81121 -1.000000 +81122 -1.000000 +81123 -1.000000 +81124 -1.000000 +81125 -1.000000 +81126 -1.000000 +81127 -1.000000 +81128 -1.000000 +81129 -1.000000 +81130 -1.000000 +81131 -1.000000 +81132 -1.000000 +81133 -1.000000 +81134 -1.000000 +81135 -1.000000 +81138 -1.000000 +81139 -1.000000 +81140 -1.000000 +81141 -1.000000 +81143 -1.000000 +81146 -1.000000 +81147 -1.000000 +81148 -1.000000 +81149 -1.000000 +81150 -1.000000 +81151 -1.000000 +81152 -1.000000 +81153 -1.000000 +81154 -1.000000 +81155 -1.000000 +81156 -1.000000 +81157 -1.000000 +81158 -1.000000 +81160 -1.000000 +81164 -1.000000 +81165 -1.000000 +81166 -1.000000 +81167 -1.000000 +81168 -1.000000 +81169 -1.000000 +81170 -1.000000 +81171 -1.000000 +81172 -1.000000 +81173 -1.000000 +81174 -1.000000 +81175 -1.000000 +81176 -1.000000 +81177 -1.000000 +81178 -1.000000 +81179 -1.000000 +81180 -1.000000 +81181 -1.000000 +81182 -1.000000 +81183 -1.000000 +81184 -1.000000 +81185 -1.000000 +81189 -1.000000 +81190 -1.000000 +81192 -1.000000 +81194 -1.000000 +81195 -1.000000 +81196 -1.000000 +81198 -1.000000 +81199 -1.000000 +81201 -1.000000 +81202 -1.000000 +81204 -1.000000 +81205 -1.000000 +81206 -1.000000 +81207 -1.000000 +81208 -1.000000 +81209 -1.000000 +81210 -1.000000 +81211 -1.000000 +81212 -1.000000 +81213 -1.000000 +81214 -1.000000 +81215 -1.000000 +81216 -1.000000 +81217 -1.000000 +81218 -1.000000 +81219 -1.000000 +81220 -1.000000 +81221 -1.000000 +81222 -1.000000 +81223 -1.000000 +81224 -1.000000 +81225 -1.000000 +81226 -1.000000 +81227 -1.000000 +81229 -1.000000 +81230 -1.000000 +81231 -1.000000 +81232 -1.000000 +81233 -1.000000 +81234 -1.000000 +81235 -1.000000 +81236 -1.000000 +81237 -1.000000 +81238 -1.000000 +81239 -1.000000 +81240 -1.000000 +81241 -1.000000 +81242 -1.000000 +81243 -1.000000 +81244 -1.000000 +81245 -1.000000 +81246 -1.000000 +81247 -1.000000 +81248 -1.000000 +81249 -1.000000 +81250 -1.000000 +81251 -1.000000 +81252 -1.000000 +81253 -1.000000 +81254 -1.000000 +81255 -1.000000 +81256 -1.000000 +81257 -1.000000 +81258 -1.000000 +81259 -1.000000 +81260 -1.000000 +81261 -1.000000 +81262 -1.000000 +81263 -1.000000 +81264 -1.000000 +81265 -1.000000 +81266 -1.000000 +81267 -1.000000 +81268 -1.000000 +81269 -1.000000 +81270 -1.000000 +81271 -1.000000 +81272 -1.000000 +81273 -1.000000 +81274 -1.000000 +81275 -1.000000 +81276 -1.000000 +81277 -1.000000 +81278 -1.000000 +81279 -1.000000 +81280 -1.000000 +81281 -1.000000 +81282 -1.000000 +81283 -1.000000 +81284 -1.000000 +81285 -1.000000 +81286 -1.000000 +81287 -1.000000 +81288 -1.000000 +81289 -1.000000 +81290 -1.000000 +81291 -1.000000 +81292 -1.000000 +81293 -1.000000 +81294 -1.000000 +81295 -1.000000 +81296 -1.000000 +81297 -1.000000 +81298 -1.000000 +81299 -1.000000 +81300 -1.000000 +81301 -1.000000 +81302 -1.000000 +81303 -1.000000 +81304 -1.000000 +81305 -1.000000 +81306 -1.000000 +81307 -1.000000 +81309 -1.000000 +81310 -1.000000 +81311 -1.000000 +81314 -1.000000 +81315 -1.000000 +81316 -1.000000 +81317 -1.000000 +81318 -1.000000 +81319 -1.000000 +81320 -1.000000 +81321 -1.000000 +81322 -1.000000 +81323 -1.000000 +81324 -1.000000 +81325 -1.000000 +81326 -1.000000 +81327 -1.000000 +81328 -1.000000 +81329 -1.000000 +81330 -1.000000 +81331 -1.000000 +81332 -1.000000 +81333 -1.000000 +81334 -1.000000 +81335 -1.000000 +81336 -1.000000 +81337 -1.000000 +81338 -1.000000 +81339 -1.000000 +81340 -1.000000 +81341 -1.000000 +81342 -1.000000 +81343 -1.000000 +81344 -1.000000 +81345 -1.000000 +81346 -1.000000 +81347 -1.000000 +81348 -1.000000 +81349 -1.000000 +81350 -1.000000 +81351 -1.000000 +81352 -1.000000 +81353 -1.000000 +81354 -1.000000 +81355 -1.000000 +81356 -1.000000 +81357 -1.000000 +81358 -1.000000 +81359 -1.000000 +81360 -1.000000 +81361 -1.000000 +81362 -1.000000 +81363 -1.000000 +81364 -1.000000 +81365 -1.000000 +81366 -1.000000 +81367 -1.000000 +81368 -1.000000 +81369 -1.000000 +81370 -1.000000 +81371 -1.000000 +81372 -1.000000 +81373 -1.000000 +81374 -1.000000 +81375 -1.000000 +81376 -1.000000 +81377 -1.000000 +81378 -1.000000 +81379 -1.000000 +81380 -1.000000 +81381 -1.000000 +81382 -1.000000 +81383 -1.000000 +81384 -1.000000 +81385 -1.000000 +81386 -1.000000 +81387 -1.000000 +81388 -1.000000 +81389 -1.000000 +81390 -1.000000 +81391 -1.000000 +81392 -1.000000 +81393 -1.000000 +81394 -1.000000 +81395 -1.000000 +81396 -1.000000 +81397 -1.000000 +81398 -1.000000 +81399 -1.000000 +81400 -1.000000 +81401 -1.000000 +81402 -1.000000 +81403 -1.000000 +81404 -1.000000 +81405 -1.000000 +81406 -1.000000 +81407 -1.000000 +81408 -1.000000 +81409 -1.000000 +81410 -1.000000 +81411 -1.000000 +81412 -1.000000 +81413 -1.000000 +81414 -1.000000 +81415 -1.000000 +81416 -1.000000 +81417 -1.000000 +81418 -1.000000 +81419 -1.000000 +81420 -1.000000 +81421 -1.000000 +81423 -1.000000 +81424 -1.000000 +81425 -1.000000 +81426 -1.000000 +81427 -1.000000 +81428 -1.000000 +81429 -1.000000 +81430 -1.000000 +81431 -1.000000 +81432 -1.000000 +81433 -1.000000 +81434 -1.000000 +81435 -1.000000 +81436 -1.000000 +81437 -1.000000 +81439 -1.000000 +81440 -1.000000 +81441 -1.000000 +81442 -1.000000 +81443 -1.000000 +81444 -1.000000 +81445 -1.000000 +81446 -1.000000 +81447 -1.000000 +81448 -1.000000 +81449 -1.000000 +81450 -1.000000 +81451 -1.000000 +81452 -1.000000 +81453 -1.000000 +81454 -1.000000 +81455 -1.000000 +81456 -1.000000 +81457 -1.000000 +81458 -1.000000 +81459 -1.000000 +81460 -1.000000 +81461 -1.000000 +81462 -1.000000 +81463 -1.000000 +81464 -1.000000 +81465 -1.000000 +81466 -1.000000 +81467 -1.000000 +81470 -1.000000 +81471 -1.000000 +81472 -1.000000 +81473 -1.000000 +81474 -1.000000 +81475 -1.000000 +81476 -1.000000 +81477 -1.000000 +81478 -1.000000 +81479 -1.000000 +81480 -1.000000 +81481 -1.000000 +81482 -1.000000 +81483 -1.000000 +81484 -1.000000 +81485 -1.000000 +81486 -1.000000 +81487 -1.000000 +81488 -1.000000 +81489 -1.000000 +81490 -1.000000 +81492 -1.000000 +81493 -1.000000 +81494 -1.000000 +81495 -1.000000 +81496 -1.000000 +81497 -1.000000 +81498 -1.000000 +81499 -1.000000 +81500 -1.000000 +81501 -1.000000 +81503 -1.000000 +81504 -1.000000 +81505 -1.000000 +81506 -1.000000 +81507 -1.000000 +81508 -1.000000 +81509 -1.000000 +81510 -1.000000 +81511 -1.000000 +81512 -1.000000 +81513 -1.000000 +81514 -1.000000 +81515 -1.000000 +81518 -1.000000 +81519 -1.000000 +81523 -1.000000 +81524 -1.000000 +81525 -1.000000 +81527 -1.000000 +81528 -1.000000 +81532 -1.000000 +81533 -1.000000 +81534 -1.000000 +81535 -1.000000 +81536 -1.000000 +81537 -1.000000 +81541 -1.000000 +81546 -1.000000 +81551 -1.000000 +81552 -1.000000 +81553 -1.000000 +81560 -1.000000 +81562 -1.000000 +81563 -1.000000 +81564 -1.000000 +81575 -1.000000 +81576 -1.000000 +81577 -1.000000 +81587 -1.000000 +81588 -1.000000 +81589 -1.000000 +81590 -1.000000 +81591 -1.000000 +81592 -1.000000 +81593 -1.000000 +81594 -1.000000 +81595 -1.000000 +81596 -1.000000 +81597 -1.000000 +81598 -1.000000 +81599 -1.000000 +81600 -1.000000 +81601 -1.000000 +81602 -1.000000 +81603 -1.000000 +81604 -1.000000 +81605 -1.000000 +81606 -1.000000 +81607 -1.000000 +81608 -1.000000 +81609 -1.000000 +81611 -1.000000 +81613 -1.000000 +81614 -1.000000 +81615 -1.000000 +81616 -1.000000 +81617 -1.000000 +81618 -1.000000 +81619 -1.000000 +81620 -1.000000 +81621 -1.000000 +81622 -1.000000 +81623 -1.000000 +81624 -1.000000 +81625 -1.000000 +81626 -1.000000 +81627 -1.000000 +81629 -1.000000 +81630 -1.000000 +81631 -1.000000 +81632 -1.000000 +81633 -1.000000 +81634 -1.000000 +81635 -1.000000 +81636 -1.000000 +81639 -1.000000 +81641 -1.000000 +81642 -1.000000 +81644 -1.000000 +81645 -1.000000 +81646 -1.000000 +81647 -1.000000 +81648 -1.000000 +81649 -1.000000 +81650 -1.000000 +81651 -1.000000 +81652 -1.000000 +81653 -1.000000 +81654 -1.000000 +81655 -1.000000 +81656 -1.000000 +81657 -1.000000 +81658 -1.000000 +81659 -1.000000 +81660 -1.000000 +81661 -1.000000 +81662 -1.000000 +81663 -1.000000 +81664 -1.000000 +81665 -1.000000 +81666 -1.000000 +81668 -1.000000 +81669 -1.000000 +81670 -1.000000 +81671 -1.000000 +81672 -1.000000 +81673 -1.000000 +81674 -1.000000 +81675 -1.000000 +81676 -1.000000 +81677 -1.000000 +81678 -1.000000 +81679 -1.000000 +81680 -1.000000 +81681 -1.000000 +81682 -1.000000 +81683 -1.000000 +81684 -1.000000 +81685 -1.000000 +81686 -1.000000 +81687 -1.000000 +81688 -1.000000 +81689 -1.000000 +81690 -1.000000 +81691 -1.000000 +81692 -1.000000 +81693 -1.000000 +81694 -1.000000 +81695 -1.000000 +81696 -1.000000 +81697 -1.000000 +81698 -1.000000 +81699 -1.000000 +81700 -1.000000 +81701 -1.000000 +81702 -1.000000 +81703 -1.000000 +81704 -1.000000 +81705 -1.000000 +81706 -1.000000 +81707 -1.000000 +81708 -1.000000 +81709 -1.000000 +81710 -1.000000 +81711 -1.000000 +81712 -1.000000 +81713 -1.000000 +81715 -1.000000 +81716 -1.000000 +81717 -1.000000 +81718 -1.000000 +81719 -1.000000 +81720 -1.000000 +81721 -1.000000 +81722 -1.000000 +81723 -1.000000 +81724 -1.000000 +81725 -1.000000 +81726 -1.000000 +81728 -1.000000 +81729 -1.000000 +81730 -1.000000 +81732 -1.000000 +81733 -1.000000 +81734 -1.000000 +81735 -1.000000 +81736 -1.000000 +81737 -1.000000 +81738 -1.000000 +81739 -1.000000 +81740 -1.000000 +81741 -1.000000 +81742 -1.000000 +81743 -1.000000 +81744 -1.000000 +81745 -1.000000 +81746 -1.000000 +81747 -1.000000 +81748 -1.000000 +81750 -1.000000 +81751 -1.000000 +81752 -1.000000 +81753 -1.000000 +81754 -1.000000 +81755 -1.000000 +81756 -1.000000 +81757 -1.000000 +81758 -1.000000 +81759 -1.000000 +81760 -1.000000 +81761 -1.000000 +81762 -1.000000 +81763 -1.000000 +81764 -1.000000 +81767 -1.000000 +81768 -1.000000 +81769 -1.000000 +81770 -1.000000 +81771 -1.000000 +81772 -1.000000 +81773 -1.000000 +81774 -1.000000 +81775 -1.000000 +81776 -1.000000 +81777 -1.000000 +81778 -1.000000 +81779 -1.000000 +81780 -1.000000 +81783 -1.000000 +81784 -1.000000 +81785 -1.000000 +81786 -1.000000 +81787 -1.000000 +81790 -1.000000 +81791 -1.000000 +81792 -1.000000 +81793 -1.000000 +81794 -1.000000 +81795 -1.000000 +81796 -1.000000 +81797 -1.000000 +81798 -1.000000 +81800 -1.000000 +81802 -1.000000 +81805 -1.000000 +81806 -1.000000 +81807 -1.000000 +81808 -1.000000 +81809 -1.000000 +81810 -1.000000 +81811 -1.000000 +81812 -1.000000 +81813 -1.000000 +81814 -1.000000 +81815 -1.000000 +81816 -1.000000 +81817 -1.000000 +81818 -1.000000 +81822 -1.000000 +81825 -1.000000 +81828 -1.000000 +81829 -1.000000 +81830 -1.000000 +81831 -1.000000 +81832 -1.000000 +81835 -1.000000 +81836 -1.000000 +81837 -1.000000 +81838 -1.000000 +81839 -1.000000 +81840 -1.000000 +81841 -1.000000 +81842 -1.000000 +81843 -1.000000 +81844 -1.000000 +81845 -1.000000 +81846 -1.000000 +81847 -1.000000 +81848 -1.000000 +81849 -1.000000 +81850 -1.000000 +81851 -1.000000 +81852 -1.000000 +81853 -1.000000 +81854 -1.000000 +81855 -1.000000 +81856 -1.000000 +81857 -1.000000 +81858 -1.000000 +81859 -1.000000 +81860 -1.000000 +81861 -1.000000 +81862 -1.000000 +81863 -1.000000 +81864 -1.000000 +81865 -1.000000 +81866 -1.000000 +81868 -1.000000 +81869 -1.000000 +81870 -1.000000 +81871 -1.000000 +81872 -1.000000 +81873 -1.000000 +81874 -1.000000 +81875 -1.000000 +81876 -1.000000 +81877 -1.000000 +81878 -1.000000 +81879 -1.000000 +81880 -1.000000 +81881 -1.000000 +81882 -1.000000 +81883 -1.000000 +81884 -1.000000 +81885 -1.000000 +81886 -1.000000 +81887 -1.000000 +81888 -1.000000 +81889 -1.000000 +81890 -1.000000 +81891 -1.000000 +81892 -1.000000 +81893 -1.000000 +81894 -1.000000 +81895 -1.000000 +81896 -1.000000 +81897 -1.000000 +81898 -1.000000 +81899 -1.000000 +81900 -1.000000 +81901 -1.000000 +81902 -1.000000 +81903 -1.000000 +81904 -1.000000 +81905 -1.000000 +81906 -1.000000 +81907 -1.000000 +81908 -1.000000 +81909 -1.000000 +81910 -1.000000 +81911 -1.000000 +81912 -1.000000 +81913 -1.000000 +81915 -1.000000 +81917 -1.000000 +81918 -1.000000 +81919 -1.000000 +81920 -1.000000 +81931 -1.000000 +81935 -1.000000 +81936 -1.000000 +81937 -1.000000 +81938 -1.000000 +81939 -1.000000 +81940 -1.000000 +81941 -1.000000 +81942 -1.000000 +81943 -1.000000 +81944 -1.000000 +81945 -1.000000 +81946 -1.000000 +81947 -1.000000 +81948 -1.000000 +81949 -1.000000 +81950 -1.000000 +81951 -1.000000 +81952 -1.000000 +81954 -1.000000 +81955 -1.000000 +81956 -1.000000 +81957 -1.000000 +81958 -1.000000 +81959 -1.000000 +81960 -1.000000 +81961 -1.000000 +81962 -1.000000 +81963 -1.000000 +81964 -1.000000 +81965 -1.000000 +81966 -1.000000 +81967 -1.000000 +81968 -1.000000 +81969 -1.000000 +81970 -1.000000 +81971 -1.000000 +81972 -1.000000 +81973 -1.000000 +81974 -1.000000 +81975 -1.000000 +81976 -1.000000 +81977 -1.000000 +81979 -1.000000 +81980 -1.000000 +81981 -1.000000 +81982 -1.000000 +81984 -1.000000 +81986 -1.000000 +81987 -1.000000 +81988 -1.000000 +81989 -1.000000 +81991 -1.000000 +81993 -1.000000 +81994 -1.000000 +81995 -1.000000 +81996 -1.000000 +81997 -1.000000 +81998 -1.000000 +81999 -1.000000 +82000 -1.000000 +82002 -1.000000 +82003 -1.000000 +82006 -1.000000 +82007 -1.000000 +82008 -1.000000 +82009 -1.000000 +82010 -1.000000 +82011 -1.000000 +82014 -1.000000 +82017 -1.000000 +82018 -1.000000 +82019 -1.000000 +82020 -1.000000 +82021 -1.000000 +82022 -1.000000 +82026 -1.000000 +82027 -1.000000 +82028 -1.000000 +82029 -1.000000 +82030 -1.000000 +82031 -1.000000 +82032 -1.000000 +82033 -1.000000 +82035 -1.000000 +82036 -1.000000 +82037 -1.000000 +82038 -1.000000 +82039 -1.000000 +82040 -1.000000 +82041 -1.000000 +82042 -1.000000 +82043 -1.000000 +82044 -1.000000 +82045 -1.000000 +82046 -1.000000 +82047 -1.000000 +82048 -1.000000 +82049 -1.000000 +82050 -1.000000 +82051 -1.000000 +82052 -1.000000 +82053 -1.000000 +82054 -1.000000 +82055 -1.000000 +82056 -1.000000 +82057 -1.000000 +82058 -1.000000 +82059 -1.000000 +82060 -1.000000 +82061 -1.000000 +82062 -1.000000 +82063 -1.000000 +82064 -1.000000 +82065 -1.000000 +82066 -1.000000 +82067 -1.000000 +82068 -1.000000 +82069 -1.000000 +82070 -1.000000 +82071 -1.000000 +82072 -1.000000 +82073 -1.000000 +82074 -1.000000 +82075 -1.000000 +82076 -1.000000 +82077 -1.000000 +82078 -1.000000 +82079 -1.000000 +82080 -1.000000 +82081 -1.000000 +82082 -1.000000 +82083 -1.000000 +82084 -1.000000 +82085 -1.000000 +82086 -1.000000 +82087 -1.000000 +82088 -1.000000 +82089 -1.000000 +82090 -1.000000 +82091 -1.000000 +82092 -1.000000 +82093 -1.000000 +82094 -1.000000 +82095 -1.000000 +82096 -1.000000 +82097 -1.000000 +82098 -1.000000 +82099 -1.000000 +82100 -1.000000 +82101 -1.000000 +82102 -1.000000 +82103 -1.000000 +82104 -1.000000 +82105 -1.000000 +82106 -1.000000 +82107 -1.000000 +82108 -1.000000 +82109 -1.000000 +82110 -1.000000 +82111 -1.000000 +82112 -1.000000 +82113 -1.000000 +82114 -1.000000 +82115 -1.000000 +82116 -1.000000 +82117 -1.000000 +82118 -1.000000 +82119 -1.000000 +82120 -1.000000 +82121 -1.000000 +82122 -1.000000 +82123 -1.000000 +82124 -1.000000 +82125 -1.000000 +82126 -1.000000 +82127 -1.000000 +82128 -1.000000 +82129 -1.000000 +82130 -1.000000 +82131 -1.000000 +82132 -1.000000 +82133 -1.000000 +82134 -1.000000 +82135 -1.000000 +82136 -1.000000 +82137 -1.000000 +82138 -1.000000 +82139 -1.000000 +82141 -1.000000 +82142 -1.000000 +82143 -1.000000 +82144 -1.000000 +82145 -1.000000 +82146 -1.000000 +82147 -1.000000 +82148 -1.000000 +82149 -1.000000 +82150 -1.000000 +82151 -1.000000 +82152 -1.000000 +82153 -1.000000 +82154 -1.000000 +82155 -1.000000 +82156 -1.000000 +82158 -1.000000 +82159 -1.000000 +82160 -1.000000 +82161 -1.000000 +82162 -1.000000 +82163 -1.000000 +82164 -1.000000 +82165 -1.000000 +82166 -1.000000 +82167 -1.000000 +82168 -1.000000 +82169 -1.000000 +82170 -1.000000 +82171 -1.000000 +82172 -1.000000 +82173 -1.000000 +82174 -1.000000 +82175 -1.000000 +82176 -1.000000 +82177 -1.000000 +82178 -1.000000 +82179 -1.000000 +82180 -1.000000 +82181 -1.000000 +82182 -1.000000 +82183 -1.000000 +82184 -1.000000 +82185 -1.000000 +82186 -1.000000 +82187 -1.000000 +82188 -1.000000 +82189 -1.000000 +82190 -1.000000 +82191 -1.000000 +82192 -1.000000 +82193 -1.000000 +82194 -1.000000 +82195 -1.000000 +82197 -1.000000 +82198 -1.000000 +82199 -1.000000 +82200 -1.000000 +82201 -1.000000 +82202 -1.000000 +82203 -1.000000 +82204 -1.000000 +82205 -1.000000 +82206 -1.000000 +82207 -1.000000 +82208 -1.000000 +82209 -1.000000 +82210 -1.000000 +82211 -1.000000 +82212 -1.000000 +82213 -1.000000 +82214 -1.000000 +82215 -1.000000 +82216 -1.000000 +82217 -1.000000 +82219 -1.000000 +82220 -1.000000 +82221 -1.000000 +82222 -1.000000 +82223 -1.000000 +82224 -1.000000 +82225 -1.000000 +82226 -1.000000 +82227 -1.000000 +82228 -1.000000 +82229 -1.000000 +82230 -1.000000 +82231 -1.000000 +82232 -1.000000 +82233 -1.000000 +82234 -1.000000 +82235 -1.000000 +82236 -1.000000 +82237 -1.000000 +82238 -1.000000 +82239 -1.000000 +82240 -1.000000 +82241 -1.000000 +82242 -1.000000 +82243 -1.000000 +82244 -1.000000 +82245 -1.000000 +82247 -1.000000 +82251 -1.000000 +82252 -1.000000 +82253 -1.000000 +82254 -1.000000 +82255 -1.000000 +82256 -1.000000 +82260 -1.000000 +82261 -1.000000 +82262 -1.000000 +82263 -1.000000 +82264 -1.000000 +82265 -1.000000 +82266 -1.000000 +82267 -1.000000 +82268 -1.000000 +82269 -1.000000 +82270 -1.000000 +82271 -1.000000 +82272 -1.000000 +82273 -1.000000 +82274 -1.000000 +82275 -1.000000 +82276 -1.000000 +82277 -1.000000 +82278 -1.000000 +82279 -1.000000 +82280 -1.000000 +82282 -1.000000 +82283 -1.000000 +82284 -1.000000 +82285 -1.000000 +82286 -1.000000 +82287 -1.000000 +82288 -1.000000 +82289 -1.000000 +82290 -1.000000 +82291 -1.000000 +82292 -1.000000 +82293 -1.000000 +82294 -1.000000 +82295 -1.000000 +82297 -1.000000 +82298 -1.000000 +82300 -1.000000 +82301 -1.000000 +82302 -1.000000 +82303 -1.000000 +82305 -1.000000 +82306 -1.000000 +82308 -1.000000 +82309 -1.000000 +82310 -1.000000 +82315 -1.000000 +82316 -1.000000 +82317 -1.000000 +82319 -1.000000 +82321 -1.000000 +82322 -1.000000 +82323 -1.000000 +82324 -1.000000 +82325 -1.000000 +82326 -1.000000 +82327 -1.000000 +82328 -1.000000 +82329 -1.000000 +82330 -1.000000 +82331 -1.000000 +82332 -1.000000 +82335 -1.000000 +82341 -1.000000 +82343 -1.000000 +82344 -1.000000 +82345 -1.000000 +82346 -1.000000 +82347 -1.000000 +82348 -1.000000 +82349 -1.000000 +82350 -1.000000 +82351 -1.000000 +82352 -1.000000 +82353 -1.000000 +82354 -1.000000 +82355 -1.000000 +82356 -1.000000 +82357 -1.000000 +82358 -1.000000 +82359 -1.000000 +82360 -1.000000 +82361 -1.000000 +82362 -1.000000 +82363 -1.000000 +82364 -1.000000 +82365 -1.000000 +82366 -1.000000 +82367 -1.000000 +82368 -1.000000 +82369 -1.000000 +82370 -1.000000 +82371 -1.000000 +82372 -1.000000 +82373 -1.000000 +82374 -1.000000 +82375 -1.000000 +82376 -1.000000 +82377 -1.000000 +82378 -1.000000 +82379 -1.000000 +82380 -1.000000 +82381 -1.000000 +82382 -1.000000 +82383 -1.000000 +82384 -1.000000 +82385 -1.000000 +82386 -1.000000 +82387 -1.000000 +82388 -1.000000 +82389 -1.000000 +82391 -1.000000 +82392 -1.000000 +82394 -1.000000 +82395 -1.000000 +82396 -1.000000 +82397 -1.000000 +82398 -1.000000 +82399 -1.000000 +82400 -1.000000 +82401 -1.000000 +82402 -1.000000 +82403 -1.000000 +82404 -1.000000 +82405 -1.000000 +82406 -1.000000 +82407 -1.000000 +82408 -1.000000 +82410 -1.000000 +82411 -1.000000 +82412 -1.000000 +82413 -1.000000 +82414 -1.000000 +82416 -1.000000 +82417 -1.000000 +82418 -1.000000 +82419 -1.000000 +82420 -1.000000 +82421 -1.000000 +82422 -1.000000 +82423 -1.000000 +82424 -1.000000 +82425 -1.000000 +82426 -1.000000 +82427 -1.000000 +82428 -1.000000 +82429 -1.000000 +82430 -1.000000 +82431 -1.000000 +82432 -1.000000 +82433 -1.000000 +82434 -1.000000 +82435 -1.000000 +82436 -1.000000 +82437 -1.000000 +82438 -1.000000 +82439 -1.000000 +82440 -1.000000 +82441 -1.000000 +82442 -1.000000 +82443 -1.000000 +82444 -1.000000 +82445 -1.000000 +82446 -1.000000 +82447 -1.000000 +82448 -1.000000 +82449 -1.000000 +82450 -1.000000 +82451 -1.000000 +82452 -1.000000 +82453 -1.000000 +82454 -1.000000 +82455 -1.000000 +82456 -1.000000 +82457 -1.000000 +82458 -1.000000 +82459 -1.000000 +82461 -1.000000 +82462 -1.000000 +82463 -1.000000 +82464 -1.000000 +82467 -1.000000 +82468 -1.000000 +82469 -1.000000 +82471 -1.000000 +82472 -1.000000 +82474 -1.000000 +82475 -1.000000 +82476 -1.000000 +82477 -1.000000 +82478 -1.000000 +82479 -1.000000 +82480 -1.000000 +82481 -1.000000 +82482 -1.000000 +82483 -1.000000 +82484 -1.000000 +82485 -1.000000 +82486 -1.000000 +82487 -1.000000 +82488 -1.000000 +82489 -1.000000 +82490 -1.000000 +82491 -1.000000 +82492 -1.000000 +82493 -1.000000 +82494 -1.000000 +82496 -1.000000 +82497 -1.000000 +82498 -1.000000 +82501 -1.000000 +82503 -1.000000 +82505 -1.000000 +82506 -1.000000 +82507 -1.000000 +82508 -1.000000 +82509 -1.000000 +82510 -1.000000 +82511 -1.000000 +82512 -1.000000 +82513 -1.000000 +82514 -1.000000 +82515 -1.000000 +82516 -1.000000 +82517 -1.000000 +82518 -1.000000 +82519 -1.000000 +82520 -1.000000 +82521 -1.000000 +82522 -1.000000 +82523 -1.000000 +82524 -1.000000 +82525 -1.000000 +82526 -1.000000 +82527 -1.000000 +82528 -1.000000 +82529 -1.000000 +82530 -1.000000 +82531 -1.000000 +82532 -1.000000 +82533 -1.000000 +82534 -1.000000 +82535 -1.000000 +82536 -1.000000 +82537 -1.000000 +82538 -1.000000 +82539 -1.000000 +82540 -1.000000 +82541 -1.000000 +82542 -1.000000 +82543 -1.000000 +82544 -1.000000 +82545 -1.000000 +82546 -1.000000 +82547 -1.000000 +82548 -1.000000 +82549 -1.000000 +82550 -1.000000 +82552 -1.000000 +82557 -1.000000 +82558 -1.000000 +82559 -1.000000 +82560 -1.000000 +82561 -1.000000 +82562 -1.000000 +82563 -1.000000 +82564 -1.000000 +82565 -1.000000 +82566 -1.000000 +82567 -1.000000 +82568 -1.000000 +82570 -1.000000 +82571 -1.000000 +82573 -1.000000 +82574 -1.000000 +82575 -1.000000 +82576 -1.000000 +82579 -1.000000 +82580 -1.000000 +82581 -1.000000 +82582 -1.000000 +82583 -1.000000 +82584 -1.000000 +82585 -1.000000 +82586 -1.000000 +82587 -1.000000 +82588 -1.000000 +82591 -1.000000 +82592 -1.000000 +82593 -1.000000 +82594 -1.000000 +82595 -1.000000 +82596 -1.000000 +82597 -1.000000 +82598 -1.000000 +82599 -1.000000 +82600 -1.000000 +82601 -1.000000 +82602 -1.000000 +82603 -1.000000 +82604 -1.000000 +82605 -1.000000 +82606 -1.000000 +82607 -1.000000 +82608 -1.000000 +82609 -1.000000 +82610 -1.000000 +82611 -1.000000 +82612 -1.000000 +82613 -1.000000 +82614 -1.000000 +82615 -1.000000 +82616 -1.000000 +82617 -1.000000 +82618 -1.000000 +82619 -1.000000 +82620 -1.000000 +82621 -1.000000 +82622 -1.000000 +82623 -1.000000 +82624 -1.000000 +82625 -1.000000 +82626 -1.000000 +82627 -1.000000 +82628 -1.000000 +82629 -1.000000 +82630 -1.000000 +82631 -1.000000 +82632 -1.000000 +82633 -1.000000 +82634 -1.000000 +82635 -1.000000 +82636 -1.000000 +82637 -1.000000 +82638 -1.000000 +82639 -1.000000 +82640 -1.000000 +82641 -1.000000 +82642 -1.000000 +82643 -1.000000 +82644 -1.000000 +82645 -1.000000 +82646 -1.000000 +82647 -1.000000 +82648 -1.000000 +82649 -1.000000 +82650 -1.000000 +82651 -1.000000 +82653 -1.000000 +82655 -1.000000 +82656 -1.000000 +82658 -1.000000 +82659 -1.000000 +82660 -1.000000 +82661 -1.000000 +82662 -1.000000 +82663 -1.000000 +82664 -1.000000 +82665 -1.000000 +82666 -1.000000 +82667 -1.000000 +82668 -1.000000 +82670 -1.000000 +82671 -1.000000 +82672 -1.000000 +82675 -1.000000 +82676 -1.000000 +82677 -1.000000 +82678 -1.000000 +82679 -1.000000 +82680 -1.000000 +82681 -1.000000 +82682 -1.000000 +82683 -1.000000 +82684 -1.000000 +82685 -1.000000 +82686 -1.000000 +82688 -1.000000 +82689 -1.000000 +82690 -1.000000 +82691 -1.000000 +82692 -1.000000 +82693 -1.000000 +82694 -1.000000 +82695 -1.000000 +82696 -1.000000 +82697 -1.000000 +82699 -1.000000 +82700 -1.000000 +82701 -1.000000 +82702 -1.000000 +82703 -1.000000 +82704 -1.000000 +82705 -1.000000 +82706 -1.000000 +82707 -1.000000 +82708 -1.000000 +82709 -1.000000 +82710 -1.000000 +82711 -1.000000 +82712 -1.000000 +82713 -1.000000 +82714 -1.000000 +82715 -1.000000 +82716 -1.000000 +82717 -1.000000 +82718 -1.000000 +82719 -1.000000 +82720 -1.000000 +82721 -1.000000 +82722 -1.000000 +82723 -1.000000 +82724 -1.000000 +82725 -1.000000 +82726 -1.000000 +82727 -1.000000 +82728 -1.000000 +82729 -1.000000 +82733 -1.000000 +82734 -1.000000 +82735 -1.000000 +82736 -1.000000 +82737 -1.000000 +82738 -1.000000 +82739 -1.000000 +82740 -1.000000 +82741 -1.000000 +82742 -1.000000 +82743 -1.000000 +82744 -1.000000 +82745 -1.000000 +82746 -1.000000 +82747 -1.000000 +82748 -1.000000 +82749 -1.000000 +82750 -1.000000 +82751 -1.000000 +82755 -1.000000 +82757 -1.000000 +82759 -1.000000 +82760 -1.000000 +82761 -1.000000 +82762 -1.000000 +82763 -1.000000 +82764 -1.000000 +82765 -1.000000 +82766 -1.000000 +82767 -1.000000 +82768 -1.000000 +82769 -1.000000 +82770 -1.000000 +82771 -1.000000 +82772 -1.000000 +82773 -1.000000 +82774 -1.000000 +82776 -1.000000 +82777 -1.000000 +82778 -1.000000 +82779 -1.000000 +82781 -1.000000 +82782 -1.000000 +82783 -1.000000 +82784 -1.000000 +82785 -1.000000 +82786 -1.000000 +82787 -1.000000 +82788 -1.000000 +82789 -1.000000 +82790 -1.000000 +82791 -1.000000 +82792 -1.000000 +82793 -1.000000 +82794 -1.000000 +82795 -1.000000 +82796 -1.000000 +82798 -1.000000 +82799 -1.000000 +82800 -1.000000 +82801 -1.000000 +82802 -1.000000 +82803 -1.000000 +82804 -1.000000 +82805 -1.000000 +82806 -1.000000 +82808 -1.000000 +82809 -1.000000 +82812 -1.000000 +82814 -1.000000 +82815 -1.000000 +82816 -1.000000 +82818 -1.000000 +82823 -1.000000 +82832 -1.000000 +82833 -1.000000 +82834 -1.000000 +82835 -1.000000 +82836 -1.000000 +82837 -1.000000 +82838 -1.000000 +82839 -1.000000 +82840 -1.000000 +82841 -1.000000 +82842 -1.000000 +82843 -1.000000 +82844 -1.000000 +82845 -1.000000 +82846 -1.000000 +82847 -1.000000 +82848 -1.000000 +82849 -1.000000 +82850 -1.000000 +82854 -1.000000 +82855 -1.000000 +82856 -1.000000 +82857 -1.000000 +82859 -1.000000 +82860 -1.000000 +82863 -1.000000 +82864 -1.000000 +82865 -1.000000 +82866 -1.000000 +82868 -1.000000 +82869 -1.000000 +82870 -1.000000 +82871 -1.000000 +82872 -1.000000 +82873 -1.000000 +82875 -1.000000 +82877 -1.000000 +82878 -1.000000 +82879 -1.000000 +82880 -1.000000 +82881 -1.000000 +82882 -1.000000 +82883 -1.000000 +82884 -1.000000 +82885 -1.000000 +82886 -1.000000 +82887 -1.000000 +82888 -1.000000 +82889 -1.000000 +82890 -1.000000 +82891 -1.000000 +82892 -1.000000 +82894 -1.000000 +82895 -1.000000 +82897 -1.000000 +82898 -1.000000 +82899 -1.000000 +82900 -1.000000 +82901 -1.000000 +82902 -1.000000 +82903 -1.000000 +82904 -1.000000 +82906 -1.000000 +82907 -1.000000 +82908 -1.000000 +82909 -1.000000 +82910 -1.000000 +82911 -1.000000 +82912 -1.000000 +82913 -1.000000 +82914 -1.000000 +82915 -1.000000 +82916 -1.000000 +82917 -1.000000 +82918 -1.000000 +82919 -1.000000 +82920 -1.000000 +82921 -1.000000 +82922 -1.000000 +82923 -1.000000 +82924 -1.000000 +82925 -1.000000 +82926 -1.000000 +82928 -1.000000 +82929 -1.000000 +82930 -1.000000 +82931 -1.000000 +82935 -1.000000 +82936 -1.000000 +82944 -1.000000 +82945 -1.000000 +82946 -1.000000 +82947 -1.000000 +82948 -1.000000 +82949 -1.000000 +82950 -1.000000 +82951 -1.000000 +82952 -1.000000 +82953 -1.000000 +82954 -1.000000 +82957 -1.000000 +82958 -1.000000 +82959 -1.000000 +82961 -1.000000 +82964 -1.000000 +82965 -1.000000 +82966 -1.000000 +82967 -1.000000 +82968 -1.000000 +82969 -1.000000 +82970 -1.000000 +82971 -1.000000 +82972 -1.000000 +82973 -1.000000 +82974 -1.000000 +82975 -1.000000 +82977 -1.000000 +82978 -1.000000 +82979 -1.000000 +82982 -1.000000 +82983 -1.000000 +82984 -1.000000 +82985 -1.000000 +82986 -1.000000 +82987 -1.000000 +82988 -1.000000 +82989 -1.000000 +82990 -1.000000 +82991 -1.000000 +82992 -1.000000 +82993 -1.000000 +82994 -1.000000 +82995 -1.000000 +82996 -1.000000 +82997 -1.000000 +82998 -1.000000 +82999 -1.000000 +83000 -1.000000 +83001 -1.000000 +83002 -1.000000 +83003 -1.000000 +83004 -1.000000 +83005 -1.000000 +83006 -1.000000 +83007 -1.000000 +83008 -1.000000 +83009 -1.000000 +83010 -1.000000 +83011 -1.000000 +83012 -1.000000 +83013 -1.000000 +83014 -1.000000 +83015 -1.000000 +83016 -1.000000 +83017 -1.000000 +83018 -1.000000 +83019 -1.000000 +83020 -1.000000 +83021 -1.000000 +83022 -1.000000 +83023 -1.000000 +83024 -1.000000 +83025 -1.000000 +83026 -1.000000 +83027 -1.000000 +83028 -1.000000 +83029 -1.000000 +83030 -1.000000 +83031 -1.000000 +83032 -1.000000 +83033 -1.000000 +83034 -1.000000 +83035 -1.000000 +83036 -1.000000 +83043 -1.000000 +83044 -1.000000 +83046 -1.000000 +83049 -1.000000 +83050 -1.000000 +83052 -1.000000 +83055 -1.000000 +83056 -1.000000 +83057 -1.000000 +83058 -1.000000 +83059 -1.000000 +83060 -1.000000 +83061 -1.000000 +83062 -1.000000 +83063 -1.000000 +83064 -1.000000 +83066 -1.000000 +83067 -1.000000 +83068 -1.000000 +83069 -1.000000 +83070 -1.000000 +83071 -1.000000 +83072 -1.000000 +83073 -1.000000 +83074 -1.000000 +83075 -1.000000 +83076 -1.000000 +83077 -1.000000 +83078 -1.000000 +83079 -1.000000 +83080 -1.000000 +83081 -1.000000 +83082 -1.000000 +83084 -1.000000 +83085 -1.000000 +83086 -1.000000 +83087 -1.000000 +83088 -1.000000 +83089 -1.000000 +83090 -1.000000 +83091 -1.000000 +83092 -1.000000 +83093 -1.000000 +83094 -1.000000 +83095 -1.000000 +83096 -1.000000 +83097 -1.000000 +83098 -1.000000 +83099 -1.000000 +83100 -1.000000 +83101 -1.000000 +83102 -1.000000 +83103 -1.000000 +83104 -1.000000 +83105 -1.000000 +83106 -1.000000 +83107 -1.000000 +83108 -1.000000 +83109 -1.000000 +83110 -1.000000 +83111 -1.000000 +83112 -1.000000 +83113 -1.000000 +83114 -1.000000 +83115 -1.000000 +83116 -1.000000 +83117 -1.000000 +83118 -1.000000 +83119 -1.000000 +83120 -1.000000 +83121 -1.000000 +83122 -1.000000 +83123 -1.000000 +83124 -1.000000 +83125 -1.000000 +83126 -1.000000 +83127 -1.000000 +83128 -1.000000 +83129 -1.000000 +83130 -1.000000 +83131 -1.000000 +83132 -1.000000 +83133 -1.000000 +83134 -1.000000 +83135 -1.000000 +83136 -1.000000 +83138 -1.000000 +83139 -1.000000 +83140 -1.000000 +83142 -1.000000 +83145 -1.000000 +83149 -1.000000 +83150 -1.000000 +83151 -1.000000 +83152 -1.000000 +83155 -1.000000 +83156 -1.000000 +83157 -1.000000 +83158 -1.000000 +83159 -1.000000 +83160 -1.000000 +83161 -1.000000 +83162 -1.000000 +83163 -1.000000 +83164 -1.000000 +83166 -1.000000 +83167 -1.000000 +83170 -1.000000 +83172 -1.000000 +83173 -1.000000 +83174 -1.000000 +83176 -1.000000 +83178 -1.000000 +83181 -1.000000 +83182 -1.000000 +83184 -1.000000 +83189 -1.000000 +83190 -1.000000 +83192 -1.000000 +83193 -1.000000 +83194 -1.000000 +83195 -1.000000 +83196 -1.000000 +83197 -1.000000 +83198 -1.000000 +83199 -1.000000 +83200 -1.000000 +83201 -1.000000 +83202 -1.000000 +83203 -1.000000 +83204 -1.000000 +83205 -1.000000 +83206 -1.000000 +83207 -1.000000 +83208 -1.000000 +83211 -1.000000 +83212 -1.000000 +83213 -1.000000 +83214 -1.000000 +83215 -1.000000 +83217 -1.000000 +83218 -1.000000 +83219 -1.000000 +83220 -1.000000 +83221 -1.000000 +83222 -1.000000 +83223 -1.000000 +83224 -1.000000 +83225 -1.000000 +83226 -1.000000 +83228 -1.000000 +83229 -1.000000 +83234 -1.000000 +83235 -1.000000 +83236 -1.000000 +83237 -1.000000 +83239 -1.000000 +83242 -1.000000 +83243 -1.000000 +83246 -1.000000 +83247 -1.000000 +83248 -1.000000 +83249 -1.000000 +83251 -1.000000 +83252 -1.000000 +83253 -1.000000 +83255 -1.000000 +83262 -1.000000 +83263 -1.000000 +83264 -1.000000 +83265 -1.000000 +83266 -1.000000 +83267 -1.000000 +83268 -1.000000 +83269 -1.000000 +83270 -1.000000 +83271 -1.000000 +83272 -1.000000 +83273 -1.000000 +83274 -1.000000 +83275 -1.000000 +83277 -1.000000 +83278 -1.000000 +83279 -1.000000 +83280 -1.000000 +83281 -1.000000 +83282 -1.000000 +83283 -1.000000 +83284 -1.000000 +83285 -1.000000 +83286 -1.000000 +83287 -1.000000 +83288 -1.000000 +83289 -1.000000 +83290 -1.000000 +83291 -1.000000 +83292 -1.000000 +83293 -1.000000 +83294 -1.000000 +83296 -1.000000 +83297 -1.000000 +83298 -1.000000 +83299 -1.000000 +83300 -1.000000 +83301 -1.000000 +83302 -1.000000 +83303 -1.000000 +83304 -1.000000 +83306 -1.000000 +83307 -1.000000 +83309 -1.000000 +83310 -1.000000 +83311 -1.000000 +83315 -1.000000 +83316 -1.000000 +83318 -1.000000 +83319 -1.000000 +83320 -1.000000 +83321 -1.000000 +83322 -1.000000 +83323 -1.000000 +83325 -1.000000 +83327 -1.000000 +83328 -1.000000 +83329 -1.000000 +83332 -1.000000 +83334 -1.000000 +83336 -1.000000 +83339 -1.000000 +83342 -1.000000 +83343 -1.000000 +83344 -1.000000 +83345 -1.000000 +83347 -1.000000 +83348 -1.000000 +83349 -1.000000 +83350 -1.000000 +83351 -1.000000 +83353 -1.000000 +83354 -1.000000 +83355 -1.000000 +83357 -1.000000 +83358 -1.000000 +83361 -1.000000 +83364 -1.000000 +83365 -1.000000 +83368 -1.000000 +83369 -1.000000 +83371 -1.000000 +83372 -1.000000 +83373 -1.000000 +83374 -1.000000 +83375 -1.000000 +83376 -1.000000 +83377 -1.000000 +83378 -1.000000 +83379 -1.000000 +83380 -1.000000 +83382 -1.000000 +83383 -1.000000 +83385 -1.000000 +83386 -1.000000 +83387 -1.000000 +83388 -1.000000 +83389 -1.000000 +83391 -1.000000 +83392 -1.000000 +83393 -1.000000 +83394 -1.000000 +83395 -1.000000 +83396 -1.000000 +83397 -1.000000 +83398 -1.000000 +83399 -1.000000 +83401 -1.000000 +83402 -1.000000 +83405 -1.000000 +83406 -1.000000 +83407 -1.000000 +83408 -1.000000 +83409 -1.000000 +83410 -1.000000 +83412 -1.000000 +83414 -1.000000 +83415 -1.000000 +83416 -1.000000 +83418 -1.000000 +83419 -1.000000 +83420 -1.000000 +83421 -1.000000 +83422 -1.000000 +83424 -1.000000 +83426 -1.000000 +83427 -1.000000 +83428 -1.000000 +83429 -1.000000 +83430 -1.000000 +83431 -1.000000 +83437 -1.000000 +83439 -1.000000 +83440 -1.000000 +83441 -1.000000 +83442 -1.000000 +83443 -1.000000 +83445 -1.000000 +83446 -1.000000 +83447 -1.000000 +83448 -1.000000 +83449 -1.000000 +83450 -1.000000 +83451 -1.000000 +83452 -1.000000 +83454 -1.000000 +83455 -1.000000 +83456 -1.000000 +83457 -1.000000 +83458 -1.000000 +83459 -1.000000 +83460 -1.000000 +83461 -1.000000 +83462 -1.000000 +83463 -1.000000 +83464 -1.000000 +83465 -1.000000 +83466 -1.000000 +83467 -1.000000 +83468 -1.000000 +83469 -1.000000 +83470 -1.000000 +83471 -1.000000 +83472 -1.000000 +83473 -1.000000 +83474 -1.000000 +83475 -1.000000 +83476 -1.000000 +83477 -1.000000 +83478 -1.000000 +83479 -1.000000 +83481 -1.000000 +83483 -1.000000 +83484 -1.000000 +83485 -1.000000 +83486 -1.000000 +83487 -1.000000 +83488 -1.000000 +83489 -1.000000 +83490 -1.000000 +83491 -1.000000 +83492 -1.000000 +83493 -1.000000 +83494 -1.000000 +83495 -1.000000 +83496 -1.000000 +83498 -1.000000 +83499 -1.000000 +83500 -1.000000 +83501 -1.000000 +83502 -1.000000 +83503 -1.000000 +83505 -1.000000 +83506 -1.000000 +83507 -1.000000 +83508 -1.000000 +83509 -1.000000 +83510 -1.000000 +83511 -1.000000 +83512 -1.000000 +83513 -1.000000 +83516 -1.000000 +83517 -1.000000 +83520 -1.000000 +83523 -1.000000 +83524 -1.000000 +83525 -1.000000 +83527 -1.000000 +83528 -1.000000 +83529 -1.000000 +83530 -1.000000 +83532 -1.000000 +83534 -1.000000 +83535 -1.000000 +83536 -1.000000 +83537 -1.000000 +83538 -1.000000 +83539 -1.000000 +83540 -1.000000 +83541 -1.000000 +83542 -1.000000 +83543 -1.000000 +83546 -1.000000 +83547 -1.000000 +83548 -1.000000 +83549 -1.000000 +83550 -1.000000 +83551 -1.000000 +83552 -1.000000 +83553 -1.000000 +83554 -1.000000 +83555 -1.000000 +83556 -1.000000 +83558 -1.000000 +83559 -1.000000 +83560 -1.000000 +83561 -1.000000 +83562 -1.000000 +83563 -1.000000 +83565 -1.000000 +83569 -1.000000 +83573 -1.000000 +83574 -1.000000 +83576 -1.000000 +83577 -1.000000 +83578 -1.000000 +83579 -1.000000 +83580 -1.000000 +83581 -1.000000 +83582 -1.000000 +83583 -1.000000 +83584 -1.000000 +83585 -1.000000 +83586 -1.000000 +83587 -1.000000 +83588 -1.000000 +83589 -1.000000 +83590 -1.000000 +83591 -1.000000 +83592 -1.000000 +83593 -1.000000 +83594 -1.000000 +83595 -1.000000 +83596 -1.000000 +83597 -1.000000 +83599 -1.000000 +83600 -1.000000 +83602 -1.000000 +83603 -1.000000 +83604 -1.000000 +83605 -1.000000 +83606 -1.000000 +83607 -1.000000 +83608 -1.000000 +83609 -1.000000 +83610 -1.000000 +83612 -1.000000 +83613 -1.000000 +83614 -1.000000 +83616 -1.000000 +83617 -1.000000 +83621 -1.000000 +83622 -1.000000 +83623 -1.000000 +83624 -1.000000 +83625 -1.000000 +83626 -1.000000 +83627 -1.000000 +83628 -1.000000 +83629 -1.000000 +83630 -1.000000 +83631 -1.000000 +83632 -1.000000 +83634 -1.000000 +83636 -1.000000 +83637 -1.000000 +83638 -1.000000 +83639 -1.000000 +83640 -1.000000 +83641 -1.000000 +83642 -1.000000 +83643 -1.000000 +83644 -1.000000 +83645 -1.000000 +83646 -1.000000 +83647 -1.000000 +83648 -1.000000 +83649 -1.000000 +83650 -1.000000 +83651 -1.000000 +83652 -1.000000 +83653 -1.000000 +83654 -1.000000 +83655 -1.000000 +83656 -1.000000 +83657 -1.000000 +83658 -1.000000 +83659 -1.000000 +83660 -1.000000 +83661 -1.000000 +83662 -1.000000 +83663 -1.000000 +83664 -1.000000 +83665 -1.000000 +83666 -1.000000 +83667 -1.000000 +83668 -1.000000 +83669 -1.000000 +83670 -1.000000 +83671 -1.000000 +83672 -1.000000 +83673 -1.000000 +83674 -1.000000 +83675 -1.000000 +83676 -1.000000 +83677 -1.000000 +83678 -1.000000 +83679 -1.000000 +83680 -1.000000 +83682 -1.000000 +83683 -1.000000 +83684 -1.000000 +83685 -1.000000 +83686 -1.000000 +83687 -1.000000 +83688 -1.000000 +83689 -1.000000 +83690 -1.000000 +83691 -1.000000 +83692 -1.000000 +83693 -1.000000 +83694 -1.000000 +83695 -1.000000 +83696 -1.000000 +83697 -1.000000 +83699 -1.000000 +83700 -1.000000 +83701 -1.000000 +83702 -1.000000 +83703 -1.000000 +83704 -1.000000 +83705 -1.000000 +83706 -1.000000 +83707 -1.000000 +83708 -1.000000 +83709 -1.000000 +83710 -1.000000 +83711 -1.000000 +83712 -1.000000 +83713 -1.000000 +83714 -1.000000 +83715 -1.000000 +83716 -1.000000 +83717 -1.000000 +83718 -1.000000 +83719 -1.000000 +83720 -1.000000 +83721 -1.000000 +83722 -1.000000 +83723 -1.000000 +83724 -1.000000 +83725 -1.000000 +83726 -1.000000 +83727 -1.000000 +83728 -1.000000 +83729 -1.000000 +83731 -1.000000 +83732 -1.000000 +83733 -1.000000 +83734 -1.000000 +83735 -1.000000 +83736 -1.000000 +83737 -1.000000 +83738 -1.000000 +83739 -1.000000 +83740 -1.000000 +83741 -1.000000 +83742 -1.000000 +83743 -1.000000 +83744 -1.000000 +83745 -1.000000 +83746 -1.000000 +83747 -1.000000 +83748 -1.000000 +83749 -1.000000 +83750 -1.000000 +83751 -1.000000 +83752 -1.000000 +83753 -1.000000 +83754 -1.000000 +83755 -1.000000 +83756 -1.000000 +83757 -1.000000 +83758 -1.000000 +83759 -1.000000 +83760 -1.000000 +83761 -1.000000 +83762 -1.000000 +83763 -1.000000 +83764 -1.000000 +83765 -1.000000 +83766 -1.000000 +83767 -1.000000 +83768 -1.000000 +83769 -1.000000 +83770 -1.000000 +83771 -1.000000 +83772 -1.000000 +83773 -1.000000 +83774 -1.000000 +83775 -1.000000 +83776 -1.000000 +83777 -1.000000 +83778 -1.000000 +83779 -1.000000 +83780 -1.000000 +83781 -1.000000 +83782 -1.000000 +83783 -1.000000 +83784 -1.000000 +83785 -1.000000 +83786 -1.000000 +83787 -1.000000 +83788 -1.000000 +83789 -1.000000 +83790 -1.000000 +83791 -1.000000 +83792 -1.000000 +83793 -1.000000 +83794 -1.000000 +83795 -1.000000 +83796 -1.000000 +83797 -1.000000 +83798 -1.000000 +83799 -1.000000 +83800 -1.000000 +83801 -1.000000 +83802 -1.000000 +83803 -1.000000 +83804 -1.000000 +83805 -1.000000 +83806 -1.000000 +83807 -1.000000 +83808 -1.000000 +83809 -1.000000 +83810 -1.000000 +83811 -1.000000 +83812 -1.000000 +83813 -1.000000 +83814 -1.000000 +83815 -1.000000 +83816 -1.000000 +83817 -1.000000 +83818 -1.000000 +83819 -1.000000 +83820 -1.000000 +83821 -1.000000 +83822 -1.000000 +83823 -1.000000 +83824 -1.000000 +83825 -1.000000 +83826 -1.000000 +83827 -1.000000 +83828 -1.000000 +83829 -1.000000 +83830 -1.000000 +83831 -1.000000 +83832 -1.000000 +83833 -1.000000 +83834 -1.000000 +83835 -1.000000 +83836 -1.000000 +83837 -1.000000 +83838 -1.000000 +83839 -1.000000 +83840 -1.000000 +83841 -1.000000 +83842 -1.000000 +83843 -1.000000 +83844 -1.000000 +83845 -1.000000 +83846 -1.000000 +83847 -1.000000 +83848 -1.000000 +83849 -1.000000 +83850 -1.000000 +83851 -1.000000 +83852 -1.000000 +83853 -1.000000 +83854 -1.000000 +83855 -1.000000 +83856 -1.000000 +83857 -1.000000 +83858 -1.000000 +83859 -1.000000 +83860 -1.000000 +83861 -1.000000 +83862 -1.000000 +83863 -1.000000 +83864 -1.000000 +83865 -1.000000 +83866 -1.000000 +83867 -1.000000 +83868 -1.000000 +83869 -1.000000 +83870 -1.000000 +83871 -1.000000 +83872 -1.000000 +83873 -1.000000 +83874 -1.000000 +83875 -1.000000 +83876 -1.000000 +83877 -1.000000 +83878 -1.000000 +83879 -1.000000 +83880 -1.000000 +83881 -1.000000 +83882 -1.000000 +83883 -1.000000 +83884 -1.000000 +83885 -1.000000 +83886 -1.000000 +83887 -1.000000 +83888 -1.000000 +83889 -1.000000 +83890 -1.000000 +83891 -1.000000 +83892 -1.000000 +83893 -1.000000 +83894 -1.000000 +83895 -1.000000 +83896 -1.000000 +83897 -1.000000 +83898 -1.000000 +83899 -1.000000 +83900 -1.000000 +83901 -1.000000 +83902 -1.000000 +83903 -1.000000 +83904 -1.000000 +83905 -1.000000 +83906 -1.000000 +83907 -1.000000 +83908 -1.000000 +83909 -1.000000 +83910 -1.000000 +83911 -1.000000 +83912 -1.000000 +83913 -1.000000 +83914 -1.000000 +83915 -1.000000 +83916 -1.000000 +83917 -1.000000 +83918 -1.000000 +83919 -1.000000 +83920 -1.000000 +83921 -1.000000 +83922 -1.000000 +83923 -1.000000 +83924 -1.000000 +83925 -1.000000 +83926 -1.000000 +83927 -1.000000 +83928 -1.000000 +83929 -1.000000 +83930 -1.000000 +83931 -1.000000 +83932 -1.000000 +83933 -1.000000 +83934 -1.000000 +83935 -1.000000 +83936 -1.000000 +83937 -1.000000 +83938 -1.000000 +83939 -1.000000 +83940 -1.000000 +83941 -1.000000 +83942 -1.000000 +83943 -1.000000 +83944 -1.000000 +83945 -1.000000 +83946 -1.000000 +83947 -1.000000 +83948 -1.000000 +83949 -1.000000 +83950 -1.000000 +83951 -1.000000 +83952 -1.000000 +83953 -1.000000 +83954 -1.000000 +83955 -1.000000 +83956 -1.000000 +83957 -1.000000 +83958 -1.000000 +83959 -1.000000 +83960 -1.000000 +83961 -1.000000 +83962 -1.000000 +83963 -1.000000 +83964 -1.000000 +83965 -1.000000 +83966 -1.000000 +83967 -1.000000 +83968 -1.000000 +83969 -1.000000 +83970 -1.000000 +83971 -1.000000 +83972 -1.000000 +83973 -1.000000 +83974 -1.000000 +83975 -1.000000 +83976 -1.000000 +83977 -1.000000 +83978 -1.000000 +83979 -1.000000 +83981 -1.000000 +83983 -1.000000 +83984 -1.000000 +83985 -1.000000 +83986 -1.000000 +83988 -1.000000 +83989 -1.000000 +83990 -1.000000 +83991 -1.000000 +83992 -1.000000 +83993 -1.000000 +83994 -1.000000 +83995 -1.000000 +83996 -1.000000 +83997 -1.000000 +83998 -1.000000 +84000 -1.000000 +84001 -1.000000 +84002 -1.000000 +84003 -1.000000 +84004 -1.000000 +84005 -1.000000 +84006 -1.000000 +84007 -1.000000 +84008 -1.000000 +84009 -1.000000 +84010 -1.000000 +84011 -1.000000 +84012 -1.000000 +84013 -1.000000 +84014 -1.000000 +84015 -1.000000 +84016 -1.000000 +84017 -1.000000 +84018 -1.000000 +84019 -1.000000 +84020 -1.000000 +84021 -1.000000 +84022 -1.000000 +84023 -1.000000 +84024 -1.000000 +84025 -1.000000 +84026 -1.000000 +84027 -1.000000 +84028 -1.000000 +84029 -1.000000 +84030 -1.000000 +84032 -1.000000 +84033 -1.000000 +84034 -1.000000 +84035 -1.000000 +84037 -1.000000 +84038 -1.000000 +84039 -1.000000 +84040 -1.000000 +84041 -1.000000 +84042 -1.000000 +84043 -1.000000 +84044 -1.000000 +84045 -1.000000 +84046 -1.000000 +84047 -1.000000 +84048 -1.000000 +84049 -1.000000 +84050 -1.000000 +84052 -1.000000 +84053 -1.000000 +84055 -1.000000 +84056 -1.000000 +84057 -1.000000 +84058 -1.000000 +84059 -1.000000 +84060 -1.000000 +84061 -1.000000 +84062 -1.000000 +84063 -1.000000 +84064 -1.000000 +84065 -1.000000 +84066 -1.000000 +84067 -1.000000 +84068 -1.000000 +84069 -1.000000 +84070 -1.000000 +84071 -1.000000 +84072 -1.000000 +84073 -1.000000 +84074 -1.000000 +84075 -1.000000 +84076 -1.000000 +84078 -1.000000 +84080 -1.000000 +84081 -1.000000 +84082 -1.000000 +84083 -1.000000 +84084 -1.000000 +84085 -1.000000 +84086 -1.000000 +84087 -1.000000 +84088 -1.000000 +84089 -1.000000 +84090 -1.000000 +84091 -1.000000 +84092 -1.000000 +84093 -1.000000 +84094 -1.000000 +84095 -1.000000 +84096 -1.000000 +84097 -1.000000 +84098 -1.000000 +84099 -1.000000 +84100 -1.000000 +84101 -1.000000 +84102 -1.000000 +84103 -1.000000 +84104 -1.000000 +84105 -1.000000 +84106 -1.000000 +84107 -1.000000 +84108 -1.000000 +84109 -1.000000 +84110 -1.000000 +84111 -1.000000 +84112 -1.000000 +84113 -1.000000 +84114 -1.000000 +84116 -1.000000 +84117 -1.000000 +84118 -1.000000 +84119 -1.000000 +84120 -1.000000 +84121 -1.000000 +84122 -1.000000 +84123 -1.000000 +84124 -1.000000 +84125 -1.000000 +84126 -1.000000 +84127 -1.000000 +84128 -1.000000 +84129 -1.000000 +84130 -1.000000 +84131 -1.000000 +84132 -1.000000 +84133 -1.000000 +84134 -1.000000 +84135 -1.000000 +84136 -1.000000 +84137 -1.000000 +84138 -1.000000 +84139 -1.000000 +84140 -1.000000 +84141 -1.000000 +84144 -1.000000 +84145 -1.000000 +84146 -1.000000 +84147 -1.000000 +84148 -1.000000 +84149 -1.000000 +84150 -1.000000 +84151 -1.000000 +84152 -1.000000 +84153 -1.000000 +84154 -1.000000 +84155 -1.000000 +84156 -1.000000 +84157 -1.000000 +84159 -1.000000 +84174 -1.000000 +84175 -1.000000 +84176 -1.000000 +84177 -1.000000 +84178 -1.000000 +84179 -1.000000 +84181 -1.000000 +84182 -1.000000 +84183 -1.000000 +84184 -1.000000 +84185 -1.000000 +84186 -1.000000 +84187 -1.000000 +84188 -1.000000 +84189 -1.000000 +84190 -1.000000 +84191 -1.000000 +84192 -1.000000 +84193 -1.000000 +84194 -1.000000 +84195 -1.000000 +84196 -1.000000 +84197 -1.000000 +84198 -1.000000 +84199 -1.000000 +84200 -1.000000 +84201 -1.000000 +84202 -1.000000 +84203 -1.000000 +84204 -1.000000 +84205 -1.000000 +84206 -1.000000 +84207 -1.000000 +84208 -1.000000 +84209 -1.000000 +84210 -1.000000 +84211 -1.000000 +84212 -1.000000 +84213 -1.000000 +84214 -1.000000 +84215 -1.000000 +84216 -1.000000 +84217 -1.000000 +84218 -1.000000 +84219 -1.000000 +84220 -1.000000 +84221 -1.000000 +84222 -1.000000 +84223 -1.000000 +84224 -1.000000 +84225 -1.000000 +84226 -1.000000 +84227 -1.000000 +84228 -1.000000 +84229 -1.000000 +84230 -1.000000 +84231 -1.000000 +84232 -1.000000 +84233 -1.000000 +84234 -1.000000 +84235 -1.000000 +84236 -1.000000 +84237 -1.000000 +84238 -1.000000 +84239 -1.000000 +84240 -1.000000 +84241 -1.000000 +84242 -1.000000 +84243 -1.000000 +84244 -1.000000 +84245 -1.000000 +84246 -1.000000 +84247 -1.000000 +84248 -1.000000 +84249 -1.000000 +84250 -1.000000 +84251 -1.000000 +84252 -1.000000 +84253 -1.000000 +84254 -1.000000 +84255 -1.000000 +84256 -1.000000 +84257 -1.000000 +84258 -1.000000 +84259 -1.000000 +84260 -1.000000 +84261 -1.000000 +84262 -1.000000 +84263 -1.000000 +84264 -1.000000 +84265 -1.000000 +84266 -1.000000 +84267 -1.000000 +84268 -1.000000 +84269 -1.000000 +84270 -1.000000 +84272 -1.000000 +84273 -1.000000 +84274 -1.000000 +84276 -1.000000 +84277 -1.000000 +84278 -1.000000 +84279 -1.000000 +84280 -1.000000 +84281 -1.000000 +84282 -1.000000 +84283 -1.000000 +84284 -1.000000 +84285 -1.000000 +84286 -1.000000 +84287 -1.000000 +84288 -1.000000 +84289 -1.000000 +84290 -1.000000 +84291 -1.000000 +84292 -1.000000 +84293 -1.000000 +84294 -1.000000 +84295 -1.000000 +84296 -1.000000 +84297 -1.000000 +84298 -1.000000 +84299 -1.000000 +84300 -1.000000 +84301 -1.000000 +84302 -1.000000 +84303 -1.000000 +84304 -1.000000 +84305 -1.000000 +84306 -1.000000 +84307 -1.000000 +84309 -1.000000 +84310 -1.000000 +84311 -1.000000 +84312 -1.000000 +84313 -1.000000 +84314 -1.000000 +84316 -1.000000 +84317 -1.000000 +84318 -1.000000 +84319 -1.000000 +84320 -1.000000 +84321 -1.000000 +84325 -1.000000 +84326 -1.000000 +84327 -1.000000 +84329 -1.000000 +84330 -1.000000 +84331 -1.000000 +84332 -1.000000 +84333 -1.000000 +84334 -1.000000 +84335 -1.000000 +84336 -1.000000 +84337 -1.000000 +84338 -1.000000 +84339 -1.000000 +84340 -1.000000 +84341 -1.000000 +84342 -1.000000 +84343 -1.000000 +84344 -1.000000 +84345 -1.000000 +84346 -1.000000 +84347 -1.000000 +84348 -1.000000 +84349 -1.000000 +84350 -1.000000 +84351 -1.000000 +84353 -1.000000 +84354 -1.000000 +84355 -1.000000 +84356 -1.000000 +84357 -1.000000 +84358 -1.000000 +84359 -1.000000 +84360 -1.000000 +84361 -1.000000 +84362 -1.000000 +84363 -1.000000 +84364 -1.000000 +84365 -1.000000 +84366 -1.000000 +84367 -1.000000 +84368 -1.000000 +84369 -1.000000 +84370 -1.000000 +84371 -1.000000 +84372 -1.000000 +84373 -1.000000 +84374 -1.000000 +84375 -1.000000 +84376 -1.000000 +84377 -1.000000 +84378 -1.000000 +84379 -1.000000 +84380 -1.000000 +84381 -1.000000 +84382 -1.000000 +84383 -1.000000 +84384 -1.000000 +84385 -1.000000 +84386 -1.000000 +84387 -1.000000 +84388 -1.000000 +84389 -1.000000 +84390 -1.000000 +84391 -1.000000 +84392 -1.000000 +84393 -1.000000 +84394 -1.000000 +84395 -1.000000 +84396 -1.000000 +84397 -1.000000 +84398 -1.000000 +84399 -1.000000 +84400 -1.000000 +84401 -1.000000 +84402 -1.000000 +84403 -1.000000 +84404 -1.000000 +84405 -1.000000 +84406 -1.000000 +84407 -1.000000 +84409 -1.000000 +84410 -1.000000 +84411 -1.000000 +84412 -1.000000 +84413 -1.000000 +84414 -1.000000 +84415 -1.000000 +84416 -1.000000 +84417 -1.000000 +84418 -1.000000 +84419 -1.000000 +84420 -1.000000 +84421 -1.000000 +84422 -1.000000 +84423 -1.000000 +84424 -1.000000 +84425 -1.000000 +84426 -1.000000 +84427 -1.000000 +84429 -1.000000 +84430 -1.000000 +84431 -1.000000 +84432 -1.000000 +84434 -1.000000 +84436 -1.000000 +84437 -1.000000 +84438 -1.000000 +84439 -1.000000 +84440 -1.000000 +84441 -1.000000 +84442 -1.000000 +84443 -1.000000 +84444 -1.000000 +84445 -1.000000 +84446 -1.000000 +84447 -1.000000 +84448 -1.000000 +84449 -1.000000 +84450 -1.000000 +84451 -1.000000 +84452 -1.000000 +84453 -1.000000 +84454 -1.000000 +84455 -1.000000 +84456 -1.000000 +84457 -1.000000 +84458 -1.000000 +84459 -1.000000 +84460 -1.000000 +84461 -1.000000 +84463 -1.000000 +84466 -1.000000 +84468 -1.000000 +84469 -1.000000 +84470 -1.000000 +84471 -1.000000 +84472 -1.000000 +84473 -1.000000 +84474 -1.000000 +84475 -1.000000 +84476 -1.000000 +84477 -1.000000 +84478 -1.000000 +84479 -1.000000 +84480 -1.000000 +84481 -1.000000 +84482 -1.000000 +84483 -1.000000 +84484 -1.000000 +84485 -1.000000 +84486 -1.000000 +84487 -1.000000 +84488 -1.000000 +84489 -1.000000 +84490 -1.000000 +84491 -1.000000 +84492 -1.000000 +84493 -1.000000 +84495 -1.000000 +84496 -1.000000 +84497 -1.000000 +84498 -1.000000 +84499 -1.000000 +84500 -1.000000 +84501 -1.000000 +84502 -1.000000 +84503 -1.000000 +84504 -1.000000 +84505 -1.000000 +84506 -1.000000 +84507 -1.000000 +84508 -1.000000 +84509 -1.000000 +84510 -1.000000 +84511 -1.000000 +84512 -1.000000 +84513 -1.000000 +84514 -1.000000 +84515 -1.000000 +84516 -1.000000 +84518 -1.000000 +84521 -1.000000 +84522 -1.000000 +84523 -1.000000 +84525 -1.000000 +84526 -1.000000 +84527 -1.000000 +84528 -1.000000 +84529 -1.000000 +84530 -1.000000 +84531 -1.000000 +84532 -1.000000 +84534 -1.000000 +84535 -1.000000 +84536 -1.000000 +84537 -1.000000 +84538 -1.000000 +84539 -1.000000 +84540 -1.000000 +84541 -1.000000 +84542 -1.000000 +84543 -1.000000 +84544 -1.000000 +84545 -1.000000 +84546 -1.000000 +84547 -1.000000 +84548 -1.000000 +84549 -1.000000 +84550 -1.000000 +84551 -1.000000 +84552 -1.000000 +84555 -1.000000 +84556 -1.000000 +84558 -1.000000 +84559 -1.000000 +84560 -1.000000 +84561 -1.000000 +84562 -1.000000 +84563 -1.000000 +84564 -1.000000 +84565 -1.000000 +84566 -1.000000 +84567 -1.000000 +84568 -1.000000 +84569 -1.000000 +84570 -1.000000 +84571 -1.000000 +84572 -1.000000 +84573 -1.000000 +84574 -1.000000 +84575 -1.000000 +84576 -1.000000 +84577 -1.000000 +84578 -1.000000 +84579 -1.000000 +84580 -1.000000 +84582 -1.000000 +84583 -1.000000 +84584 -1.000000 +84585 -1.000000 +84586 -1.000000 +84587 -1.000000 +84589 -1.000000 +84590 -1.000000 +84591 -1.000000 +84592 -1.000000 +84596 -1.000000 +84597 -1.000000 +84598 -1.000000 +84599 -1.000000 +84601 -1.000000 +84602 -1.000000 +84603 -1.000000 +84604 -1.000000 +84605 -1.000000 +84606 -1.000000 +84607 -1.000000 +84608 -1.000000 +84609 -1.000000 +84610 -1.000000 +84611 -1.000000 +84615 -1.000000 +84616 -1.000000 +84617 -1.000000 +84618 -1.000000 +84619 -1.000000 +84621 -1.000000 +84622 -1.000000 +84623 -1.000000 +84624 -1.000000 +84625 -1.000000 +84626 -1.000000 +84627 -1.000000 +84629 -1.000000 +84630 -1.000000 +84631 -1.000000 +84632 -1.000000 +84633 -1.000000 +84634 -1.000000 +84635 -1.000000 +84636 -1.000000 +84637 -1.000000 +84638 -1.000000 +84641 -1.000000 +84642 -1.000000 +84643 -1.000000 +84644 -1.000000 +84645 -1.000000 +84646 -1.000000 +84647 -1.000000 +84649 -1.000000 +84650 -1.000000 +84651 -1.000000 +84652 -1.000000 +84653 -1.000000 +84654 -1.000000 +84659 -1.000000 +84660 -1.000000 +84661 -1.000000 +84662 -1.000000 +84663 -1.000000 +84664 -1.000000 +84665 -1.000000 +84666 -1.000000 +84667 -1.000000 +84668 -1.000000 +84669 -1.000000 +84670 -1.000000 +84672 -1.000000 +84673 -1.000000 +84678 -1.000000 +84680 -1.000000 +84682 -1.000000 +84683 -1.000000 +84684 -1.000000 +84685 -1.000000 +84686 -1.000000 +84687 -1.000000 +84688 -1.000000 +84689 -1.000000 +84692 -1.000000 +84693 -1.000000 +84694 -1.000000 +84697 -1.000000 +84698 -1.000000 +84699 -1.000000 +84700 -1.000000 +84701 -1.000000 +84702 -1.000000 +84703 -1.000000 +84704 -1.000000 +84705 -1.000000 +84706 -1.000000 +84708 -1.000000 +84709 -1.000000 +84712 -1.000000 +84713 -1.000000 +84714 -1.000000 +84717 -1.000000 +84718 -1.000000 +84719 -1.000000 +84720 -1.000000 +84721 -1.000000 +84722 -1.000000 +84723 -1.000000 +84724 -1.000000 +84725 -1.000000 +84726 -1.000000 +84727 -1.000000 +84728 -1.000000 +84729 -1.000000 +84730 -1.000000 +84731 -1.000000 +84732 -1.000000 +84733 -1.000000 +84734 -1.000000 +84735 -1.000000 +84736 -1.000000 +84737 -1.000000 +84738 -1.000000 +84739 -1.000000 +84740 -1.000000 +84741 -1.000000 +84742 -1.000000 +84743 -1.000000 +84744 -1.000000 +84745 -1.000000 +84746 -1.000000 +84749 -1.000000 +84750 -1.000000 +84751 -1.000000 +84753 -1.000000 +84755 -1.000000 +84756 -1.000000 +84757 -1.000000 +84758 -1.000000 +84759 -1.000000 +84760 -1.000000 +84761 -1.000000 +84762 -1.000000 +84765 -1.000000 +84767 -1.000000 +84768 -1.000000 +84769 -1.000000 +84770 -1.000000 +84772 -1.000000 +84773 -1.000000 +84774 -1.000000 +84776 -1.000000 +84777 -1.000000 +84778 -1.000000 +84779 -1.000000 +84780 -1.000000 +84781 -1.000000 +84782 -1.000000 +84784 -1.000000 +84785 -1.000000 +84786 -1.000000 +84787 -1.000000 +84788 -1.000000 +84789 -1.000000 +84790 -1.000000 +84791 -1.000000 +84792 -1.000000 +84793 -1.000000 +84794 -1.000000 +84795 -1.000000 +84796 -1.000000 +84797 -1.000000 +84798 -1.000000 +84799 -1.000000 +84800 -1.000000 +84801 -1.000000 +84802 -1.000000 +84803 -1.000000 +84804 -1.000000 +84805 -1.000000 +84806 -1.000000 +84807 -1.000000 +84809 -1.000000 +84810 -1.000000 +84811 -1.000000 +84812 -1.000000 +84813 -1.000000 +84814 -1.000000 +84815 -1.000000 +84816 -1.000000 +84817 -1.000000 +84821 -1.000000 +84822 -1.000000 +84823 -1.000000 +84824 -1.000000 +84825 -1.000000 +84830 -1.000000 +84831 -1.000000 +84833 -1.000000 +84834 -1.000000 +84835 -1.000000 +84836 -1.000000 +84840 -1.000000 +84842 -1.000000 +84843 -1.000000 +84844 -1.000000 +84845 -1.000000 +84847 -1.000000 +84848 -1.000000 +84849 -1.000000 +84850 -1.000000 +84851 -1.000000 +84852 -1.000000 +84854 -1.000000 +84855 -1.000000 +84856 -1.000000 +84859 -1.000000 +84863 -1.000000 +84864 -1.000000 +84866 -1.000000 +84868 -1.000000 +84871 -1.000000 +84872 -1.000000 +84873 -1.000000 +84874 -1.000000 +84875 -1.000000 +84876 -1.000000 +84877 -1.000000 +84879 -1.000000 +84880 -1.000000 +84881 -1.000000 +84882 -1.000000 +84883 -1.000000 +84884 -1.000000 +84889 -1.000000 +84890 -1.000000 +84892 -1.000000 +84895 -1.000000 +84896 -1.000000 +84897 -1.000000 +84900 -1.000000 +84902 -1.000000 +84903 -1.000000 +84904 -1.000000 +84905 -1.000000 +84906 -1.000000 +84907 -1.000000 +84908 -1.000000 +84909 -1.000000 +84910 -1.000000 +84911 -1.000000 +84913 -1.000000 +84914 -1.000000 +84915 -1.000000 +84916 -1.000000 +84917 -1.000000 +84918 -1.000000 +84919 -1.000000 +84920 -1.000000 +84921 -1.000000 +84922 -1.000000 +84923 -1.000000 +84924 -1.000000 +84925 -1.000000 +84926 -1.000000 +84927 -1.000000 +84928 -1.000000 +84929 -1.000000 +84930 -1.000000 +84931 -1.000000 +84932 -1.000000 +84934 -1.000000 +84936 -1.000000 +84938 -1.000000 +84939 -1.000000 +84940 -1.000000 +84941 -1.000000 +84944 -1.000000 +84945 -1.000000 +84948 -1.000000 +84949 -1.000000 +84951 -1.000000 +84952 -1.000000 +84953 -1.000000 +84954 -1.000000 +84956 -1.000000 +84958 -1.000000 +84959 -1.000000 +84961 -1.000000 +84963 -1.000000 +84964 -1.000000 +84966 -1.000000 +84967 -1.000000 +84968 -1.000000 +84969 -1.000000 +84970 -1.000000 +84971 -1.000000 +84972 -1.000000 +84973 -1.000000 +84974 -1.000000 +84975 -1.000000 +84976 -1.000000 +84977 -1.000000 +84978 -1.000000 +84979 -1.000000 +84980 -1.000000 +84981 -1.000000 +84982 -1.000000 +84992 -1.000000 +84997 -1.000000 +85001 -1.000000 +85002 -1.000000 +85003 -1.000000 +85004 -1.000000 +85006 -1.000000 +85007 -1.000000 +85008 -1.000000 +85009 -1.000000 +85010 -1.000000 +85011 -1.000000 +85012 -1.000000 +85014 -1.000000 +85015 -1.000000 +85016 -1.000000 +85017 -1.000000 +85018 -1.000000 +85022 -1.000000 +85023 -1.000000 +85025 -1.000000 +85026 -1.000000 +85028 -1.000000 +85029 -1.000000 +85030 -1.000000 +85031 -1.000000 +85032 -1.000000 +85034 -1.000000 +85035 -1.000000 +85036 -1.000000 +85037 -1.000000 +85038 -1.000000 +85039 -1.000000 +85040 -1.000000 +85041 -1.000000 +85042 -1.000000 +85043 -1.000000 +85044 -1.000000 +85045 -1.000000 +85046 -1.000000 +85047 -1.000000 +85049 -1.000000 +85050 -1.000000 +85051 -1.000000 +85052 -1.000000 +85054 -1.000000 +85056 -1.000000 +85057 -1.000000 +85059 -1.000000 +85063 -1.000000 +85064 -1.000000 +85065 -1.000000 +85066 -1.000000 +85068 -1.000000 +85069 -1.000000 +85075 -1.000000 +85079 -1.000000 +85080 -1.000000 +85081 -1.000000 +85082 -1.000000 +85083 -1.000000 +85084 -1.000000 +85085 -1.000000 +85086 -1.000000 +85087 -1.000000 +85088 -1.000000 +85089 -1.000000 +85090 -1.000000 +85091 -1.000000 +85092 -1.000000 +85093 -1.000000 +85094 -1.000000 +85095 -1.000000 +85096 -1.000000 +85097 -1.000000 +85098 -1.000000 +85099 -1.000000 +85100 -1.000000 +85101 -1.000000 +85102 -1.000000 +85105 -1.000000 +85107 -1.000000 +85109 -1.000000 +85110 -1.000000 +85111 -1.000000 +85112 -1.000000 +85113 -1.000000 +85114 -1.000000 +85115 -1.000000 +85116 -1.000000 +85117 -1.000000 +85118 -1.000000 +85119 -1.000000 +85120 -1.000000 +85121 -1.000000 +85122 -1.000000 +85123 -1.000000 +85124 -1.000000 +85125 -1.000000 +85126 -1.000000 +85127 -1.000000 +85128 -1.000000 +85129 -1.000000 +85130 -1.000000 +85131 -1.000000 +85132 -1.000000 +85133 -1.000000 +85134 -1.000000 +85135 -1.000000 +85136 -1.000000 +85137 -1.000000 +85138 -1.000000 +85139 -1.000000 +85140 -1.000000 +85141 -1.000000 +85142 -1.000000 +85143 -1.000000 +85144 -1.000000 +85145 -1.000000 +85146 -1.000000 +85147 -1.000000 +85148 -1.000000 +85149 -1.000000 +85150 -1.000000 +85151 -1.000000 +85152 -1.000000 +85153 -1.000000 +85154 -1.000000 +85155 -1.000000 +85156 -1.000000 +85157 -1.000000 +85158 -1.000000 +85159 -1.000000 +85160 -1.000000 +85161 -1.000000 +85163 -1.000000 +85166 -1.000000 +85167 -1.000000 +85169 -1.000000 +85172 -1.000000 +85173 -1.000000 +85174 -1.000000 +85175 -1.000000 +85178 -1.000000 +85180 -1.000000 +85181 -1.000000 +85183 -1.000000 +85184 -1.000000 +85186 -1.000000 +85194 -1.000000 +85195 -1.000000 +85196 -1.000000 +85197 -1.000000 +85198 -1.000000 +85199 -1.000000 +85201 -1.000000 +85203 -1.000000 +85205 -1.000000 +85206 -1.000000 +85207 -1.000000 +85208 -1.000000 +85209 -1.000000 +85213 -1.000000 +85214 -1.000000 +85215 -1.000000 +85216 -1.000000 +85217 -1.000000 +85218 -1.000000 +85219 -1.000000 +85220 -1.000000 +85221 -1.000000 +85223 -1.000000 +85224 -1.000000 +85225 -1.000000 +85226 -1.000000 +85227 -1.000000 +85228 -1.000000 +85230 -1.000000 +85231 -1.000000 +85232 -1.000000 +85233 -1.000000 +85234 -1.000000 +85235 -1.000000 +85237 -1.000000 +85242 -1.000000 +85243 -1.000000 +85244 -1.000000 +85245 -1.000000 +85246 -1.000000 +85247 -1.000000 +85248 -1.000000 +85249 -1.000000 +85250 -1.000000 +85254 -1.000000 +85255 -1.000000 +85256 -1.000000 +85260 -1.000000 +85261 -1.000000 +85262 -1.000000 +85263 -1.000000 +85264 -1.000000 +85265 -1.000000 +85266 -1.000000 +85268 -1.000000 +85269 -1.000000 +85270 -1.000000 +85271 -1.000000 +85272 -1.000000 +85273 -1.000000 +85274 -1.000000 +85276 -1.000000 +85277 -1.000000 +85278 -1.000000 +85279 -1.000000 +85280 -1.000000 +85281 -1.000000 +85282 -1.000000 +85283 -1.000000 +85284 -1.000000 +85288 -1.000000 +85289 -1.000000 +85296 -1.000000 +85297 -1.000000 +85298 -1.000000 +85299 -1.000000 +85301 -1.000000 +85302 -1.000000 +85303 -1.000000 +85304 -1.000000 +85305 -1.000000 +85306 -1.000000 +85307 -1.000000 +85308 -1.000000 +85309 -1.000000 +85310 -1.000000 +85311 -1.000000 +85312 -1.000000 +85313 -1.000000 +85314 -1.000000 +85315 -1.000000 +85316 -1.000000 +85319 -1.000000 +85320 -1.000000 +85321 -1.000000 +85322 -1.000000 +85323 -1.000000 +85324 -1.000000 +85325 -1.000000 +85326 -1.000000 +85327 -1.000000 +85328 -1.000000 +85329 -1.000000 +85330 -1.000000 +85331 -1.000000 +85332 -1.000000 +85333 -1.000000 +85334 -1.000000 +85335 -1.000000 +85336 -1.000000 +85337 -1.000000 +85338 -1.000000 +85339 -1.000000 +85340 -1.000000 +85341 -1.000000 +85342 -1.000000 +85343 -1.000000 +85344 -1.000000 +85345 -1.000000 +85346 -1.000000 +85347 -1.000000 +85348 -1.000000 +85349 -1.000000 +85350 -1.000000 +85351 -1.000000 +85352 -1.000000 +85353 -1.000000 +85354 -1.000000 +85355 -1.000000 +85356 -1.000000 +85357 -1.000000 +85358 -1.000000 +85361 -1.000000 +85364 -1.000000 +85365 -1.000000 +85366 -1.000000 +85367 -1.000000 +85368 -1.000000 +85369 -1.000000 +85370 -1.000000 +85371 -1.000000 +85372 -1.000000 +85373 -1.000000 +85374 -1.000000 +85375 -1.000000 +85376 -1.000000 +85378 -1.000000 +85380 -1.000000 +85381 -1.000000 +85388 -1.000000 +85389 -1.000000 +85391 -1.000000 +85392 -1.000000 +85393 -1.000000 +85394 -1.000000 +85398 -1.000000 +85399 -1.000000 +85400 -1.000000 +85402 -1.000000 +85403 -1.000000 +85405 -1.000000 +85406 -1.000000 +85407 -1.000000 +85412 -1.000000 +85415 -1.000000 +85416 -1.000000 +85418 -1.000000 +85419 -1.000000 +85420 -1.000000 +85423 -1.000000 +85425 -1.000000 +85428 -1.000000 +85431 -1.000000 +85432 -1.000000 +85433 -1.000000 +85435 -1.000000 +85436 -1.000000 +85437 -1.000000 +85438 -1.000000 +85439 -1.000000 +85441 -1.000000 +85444 -1.000000 +85458 -1.000000 +85459 -1.000000 +85461 -1.000000 +85462 -1.000000 +85463 -1.000000 +85464 -1.000000 +85465 -1.000000 +85466 -1.000000 +85468 -1.000000 +85469 -1.000000 +85470 -1.000000 +85471 -1.000000 +85474 -1.000000 +85475 -1.000000 +85476 -1.000000 +85477 -1.000000 +85478 -1.000000 +85479 -1.000000 +85480 -1.000000 +85481 -1.000000 +85485 -1.000000 +85486 -1.000000 +85487 -1.000000 +85488 -1.000000 +85489 -1.000000 +85490 -1.000000 +85491 -1.000000 +85492 -1.000000 +85493 -1.000000 +85494 -1.000000 +85495 -1.000000 +85496 -1.000000 +85497 -1.000000 +85498 -1.000000 +85499 -1.000000 +85500 -1.000000 +85501 -1.000000 +85502 -1.000000 +85503 -1.000000 +85504 -1.000000 +85505 -1.000000 +85506 -1.000000 +85507 -1.000000 +85508 -1.000000 +85509 -1.000000 +85510 -1.000000 +85511 -1.000000 +85512 -1.000000 +85513 -1.000000 +85514 -1.000000 +85515 -1.000000 +85516 -1.000000 +85517 -1.000000 +85518 -1.000000 +85519 -1.000000 +85520 -1.000000 +85521 -1.000000 +85522 -1.000000 +85523 -1.000000 +85524 -1.000000 +85525 -1.000000 +85526 -1.000000 +85527 -1.000000 +85528 -1.000000 +85529 -1.000000 +85530 -1.000000 +85531 -1.000000 +85532 -1.000000 +85533 -1.000000 +85534 -1.000000 +85535 -1.000000 +85536 -1.000000 +85537 -1.000000 +85538 -1.000000 +85539 -1.000000 +85540 -1.000000 +85541 -1.000000 +85542 -1.000000 +85543 -1.000000 +85544 -1.000000 +85545 -1.000000 +85546 -1.000000 +85547 -1.000000 +85548 -1.000000 +85549 -1.000000 +85550 -1.000000 +85552 -1.000000 +85553 -1.000000 +85555 -1.000000 +85556 -1.000000 +85557 -1.000000 +85558 -1.000000 +85559 -1.000000 +85560 -1.000000 +85561 -1.000000 +85562 -1.000000 +85563 -1.000000 +85564 -1.000000 +85565 -1.000000 +85566 -1.000000 +85567 -1.000000 +85568 -1.000000 +85569 -1.000000 +85570 -1.000000 +85571 -1.000000 +85572 -1.000000 +85573 -1.000000 +85574 -1.000000 +85575 -1.000000 +85576 -1.000000 +85577 -1.000000 +85578 -1.000000 +85579 -1.000000 +85580 -1.000000 +85581 -1.000000 +85582 -1.000000 +85583 -1.000000 +85584 -1.000000 +85585 -1.000000 +85586 -1.000000 +85587 -1.000000 +85588 -1.000000 +85589 -1.000000 +85590 -1.000000 +85591 -1.000000 +85592 -1.000000 +85594 -1.000000 +85595 -1.000000 +85596 -1.000000 +85597 -1.000000 +85598 -1.000000 +85599 -1.000000 +85600 -1.000000 +85601 -1.000000 +85602 -1.000000 +85604 -1.000000 +85606 -1.000000 +85607 -1.000000 +85608 -1.000000 +85609 -1.000000 +85610 -1.000000 +85611 -1.000000 +85612 -1.000000 +85613 -1.000000 +85614 -1.000000 +85615 -1.000000 +85616 -1.000000 +85617 -1.000000 +85618 -1.000000 +85619 -1.000000 +85620 -1.000000 +85621 -1.000000 +85622 -1.000000 +85623 -1.000000 +85624 -1.000000 +85625 -1.000000 +85626 -1.000000 +85627 -1.000000 +85628 -1.000000 +85629 -1.000000 +85630 -1.000000 +85631 -1.000000 +85632 -1.000000 +85633 -1.000000 +85634 -1.000000 +85635 -1.000000 +85636 -1.000000 +85637 -1.000000 +85638 -1.000000 +85639 -1.000000 +85640 -1.000000 +85641 -1.000000 +85642 -1.000000 +85644 -1.000000 +85645 -1.000000 +85646 -1.000000 +85647 -1.000000 +85648 -1.000000 +85649 -1.000000 +85650 -1.000000 +85651 -1.000000 +85652 -1.000000 +85653 -1.000000 +85654 -1.000000 +85655 -1.000000 +85656 -1.000000 +85657 -1.000000 +85658 -1.000000 +85659 -1.000000 +85660 -1.000000 +85661 -1.000000 +85662 -1.000000 +85663 -1.000000 +85664 -1.000000 +85665 -1.000000 +85666 -1.000000 +85667 -1.000000 +85668 -1.000000 +85670 -1.000000 +85671 -1.000000 +85672 -1.000000 +85673 -1.000000 +85676 -1.000000 +85677 -1.000000 +85678 -1.000000 +85679 -1.000000 +85680 -1.000000 +85681 -1.000000 +85682 -1.000000 +85683 -1.000000 +85684 -1.000000 +85685 -1.000000 +85686 -1.000000 +85687 -1.000000 +85688 -1.000000 +85689 -1.000000 +85690 -1.000000 +85691 -1.000000 +85692 -1.000000 +85693 -1.000000 +85694 -1.000000 +85695 -1.000000 +85696 -1.000000 +85697 -1.000000 +85698 -1.000000 +85699 -1.000000 +85700 -1.000000 +85701 -1.000000 +85702 -1.000000 +85703 -1.000000 +85704 -1.000000 +85705 -1.000000 +85706 -1.000000 +85707 -1.000000 +85708 -1.000000 +85709 -1.000000 +85710 -1.000000 +85711 -1.000000 +85712 -1.000000 +85713 -1.000000 +85714 -1.000000 +85715 -1.000000 +85716 -1.000000 +85717 -1.000000 +85718 -1.000000 +85719 -1.000000 +85721 -1.000000 +85723 -1.000000 +85724 -1.000000 +85725 -1.000000 +85726 -1.000000 +85728 -1.000000 +85729 -1.000000 +85730 -1.000000 +85731 -1.000000 +85732 -1.000000 +85733 -1.000000 +85734 -1.000000 +85735 -1.000000 +85736 -1.000000 +85737 -1.000000 +85738 -1.000000 +85739 -1.000000 +85740 -1.000000 +85741 -1.000000 +85742 -1.000000 +85743 -1.000000 +85744 -1.000000 +85745 -1.000000 +85746 -1.000000 +85747 -1.000000 +85748 -1.000000 +85749 -1.000000 +85750 -1.000000 +85751 -1.000000 +85752 -1.000000 +85754 -1.000000 +85755 -1.000000 +85756 -1.000000 +85757 -1.000000 +85758 -1.000000 +85759 -1.000000 +85760 -1.000000 +85761 -1.000000 +85762 -1.000000 +85764 -1.000000 +85765 -1.000000 +85766 -1.000000 +85768 -1.000000 +85769 -1.000000 +85770 -1.000000 +85772 -1.000000 +85773 -1.000000 +85775 -1.000000 +85776 -1.000000 +85777 -1.000000 +85781 -1.000000 +85782 -1.000000 +85783 -1.000000 +85784 -1.000000 +85785 -1.000000 +85786 -1.000000 +85787 -1.000000 +85788 -1.000000 +85789 -1.000000 +85790 -1.000000 +85791 -1.000000 +85793 -1.000000 +85794 -1.000000 +85795 -1.000000 +85796 -1.000000 +85797 -1.000000 +85798 -1.000000 +85799 -1.000000 +85800 -1.000000 +85802 -1.000000 +85810 -1.000000 +85811 -1.000000 +85812 -1.000000 +85813 -1.000000 +85814 -1.000000 +85815 -1.000000 +85816 -1.000000 +85817 -1.000000 +85818 -1.000000 +85819 -1.000000 +85820 -1.000000 +85821 -1.000000 +85822 -1.000000 +85826 -1.000000 +85827 -1.000000 +85828 -1.000000 +85829 -1.000000 +85830 -1.000000 +85831 -1.000000 +85832 -1.000000 +85833 -1.000000 +85834 -1.000000 +85835 -1.000000 +85836 -1.000000 +85837 -1.000000 +85838 -1.000000 +85839 -1.000000 +85840 -1.000000 +85841 -1.000000 +85842 -1.000000 +85844 -1.000000 +85845 -1.000000 +85846 -1.000000 +85847 -1.000000 +85849 -1.000000 +85850 -1.000000 +85851 -1.000000 +85852 -1.000000 +85853 -1.000000 +85854 -1.000000 +85855 -1.000000 +85856 -1.000000 +85859 -1.000000 +85860 -1.000000 +85862 -1.000000 +85863 -1.000000 +85864 -1.000000 +85866 -1.000000 +85868 -1.000000 +85870 -1.000000 +85871 -1.000000 +85872 -1.000000 +85873 -1.000000 +85874 -1.000000 +85875 -1.000000 +85876 -1.000000 +85877 -1.000000 +85878 -1.000000 +85879 -1.000000 +85880 -1.000000 +85881 -1.000000 +85884 -1.000000 +85885 -1.000000 +85886 -1.000000 +85887 -1.000000 +85889 -1.000000 +85890 -1.000000 +85891 -1.000000 +85892 -1.000000 +85893 -1.000000 +85894 -1.000000 +85895 -1.000000 +85896 -1.000000 +85897 -1.000000 +85898 -1.000000 +85899 -1.000000 +85900 -1.000000 +85901 -1.000000 +85902 -1.000000 +85903 -1.000000 +85904 -1.000000 +85905 -1.000000 +85906 -1.000000 +85907 -1.000000 +85908 -1.000000 +85909 -1.000000 +85910 -1.000000 +85911 -1.000000 +85912 -1.000000 +85913 -1.000000 +85914 -1.000000 +85915 -1.000000 +85916 -1.000000 +85917 -1.000000 +85918 -1.000000 +85919 -1.000000 +85920 -1.000000 +85921 -1.000000 +85922 -1.000000 +85923 -1.000000 +85924 -1.000000 +85926 -1.000000 +85927 -1.000000 +85928 -1.000000 +85929 -1.000000 +85930 -1.000000 +85931 -1.000000 +85932 -1.000000 +85933 -1.000000 +85934 -1.000000 +85935 -1.000000 +85936 -1.000000 +85937 -1.000000 +85945 -1.000000 +85949 -1.000000 +85950 -1.000000 +85951 -1.000000 +85952 -1.000000 +85953 -1.000000 +85954 -1.000000 +85955 -1.000000 +85956 -1.000000 +85957 -1.000000 +85960 -1.000000 +85961 -1.000000 +85962 -1.000000 +85963 -1.000000 +85965 -1.000000 +85966 -1.000000 +85979 -1.000000 +85980 -1.000000 +85981 -1.000000 +85982 -1.000000 +85983 -1.000000 +85984 -1.000000 +85985 -1.000000 +85986 -1.000000 +85987 -1.000000 +85988 -1.000000 +85989 -1.000000 +85990 -1.000000 +85991 -1.000000 +85992 -1.000000 +85993 -1.000000 +85994 -1.000000 +85995 -1.000000 +85996 -1.000000 +85997 -1.000000 +85998 -1.000000 +85999 -1.000000 +86000 -1.000000 +86001 -1.000000 +86002 -1.000000 +86003 -1.000000 +86004 -1.000000 +86005 -1.000000 +86006 -1.000000 +86007 -1.000000 +86008 -1.000000 +86009 -1.000000 +86010 -1.000000 +86011 -1.000000 +86012 -1.000000 +86013 -1.000000 +86014 -1.000000 +86015 -1.000000 +86016 -1.000000 +86017 -1.000000 +86018 -1.000000 +86019 -1.000000 +86020 -1.000000 +86021 -1.000000 +86022 -1.000000 +86023 -1.000000 +86024 -1.000000 +86025 -1.000000 +86026 -1.000000 +86027 -1.000000 +86028 -1.000000 +86029 -1.000000 +86030 -1.000000 +86031 -1.000000 +86032 -1.000000 +86033 -1.000000 +86034 -1.000000 +86035 -1.000000 +86036 -1.000000 +86037 -1.000000 +86038 -1.000000 +86039 -1.000000 +86040 -1.000000 +86041 -1.000000 +86042 -1.000000 +86043 -1.000000 +86044 -1.000000 +86045 -1.000000 +86046 -1.000000 +86047 -1.000000 +86048 -1.000000 +86049 -1.000000 +86050 -1.000000 +86051 -1.000000 +86052 -1.000000 +86053 -1.000000 +86054 -1.000000 +86055 -1.000000 +86056 -1.000000 +86057 -1.000000 +86058 -1.000000 +86059 -1.000000 +86060 -1.000000 +86061 -1.000000 +86062 -1.000000 +86063 -1.000000 +86064 -1.000000 +86065 -1.000000 +86066 -1.000000 +86067 -1.000000 +86068 -1.000000 +86069 -1.000000 +86070 -1.000000 +86071 -1.000000 +86072 -1.000000 +86073 -1.000000 +86074 -1.000000 +86075 -1.000000 +86076 -1.000000 +86077 -1.000000 +86078 -1.000000 +86079 -1.000000 +86080 -1.000000 +86081 -1.000000 +86082 -1.000000 +86083 -1.000000 +86084 -1.000000 +86085 -1.000000 +86086 -1.000000 +86087 -1.000000 +86088 -1.000000 +86089 -1.000000 +86090 -1.000000 +86091 -1.000000 +86092 -1.000000 +86093 -1.000000 +86094 -1.000000 +86095 -1.000000 +86096 -1.000000 +86097 -1.000000 +86098 -1.000000 +86099 -1.000000 +86100 -1.000000 +86101 -1.000000 +86103 -1.000000 +86104 -1.000000 +86105 -1.000000 +86106 -1.000000 +86107 -1.000000 +86108 -1.000000 +86109 -1.000000 +86110 -1.000000 +86111 -1.000000 +86112 -1.000000 +86113 -1.000000 +86114 -1.000000 +86115 -1.000000 +86116 -1.000000 +86117 -1.000000 +86118 -1.000000 +86119 -1.000000 +86120 -1.000000 +86121 -1.000000 +86122 -1.000000 +86123 -1.000000 +86124 -1.000000 +86125 -1.000000 +86126 -1.000000 +86127 -1.000000 +86128 -1.000000 +86130 -1.000000 +86132 -1.000000 +86133 -1.000000 +86134 -1.000000 +86135 -1.000000 +86136 -1.000000 +86137 -1.000000 +86138 -1.000000 +86139 -1.000000 +86141 -1.000000 +86142 -1.000000 +86143 -1.000000 +86144 -1.000000 +86145 -1.000000 +86146 -1.000000 +86147 -1.000000 +86148 -1.000000 +86149 -1.000000 +86150 -1.000000 +86151 -1.000000 +86152 -1.000000 +86154 -1.000000 +86155 -1.000000 +86156 -1.000000 +86157 -1.000000 +86158 -1.000000 +86159 -1.000000 +86160 -1.000000 +86161 -1.000000 +86162 -1.000000 +86163 -1.000000 +86164 -1.000000 +86165 -1.000000 +86166 -1.000000 +86167 -1.000000 +86168 -1.000000 +86169 -1.000000 +86170 -1.000000 +86171 -1.000000 +86172 -1.000000 +86173 -1.000000 +86174 -1.000000 +86175 -1.000000 +86176 -1.000000 +86177 -1.000000 +86178 -1.000000 +86179 -1.000000 +86180 -1.000000 +86181 -1.000000 +86182 -1.000000 +86183 -1.000000 +86184 -1.000000 +86185 -1.000000 +86186 -1.000000 +86187 -1.000000 +86188 -1.000000 +86189 -1.000000 +86190 -1.000000 +86191 -1.000000 +86192 -1.000000 +86193 -1.000000 +86194 -1.000000 +86195 -1.000000 +86196 -1.000000 +86197 -1.000000 +86198 -1.000000 +86199 -1.000000 +86200 -1.000000 +86201 -1.000000 +86202 -1.000000 +86203 -1.000000 +86204 -1.000000 +86205 -1.000000 +86206 -1.000000 +86207 -1.000000 +86208 -1.000000 +86209 -1.000000 +86210 -1.000000 +86211 -1.000000 +86212 -1.000000 +86213 -1.000000 +86214 -1.000000 +86215 -1.000000 +86216 -1.000000 +86217 -1.000000 +86218 -1.000000 +86219 -1.000000 +86220 -1.000000 +86221 -1.000000 +86222 -1.000000 +86223 -1.000000 +86224 -1.000000 +86225 -1.000000 +86226 -1.000000 +86227 -1.000000 +86228 -1.000000 +86229 -1.000000 +86230 -1.000000 +86231 -1.000000 +86232 -1.000000 +86233 -1.000000 +86234 -1.000000 +86235 -1.000000 +86236 -1.000000 +86237 -1.000000 +86238 -1.000000 +86239 -1.000000 +86240 -1.000000 +86241 -1.000000 +86242 -1.000000 +86243 -1.000000 +86244 -1.000000 +86245 -1.000000 +86246 -1.000000 +86247 -1.000000 +86248 -1.000000 +86249 -1.000000 +86250 -1.000000 +86251 -1.000000 +86252 -1.000000 +86253 -1.000000 +86254 -1.000000 +86255 -1.000000 +86256 -1.000000 +86257 -1.000000 +86258 -1.000000 +86259 -1.000000 +86260 -1.000000 +86261 -1.000000 +86262 -1.000000 +86263 -1.000000 +86264 -1.000000 +86265 -1.000000 +86267 -1.000000 +86269 -1.000000 +86270 -1.000000 +86271 -1.000000 +86272 -1.000000 +86273 -1.000000 +86274 -1.000000 +86275 -1.000000 +86276 -1.000000 +86277 -1.000000 +86279 -1.000000 +86281 -1.000000 +86282 -1.000000 +86283 -1.000000 +86284 -1.000000 +86285 -1.000000 +86286 -1.000000 +86287 -1.000000 +86294 -1.000000 +86295 -1.000000 +86296 -1.000000 +86297 -1.000000 +86298 -1.000000 +86299 -1.000000 +86301 -1.000000 +86302 -1.000000 +86303 -1.000000 +86304 -1.000000 +86305 -1.000000 +86307 -1.000000 +86309 -1.000000 +86310 -1.000000 +86311 -1.000000 +86312 -1.000000 +86314 -1.000000 +86316 -1.000000 +86318 -1.000000 +86320 -1.000000 +86322 -1.000000 +86323 -1.000000 +86324 -1.000000 +86325 -1.000000 +86326 -1.000000 +86327 -1.000000 +86328 -1.000000 +86329 -1.000000 +86330 -1.000000 +86331 -1.000000 +86332 -1.000000 +86333 -1.000000 +86334 -1.000000 +86335 -1.000000 +86336 -1.000000 +86338 -1.000000 +86339 -1.000000 +86340 -1.000000 +86341 -1.000000 +86342 -1.000000 +86343 -1.000000 +86344 -1.000000 +86345 -1.000000 +86346 -1.000000 +86347 -1.000000 +86349 -1.000000 +86351 -1.000000 +86352 -1.000000 +86353 -1.000000 +86354 -1.000000 +86355 -1.000000 +86356 -1.000000 +86357 -1.000000 +86358 -1.000000 +86359 -1.000000 +86361 -1.000000 +86364 -1.000000 +86365 -1.000000 +86366 -1.000000 +86367 -1.000000 +86368 -1.000000 +86369 -1.000000 +86370 -1.000000 +86371 -1.000000 +86372 -1.000000 +86373 -1.000000 +86374 -1.000000 +86375 -1.000000 +86376 -1.000000 +86377 -1.000000 +86379 -1.000000 +86380 -1.000000 +86382 -1.000000 +86384 -1.000000 +86385 -1.000000 +86386 -1.000000 +86387 -1.000000 +86388 -1.000000 +86389 -1.000000 +86390 -1.000000 +86391 -1.000000 +86392 -1.000000 +86393 -1.000000 +86394 -1.000000 +86395 -1.000000 +86396 -1.000000 +86397 -1.000000 +86398 -1.000000 +86399 -1.000000 +86400 -1.000000 +86401 -1.000000 +86402 -1.000000 +86403 -1.000000 +86404 -1.000000 +86405 -1.000000 +86406 -1.000000 +86407 -1.000000 +86409 -1.000000 +86410 -1.000000 +86411 -1.000000 +86412 -1.000000 +86413 -1.000000 +86414 -1.000000 +86416 -1.000000 +86417 -1.000000 +86418 -1.000000 +86419 -1.000000 +86420 -1.000000 +86421 -1.000000 +86422 -1.000000 +86423 -1.000000 +86424 -1.000000 +86425 -1.000000 +86427 -1.000000 +86428 -1.000000 +86429 -1.000000 +86431 -1.000000 +86432 -1.000000 +86433 -1.000000 +86434 -1.000000 +86435 -1.000000 +86436 -1.000000 +86437 -1.000000 +86438 -1.000000 +86439 -1.000000 +86440 -1.000000 +86441 -1.000000 +86442 -1.000000 +86443 -1.000000 +86444 -1.000000 +86445 -1.000000 +86447 -1.000000 +86450 -1.000000 +86451 -1.000000 +86452 -1.000000 +86453 -1.000000 +86454 -1.000000 +86456 -1.000000 +86457 -1.000000 +86458 -1.000000 +86459 -1.000000 +86460 -1.000000 +86461 -1.000000 +86462 -1.000000 +86463 -1.000000 +86464 -1.000000 +86465 -1.000000 +86466 -1.000000 +86469 -1.000000 +86470 -1.000000 +86471 -1.000000 +86472 -1.000000 +86473 -1.000000 +86474 -1.000000 +86475 -1.000000 +86476 -1.000000 +86477 -1.000000 +86478 -1.000000 +86479 -1.000000 +86480 -1.000000 +86481 -1.000000 +86482 -1.000000 +86483 -1.000000 +86484 -1.000000 +86485 -1.000000 +86486 -1.000000 +86487 -1.000000 +86488 -1.000000 +86489 -1.000000 +86490 -1.000000 +86491 -1.000000 +86492 -1.000000 +86493 -1.000000 +86494 -1.000000 +86495 -1.000000 +86496 -1.000000 +86497 -1.000000 +86498 -1.000000 +86499 -1.000000 +86500 -1.000000 +86501 -1.000000 +86502 -1.000000 +86503 -1.000000 +86504 -1.000000 +86505 -1.000000 +86506 -1.000000 +86507 -1.000000 +86508 -1.000000 +86509 -1.000000 +86510 -1.000000 +86511 -1.000000 +86512 -1.000000 +86513 -1.000000 +86514 -1.000000 +86515 -1.000000 +86516 -1.000000 +86517 -1.000000 +86518 -1.000000 +86519 -1.000000 +86520 -1.000000 +86521 -1.000000 +86522 -1.000000 +86526 -1.000000 +86528 -1.000000 +86529 -1.000000 +86530 -1.000000 +86531 -1.000000 +86532 -1.000000 +86533 -1.000000 +86535 -1.000000 +86536 -1.000000 +86537 -1.000000 +86538 -1.000000 +86539 -1.000000 +86540 -1.000000 +86541 -1.000000 +86543 -1.000000 +86546 -1.000000 +86547 -1.000000 +86548 -1.000000 +86549 -1.000000 +86551 -1.000000 +86552 -1.000000 +86555 -1.000000 +86556 -1.000000 +86557 -1.000000 +86560 -1.000000 +86561 -1.000000 +86563 -1.000000 +86564 -1.000000 +86565 -1.000000 +86566 -1.000000 +86567 -1.000000 +86568 -1.000000 +86569 -1.000000 +86570 -1.000000 +86571 -1.000000 +86572 -1.000000 +86573 -1.000000 +86574 -1.000000 +86575 -1.000000 +86576 -1.000000 +86577 -1.000000 +86578 -1.000000 +86579 -1.000000 +86581 -1.000000 +86584 -1.000000 +86589 -1.000000 +86590 -1.000000 +86593 -1.000000 +86594 -1.000000 +86595 -1.000000 +86596 -1.000000 +86597 -1.000000 +86598 -1.000000 +86599 -1.000000 +86600 -1.000000 +86605 -1.000000 +86606 -1.000000 +86607 -1.000000 +86609 -1.000000 +86610 -1.000000 +86611 -1.000000 +86612 -1.000000 +86613 -1.000000 +86616 -1.000000 +86618 -1.000000 +86621 -1.000000 +86623 -1.000000 +86625 -1.000000 +86631 -1.000000 +86633 -1.000000 +86634 -1.000000 +86637 -1.000000 +86638 -1.000000 +86639 -1.000000 +86640 -1.000000 +86641 -1.000000 +86642 -1.000000 +86643 -1.000000 +86645 -1.000000 +86647 -1.000000 +86648 -1.000000 +86649 -1.000000 +86650 -1.000000 +86651 -1.000000 +86652 -1.000000 +86653 -1.000000 +86654 -1.000000 +86656 -1.000000 +86657 -1.000000 +86662 -1.000000 +86663 -1.000000 +86665 -1.000000 +86666 -1.000000 +86667 -1.000000 +86668 -1.000000 +86669 -1.000000 +86671 -1.000000 +86672 -1.000000 +86673 -1.000000 +86675 -1.000000 +86676 -1.000000 +86677 -1.000000 +86679 -1.000000 +86680 -1.000000 +86681 -1.000000 +86682 -1.000000 +86683 -1.000000 +86684 -1.000000 +86685 -1.000000 +86687 -1.000000 +86689 -1.000000 +86690 -1.000000 +86697 -1.000000 +86700 -1.000000 +86703 -1.000000 +86704 -1.000000 +86705 -1.000000 +86706 -1.000000 +86707 -1.000000 +86708 -1.000000 +86709 -1.000000 +86710 -1.000000 +86711 -1.000000 +86713 -1.000000 +86715 -1.000000 +86717 -1.000000 +86718 -1.000000 +86719 -1.000000 +86720 -1.000000 +86723 -1.000000 +86724 -1.000000 +86725 -1.000000 +86727 -1.000000 +86728 -1.000000 +86729 -1.000000 +86730 -1.000000 +86731 -1.000000 +86732 -1.000000 +86733 -1.000000 +86734 -1.000000 +86735 -1.000000 +86736 -1.000000 +86737 -1.000000 +86738 -1.000000 +86739 -1.000000 +86740 -1.000000 +86741 -1.000000 +86742 -1.000000 +86743 -1.000000 +86744 -1.000000 +86745 -1.000000 +86746 -1.000000 +86747 -1.000000 +86748 -1.000000 +86749 -1.000000 +86751 -1.000000 +86752 -1.000000 +86753 -1.000000 +86754 -1.000000 +86755 -1.000000 +86756 -1.000000 +86757 -1.000000 +86758 -1.000000 +86759 -1.000000 +86760 -1.000000 +86761 -1.000000 +86762 -1.000000 +86763 -1.000000 +86764 -1.000000 +86765 -1.000000 +86766 -1.000000 +86767 -1.000000 +86768 -1.000000 +86769 -1.000000 +86770 -1.000000 +86771 -1.000000 +86772 -1.000000 +86773 -1.000000 +86774 -1.000000 +86775 -1.000000 +86778 -1.000000 +86779 -1.000000 +86780 -1.000000 +86781 -1.000000 +86782 -1.000000 +86783 -1.000000 +86784 -1.000000 +86785 -1.000000 +86786 -1.000000 +86787 -1.000000 +86788 -1.000000 +86789 -1.000000 +86790 -1.000000 +86791 -1.000000 +86792 -1.000000 +86794 -1.000000 +86795 -1.000000 +86797 -1.000000 +86800 -1.000000 +86803 -1.000000 +86804 -1.000000 +86805 -1.000000 +86806 -1.000000 +86807 -1.000000 +86808 -1.000000 +86809 -1.000000 +86810 -1.000000 +86811 -1.000000 +86812 -1.000000 +86813 -1.000000 +86814 -1.000000 +86815 -1.000000 +86816 -1.000000 +86817 -1.000000 +86818 -1.000000 +86820 -1.000000 +86821 -1.000000 +86822 -1.000000 +86823 -1.000000 +86824 -1.000000 +86825 -1.000000 +86827 -1.000000 +86828 -1.000000 +86829 -1.000000 +86830 -1.000000 +86831 -1.000000 +86833 -1.000000 +86834 -1.000000 +86835 -1.000000 +86837 -1.000000 +86838 -1.000000 +86839 -1.000000 +86840 -1.000000 +86841 -1.000000 +86842 -1.000000 +86843 -1.000000 +86844 -1.000000 +86846 -1.000000 +86847 -1.000000 +86849 -1.000000 +86850 -1.000000 +86851 -1.000000 +86852 -1.000000 +86853 -1.000000 +86854 -1.000000 +86862 -1.000000 +86868 -1.000000 +86869 -1.000000 +86870 -1.000000 +86872 -1.000000 +86873 -1.000000 +86876 -1.000000 +86888 -1.000000 +86890 -1.000000 +86891 -1.000000 +86892 -1.000000 +86893 -1.000000 +86895 -1.000000 +86897 -1.000000 +86900 -1.000000 +86901 -1.000000 +86902 -1.000000 +86903 -1.000000 +86905 -1.000000 +86906 -1.000000 +86907 -1.000000 +86908 -1.000000 +86909 -1.000000 +86910 -1.000000 +86912 -1.000000 +86913 -1.000000 +86915 -1.000000 +86917 -1.000000 +86918 -1.000000 +86919 -1.000000 +86921 -1.000000 +86922 -1.000000 +86923 -1.000000 +86928 -1.000000 +86930 -1.000000 +86931 -1.000000 +86932 -1.000000 +86934 -1.000000 +86936 -1.000000 +86941 -1.000000 +86942 -1.000000 +86943 -1.000000 +86944 -1.000000 +86945 -1.000000 +86946 -1.000000 +86947 -1.000000 +86948 -1.000000 +86950 -1.000000 +86952 -1.000000 +86954 -1.000000 +86955 -1.000000 +86957 -1.000000 +86958 -1.000000 +86960 -1.000000 +86962 -1.000000 +86963 -1.000000 +86964 -1.000000 +86965 -1.000000 +86966 -1.000000 +86967 -1.000000 +86968 -1.000000 +86970 -1.000000 +86971 -1.000000 +86972 -1.000000 +86973 -1.000000 +86974 -1.000000 +86975 -1.000000 +86976 -1.000000 +86977 -1.000000 +86978 -1.000000 +86979 -1.000000 +86980 -1.000000 +86981 -1.000000 +86982 -1.000000 +86983 -1.000000 +86984 -1.000000 +86985 -1.000000 +86987 -1.000000 +86988 -1.000000 +86989 -1.000000 +86990 -1.000000 +86992 -1.000000 +86994 -1.000000 +86995 -1.000000 +86996 -1.000000 +86997 -1.000000 +86998 -1.000000 +86999 -1.000000 +87000 -1.000000 +87001 -1.000000 +87002 -1.000000 +87003 -1.000000 +87004 -1.000000 +87005 -1.000000 +87006 -1.000000 +87007 -1.000000 +87009 -1.000000 +87010 -1.000000 +87012 -1.000000 +87013 -1.000000 +87016 -1.000000 +87017 -1.000000 +87018 -1.000000 +87019 -1.000000 +87020 -1.000000 +87021 -1.000000 +87022 -1.000000 +87024 -1.000000 +87028 -1.000000 +87029 -1.000000 +87030 -1.000000 +87031 -1.000000 +87032 -1.000000 +87033 -1.000000 +87034 -1.000000 +87035 -1.000000 +87036 -1.000000 +87037 -1.000000 +87038 -1.000000 +87039 -1.000000 +87041 -1.000000 +87042 -1.000000 +87044 -1.000000 +87045 -1.000000 +87046 -1.000000 +87047 -1.000000 +87048 -1.000000 +87049 -1.000000 +87052 -1.000000 +87053 -1.000000 +87054 -1.000000 +87055 -1.000000 +87057 -1.000000 +87058 -1.000000 +87059 -1.000000 +87060 -1.000000 +87062 -1.000000 +87063 -1.000000 +87064 -1.000000 +87067 -1.000000 +87069 -1.000000 +87070 -1.000000 +87071 -1.000000 +87072 -1.000000 +87073 -1.000000 +87075 -1.000000 +87076 -1.000000 +87077 -1.000000 +87078 -1.000000 +87079 -1.000000 +87080 -1.000000 +87081 -1.000000 +87082 -1.000000 +87083 -1.000000 +87084 -1.000000 +87085 -1.000000 +87086 -1.000000 +87087 -1.000000 +87088 -1.000000 +87089 -1.000000 +87090 -1.000000 +87091 -1.000000 +87092 -1.000000 +87093 -1.000000 +87094 -1.000000 +87095 -1.000000 +87096 -1.000000 +87097 -1.000000 +87098 -1.000000 +87099 -1.000000 +87100 -1.000000 +87102 -1.000000 +87103 -1.000000 +87104 -1.000000 +87105 -1.000000 +87106 -1.000000 +87107 -1.000000 +87108 -1.000000 +87109 -1.000000 +87110 -1.000000 +87111 -1.000000 +87112 -1.000000 +87113 -1.000000 +87114 -1.000000 +87115 -1.000000 +87116 -1.000000 +87117 -1.000000 +87118 -1.000000 +87120 -1.000000 +87121 -1.000000 +87122 -1.000000 +87123 -1.000000 +87124 -1.000000 +87137 -1.000000 +87139 -1.000000 +87140 -1.000000 +87141 -1.000000 +87143 -1.000000 +87144 -1.000000 +87145 -1.000000 +87157 -1.000000 +87158 -1.000000 +87159 -1.000000 +87160 -1.000000 +87161 -1.000000 +87162 -1.000000 +87163 -1.000000 +87164 -1.000000 +87165 -1.000000 +87166 -1.000000 +87167 -1.000000 +87168 -1.000000 +87169 -1.000000 +87170 -1.000000 +87172 -1.000000 +87173 -1.000000 +87174 -1.000000 +87176 -1.000000 +87177 -1.000000 +87178 -1.000000 +87179 -1.000000 +87180 -1.000000 +87181 -1.000000 +87182 -1.000000 +87183 -1.000000 +87185 -1.000000 +87186 -1.000000 +87187 -1.000000 +87188 -1.000000 +87190 -1.000000 +87191 -1.000000 +87192 -1.000000 +87193 -1.000000 +87194 -1.000000 +87195 -1.000000 +87196 -1.000000 +87199 -1.000000 +87200 -1.000000 +87206 -1.000000 +87207 -1.000000 +87208 -1.000000 +87215 -1.000000 +87217 -1.000000 +87218 -1.000000 +87220 -1.000000 +87221 -1.000000 +87224 -1.000000 +87225 -1.000000 +87227 -1.000000 +87229 -1.000000 +87230 -1.000000 +87231 -1.000000 +87233 -1.000000 +87234 -1.000000 +87235 -1.000000 +87236 -1.000000 +87237 -1.000000 +87238 -1.000000 +87239 -1.000000 +87240 -1.000000 +87241 -1.000000 +87242 -1.000000 +87245 -1.000000 +87246 -1.000000 +87247 -1.000000 +87249 -1.000000 +87250 -1.000000 +87251 -1.000000 +87252 -1.000000 +87256 -1.000000 +87257 -1.000000 +87258 -1.000000 +87259 -1.000000 +87260 -1.000000 +87261 -1.000000 +87262 -1.000000 +87263 -1.000000 +87264 -1.000000 +87265 -1.000000 +87266 -1.000000 +87267 -1.000000 +87269 -1.000000 +87270 -1.000000 +87271 -1.000000 +87272 -1.000000 +87273 -1.000000 +87274 -1.000000 +87275 -1.000000 +87276 -1.000000 +87277 -1.000000 +87278 -1.000000 +87279 -1.000000 +87280 -1.000000 +87281 -1.000000 +87282 -1.000000 +87283 -1.000000 +87284 -1.000000 +87285 -1.000000 +87286 -1.000000 +87288 -1.000000 +87289 -1.000000 +87290 -1.000000 +87291 -1.000000 +87292 -1.000000 +87294 -1.000000 +87295 -1.000000 +87297 -1.000000 +87298 -1.000000 +87301 -1.000000 +87302 -1.000000 +87303 -1.000000 +87304 -1.000000 +87307 -1.000000 +87308 -1.000000 +87309 -1.000000 +87310 -1.000000 +87311 -1.000000 +87312 -1.000000 +87313 -1.000000 +87314 -1.000000 +87315 -1.000000 +87316 -1.000000 +87317 -1.000000 +87319 -1.000000 +87320 -1.000000 +87321 -1.000000 +87322 -1.000000 +87323 -1.000000 +87324 -1.000000 +87327 -1.000000 +87330 -1.000000 +87331 -1.000000 +87332 -1.000000 +87333 -1.000000 +87335 -1.000000 +87336 -1.000000 +87337 -1.000000 +87338 -1.000000 +87339 -1.000000 +87340 -1.000000 +87341 -1.000000 +87342 -1.000000 +87343 -1.000000 +87344 -1.000000 +87345 -1.000000 +87346 -1.000000 +87347 -1.000000 +87348 -1.000000 +87349 -1.000000 +87350 -1.000000 +87351 -1.000000 +87352 -1.000000 +87353 -1.000000 +87354 -1.000000 +87357 -1.000000 +87359 -1.000000 +87360 -1.000000 +87361 -1.000000 +87362 -1.000000 +87363 -1.000000 +87364 -1.000000 +87365 -1.000000 +87366 -1.000000 +87367 -1.000000 +87368 -1.000000 +87369 -1.000000 +87370 -1.000000 +87371 -1.000000 +87372 -1.000000 +87373 -1.000000 +87374 -1.000000 +87375 -1.000000 +87376 -1.000000 +87378 -1.000000 +87379 -1.000000 +87380 -1.000000 +87381 -1.000000 +87382 -1.000000 +87383 -1.000000 +87384 -1.000000 +87386 -1.000000 +87388 -1.000000 +87389 -1.000000 +87390 -1.000000 +87391 -1.000000 +87392 -1.000000 +87393 -1.000000 +87394 -1.000000 +87395 -1.000000 +87396 -1.000000 +87397 -1.000000 +87398 -1.000000 +87399 -1.000000 +87400 -1.000000 +87401 -1.000000 +87403 -1.000000 +87404 -1.000000 +87405 -1.000000 +87406 -1.000000 +87407 -1.000000 +87408 -1.000000 +87409 -1.000000 +87410 -1.000000 +87411 -1.000000 +87413 -1.000000 +87414 -1.000000 +87415 -1.000000 +87416 -1.000000 +87417 -1.000000 +87418 -1.000000 +87419 -1.000000 +87420 -1.000000 +87421 -1.000000 +87422 -1.000000 +87423 -1.000000 +87424 -1.000000 +87425 -1.000000 +87426 -1.000000 +87427 -1.000000 +87428 -1.000000 +87429 -1.000000 +87430 -1.000000 +87431 -1.000000 +87432 -1.000000 +87433 -1.000000 +87436 -1.000000 +87438 -1.000000 +87440 -1.000000 +87442 -1.000000 +87445 -1.000000 +87446 -1.000000 +87447 -1.000000 +87448 -1.000000 +87449 -1.000000 +87450 -1.000000 +87451 -1.000000 +87452 -1.000000 +87453 -1.000000 +87454 -1.000000 +87455 -1.000000 +87456 -1.000000 +87457 -1.000000 +87458 -1.000000 +87459 -1.000000 +87460 -1.000000 +87461 -1.000000 +87462 -1.000000 +87463 -1.000000 +87464 -1.000000 +87465 -1.000000 +87466 -1.000000 +87467 -1.000000 +87468 -1.000000 +87469 -1.000000 +87470 -1.000000 +87471 -1.000000 +87472 -1.000000 +87474 -1.000000 +87476 -1.000000 +87479 -1.000000 +87481 -1.000000 +87482 -1.000000 +87483 -1.000000 +87485 -1.000000 +87486 -1.000000 +87488 -1.000000 +87489 -1.000000 +87492 -1.000000 +87497 -1.000000 +87499 -1.000000 +87500 -1.000000 +87501 -1.000000 +87504 -1.000000 +87505 -1.000000 +87506 -1.000000 +87508 -1.000000 +87511 -1.000000 +87512 -1.000000 +87513 -1.000000 +87514 -1.000000 +87517 -1.000000 +87519 -1.000000 +87520 -1.000000 +87523 -1.000000 +87524 -1.000000 +87527 -1.000000 +87528 -1.000000 +87529 -1.000000 +87533 -1.000000 +87535 -1.000000 +87536 -1.000000 +87537 -1.000000 +87538 -1.000000 +87539 -1.000000 +87551 -1.000000 +87554 -1.000000 +87556 -1.000000 +87557 -1.000000 +87558 -1.000000 +87559 -1.000000 +87560 -1.000000 +87561 -1.000000 +87562 -1.000000 +87563 -1.000000 +87564 -1.000000 +87565 -1.000000 +87566 -1.000000 +87567 -1.000000 +87568 -1.000000 +87570 -1.000000 +87584 -1.000000 +87585 -1.000000 +87586 -1.000000 +87587 -1.000000 +87588 -1.000000 +87590 -1.000000 +87591 -1.000000 +87592 -1.000000 +87593 -1.000000 +87594 -1.000000 +87595 -1.000000 +87596 -1.000000 +87597 -1.000000 +87598 -1.000000 +87603 -1.000000 +87604 -1.000000 +87605 -1.000000 +87606 -1.000000 +87607 -1.000000 +87608 -1.000000 +87609 -1.000000 +87610 -1.000000 +87611 -1.000000 +87612 -1.000000 +87613 -1.000000 +87614 -1.000000 +87615 -1.000000 +87616 -1.000000 +87617 -1.000000 +87618 -1.000000 +87619 -1.000000 +87620 -1.000000 +87621 -1.000000 +87622 -1.000000 +87623 -1.000000 +87624 -1.000000 +87625 -1.000000 +87626 -1.000000 +87627 -1.000000 +87628 -1.000000 +87629 -1.000000 +87630 -1.000000 +87631 -1.000000 +87632 -1.000000 +87633 -1.000000 +87634 -1.000000 +87635 -1.000000 +87636 -1.000000 +87637 -1.000000 +87638 -1.000000 +87639 -1.000000 +87641 -1.000000 +87642 -1.000000 +87645 -1.000000 +87646 -1.000000 +87648 -1.000000 +87650 -1.000000 +87651 -1.000000 +87652 -1.000000 +87653 -1.000000 +87654 -1.000000 +87655 -1.000000 +87656 -1.000000 +87657 -1.000000 +87658 -1.000000 +87659 -1.000000 +87660 -1.000000 +87661 -1.000000 +87662 -1.000000 +87663 -1.000000 +87664 -1.000000 +87665 -1.000000 +87666 -1.000000 +87667 -1.000000 +87668 -1.000000 +87669 -1.000000 +87670 -1.000000 +87671 -1.000000 +87672 -1.000000 +87673 -1.000000 +87674 -1.000000 +87675 -1.000000 +87676 -1.000000 +87677 -1.000000 +87678 -1.000000 +87679 -1.000000 +87680 -1.000000 +87681 -1.000000 +87682 -1.000000 +87683 -1.000000 +87684 -1.000000 +87685 -1.000000 +87686 -1.000000 +87687 -1.000000 +87688 -1.000000 +87689 -1.000000 +87690 -1.000000 +87691 -1.000000 +87692 -1.000000 +87693 -1.000000 +87694 -1.000000 +87695 -1.000000 +87696 -1.000000 +87698 -1.000000 +87699 -1.000000 +87700 -1.000000 +87701 -1.000000 +87703 -1.000000 +87704 -1.000000 +87705 -1.000000 +87706 -1.000000 +87707 -1.000000 +87708 -1.000000 +87709 -1.000000 +87710 -1.000000 +87711 -1.000000 +87712 -1.000000 +87715 -1.000000 +87716 -1.000000 +87717 -1.000000 +87718 -1.000000 +87721 -1.000000 +87723 -1.000000 +87724 -1.000000 +87725 -1.000000 +87727 -1.000000 +87729 -1.000000 +87730 -1.000000 +87731 -1.000000 +87732 -1.000000 +87733 -1.000000 +87734 -1.000000 +87735 -1.000000 +87736 -1.000000 +87737 -1.000000 +87738 -1.000000 +87739 -1.000000 +87741 -1.000000 +87742 -1.000000 +87744 -1.000000 +87745 -1.000000 +87747 -1.000000 +87748 -1.000000 +87749 -1.000000 +87750 -1.000000 +87752 -1.000000 +87753 -1.000000 +87754 -1.000000 +87755 -1.000000 +87756 -1.000000 +87758 -1.000000 +87759 -1.000000 +87760 -1.000000 +87761 -1.000000 +87762 -1.000000 +87763 -1.000000 +87764 -1.000000 +87765 -1.000000 +87766 -1.000000 +87767 -1.000000 +87768 -1.000000 +87769 -1.000000 +87770 -1.000000 +87771 -1.000000 +87772 -1.000000 +87773 -1.000000 +87774 -1.000000 +87775 -1.000000 +87776 -1.000000 +87777 -1.000000 +87778 -1.000000 +87779 -1.000000 +87780 -1.000000 +87781 -1.000000 +87783 -1.000000 +87784 -1.000000 +87785 -1.000000 +87786 -1.000000 +87788 -1.000000 +87789 -1.000000 +87792 -1.000000 +87793 -1.000000 +87794 -1.000000 +87795 -1.000000 +87799 -1.000000 +87800 -1.000000 +87801 -1.000000 +87802 -1.000000 +87803 -1.000000 +87805 -1.000000 +87806 -1.000000 +87807 -1.000000 +87808 -1.000000 +87809 -1.000000 +87810 -1.000000 +87814 -1.000000 +87815 -1.000000 +87816 -1.000000 +87817 -1.000000 +87818 -1.000000 +87819 -1.000000 +87823 -1.000000 +87825 -1.000000 +87829 -1.000000 +87830 -1.000000 +87831 -1.000000 +87833 -1.000000 +87834 -1.000000 +87835 -1.000000 +87836 -1.000000 +87837 -1.000000 +87838 -1.000000 +87839 -1.000000 +87840 -1.000000 +87841 -1.000000 +87842 -1.000000 +87843 -1.000000 +87844 -1.000000 +87845 -1.000000 +87846 -1.000000 +87847 -1.000000 +87849 -1.000000 +87850 -1.000000 +87851 -1.000000 +87852 -1.000000 +87853 -1.000000 +87854 -1.000000 +87855 -1.000000 +87858 -1.000000 +87860 -1.000000 +87862 -1.000000 +87865 -1.000000 +87866 -1.000000 +87867 -1.000000 +87868 -1.000000 +87869 -1.000000 +87870 -1.000000 +87871 -1.000000 +87872 -1.000000 +87873 -1.000000 +87874 -1.000000 +87875 -1.000000 +87876 -1.000000 +87877 -1.000000 +87879 -1.000000 +87880 -1.000000 +87881 -1.000000 +87882 -1.000000 +87883 -1.000000 +87884 -1.000000 +87885 -1.000000 +87886 -1.000000 +87887 -1.000000 +87888 -1.000000 +87889 -1.000000 +87890 -1.000000 +87891 -1.000000 +87892 -1.000000 +87893 -1.000000 +87894 -1.000000 +87895 -1.000000 +87896 -1.000000 +87897 -1.000000 +87898 -1.000000 +87899 -1.000000 +87900 -1.000000 +87901 -1.000000 +87902 -1.000000 +87903 -1.000000 +87904 -1.000000 +87905 -1.000000 +87906 -1.000000 +87907 -1.000000 +87908 -1.000000 +87909 -1.000000 +87910 -1.000000 +87911 -1.000000 +87912 -1.000000 +87913 -1.000000 +87914 -1.000000 +87915 -1.000000 +87916 -1.000000 +87917 -1.000000 +87918 -1.000000 +87919 -1.000000 +87920 -1.000000 +87921 -1.000000 +87922 -1.000000 +87923 -1.000000 +87924 -1.000000 +87925 -1.000000 +87926 -1.000000 +87927 -1.000000 +87928 -1.000000 +87929 -1.000000 +87930 -1.000000 +87931 -1.000000 +87932 -1.000000 +87933 -1.000000 +87934 -1.000000 +87935 -1.000000 +87936 -1.000000 +87943 -1.000000 +87944 -1.000000 +87945 -1.000000 +87946 -1.000000 +87947 -1.000000 +87948 -1.000000 +87949 -1.000000 +87950 -1.000000 +87951 -1.000000 +87952 -1.000000 +87953 -1.000000 +87954 -1.000000 +87955 -1.000000 +87956 -1.000000 +87957 -1.000000 +87958 -1.000000 +87959 -1.000000 +87960 -1.000000 +87961 -1.000000 +87962 -1.000000 +87963 -1.000000 +87964 -1.000000 +87965 -1.000000 +87966 -1.000000 +87967 -1.000000 +87968 -1.000000 +87969 -1.000000 +87970 -1.000000 +87971 -1.000000 +87972 -1.000000 +87973 -1.000000 +87974 -1.000000 +87975 -1.000000 +87976 -1.000000 +87977 -1.000000 +87978 -1.000000 +87979 -1.000000 +87980 -1.000000 +87981 -1.000000 +87982 -1.000000 +87983 -1.000000 +87984 -1.000000 +87985 -1.000000 +87986 -1.000000 +87987 -1.000000 +87988 -1.000000 +87989 -1.000000 +87990 -1.000000 +87991 -1.000000 +87992 -1.000000 +87993 -1.000000 +87994 -1.000000 +87995 -1.000000 +87996 -1.000000 +87997 -1.000000 +87998 -1.000000 +87999 -1.000000 +88000 -1.000000 +88001 -1.000000 +88002 -1.000000 +88003 -1.000000 +88004 -1.000000 +88005 -1.000000 +88006 -1.000000 +88007 -1.000000 +88008 -1.000000 +88009 -1.000000 +88010 -1.000000 +88011 -1.000000 +88012 -1.000000 +88013 -1.000000 +88014 -1.000000 +88015 -1.000000 +88016 -1.000000 +88017 -1.000000 +88018 -1.000000 +88019 -1.000000 +88020 -1.000000 +88021 -1.000000 +88022 -1.000000 +88023 -1.000000 +88024 -1.000000 +88025 -1.000000 +88026 -1.000000 +88027 -1.000000 +88028 -1.000000 +88029 -1.000000 +88030 -1.000000 +88031 -1.000000 +88032 -1.000000 +88033 -1.000000 +88034 -1.000000 +88035 -1.000000 +88036 -1.000000 +88037 -1.000000 +88038 -1.000000 +88039 -1.000000 +88040 -1.000000 +88041 -1.000000 +88042 -1.000000 +88043 -1.000000 +88044 -1.000000 +88045 -1.000000 +88046 -1.000000 +88047 -1.000000 +88048 -1.000000 +88049 -1.000000 +88050 -1.000000 +88051 -1.000000 +88052 -1.000000 +88053 -1.000000 +88054 -1.000000 +88055 -1.000000 +88056 -1.000000 +88057 -1.000000 +88058 -1.000000 +88059 -1.000000 +88060 -1.000000 +88061 -1.000000 +88062 -1.000000 +88063 -1.000000 +88064 -1.000000 +88065 -1.000000 +88066 -1.000000 +88067 -1.000000 +88068 -1.000000 +88069 -1.000000 +88070 -1.000000 +88071 -1.000000 +88072 -1.000000 +88073 -1.000000 +88074 -1.000000 +88075 -1.000000 +88076 -1.000000 +88077 -1.000000 +88078 -1.000000 +88079 -1.000000 +88080 -1.000000 +88081 -1.000000 +88082 -1.000000 +88083 -1.000000 +88084 -1.000000 +88085 -1.000000 +88086 -1.000000 +88087 -1.000000 +88088 -1.000000 +88089 -1.000000 +88090 -1.000000 +88091 -1.000000 +88092 -1.000000 +88093 -1.000000 +88094 -1.000000 +88095 -1.000000 +88096 -1.000000 +88097 -1.000000 +88099 -1.000000 +88100 -1.000000 +88101 -1.000000 +88102 -1.000000 +88103 -1.000000 +88104 -1.000000 +88105 -1.000000 +88106 -1.000000 +88107 -1.000000 +88108 -1.000000 +88109 -1.000000 +88110 -1.000000 +88111 -1.000000 +88112 -1.000000 +88113 -1.000000 +88114 -1.000000 +88115 -1.000000 +88116 -1.000000 +88117 -1.000000 +88118 -1.000000 +88119 -1.000000 +88120 -1.000000 +88121 -1.000000 +88122 -1.000000 +88123 -1.000000 +88124 -1.000000 +88125 -1.000000 +88126 -1.000000 +88127 -1.000000 +88128 -1.000000 +88129 -1.000000 +88130 -1.000000 +88131 -1.000000 +88132 -1.000000 +88133 -1.000000 +88134 -1.000000 +88135 -1.000000 +88136 -1.000000 +88137 -1.000000 +88138 -1.000000 +88139 -1.000000 +88140 -1.000000 +88141 -1.000000 +88142 -1.000000 +88143 -1.000000 +88144 -1.000000 +88145 -1.000000 +88146 -1.000000 +88147 -1.000000 +88148 -1.000000 +88149 -1.000000 +88150 -1.000000 +88151 -1.000000 +88152 -1.000000 +88153 -1.000000 +88154 -1.000000 +88155 -1.000000 +88156 -1.000000 +88157 -1.000000 +88158 -1.000000 +88159 -1.000000 +88160 -1.000000 +88161 -1.000000 +88162 -1.000000 +88163 -1.000000 +88164 -1.000000 +88165 -1.000000 +88166 -1.000000 +88167 -1.000000 +88168 -1.000000 +88169 -1.000000 +88170 -1.000000 +88171 -1.000000 +88172 -1.000000 +88173 -1.000000 +88174 -1.000000 +88175 -1.000000 +88176 -1.000000 +88177 -1.000000 +88178 -1.000000 +88179 -1.000000 +88180 -1.000000 +88181 -1.000000 +88183 -1.000000 +88184 -1.000000 +88185 -1.000000 +88186 -1.000000 +88187 -1.000000 +88188 -1.000000 +88193 -1.000000 +88194 -1.000000 +88195 -1.000000 +88196 -1.000000 +88198 -1.000000 +88199 -1.000000 +88200 -1.000000 +88201 -1.000000 +88202 -1.000000 +88203 -1.000000 +88204 -1.000000 +88205 -1.000000 +88206 -1.000000 +88207 -1.000000 +88208 -1.000000 +88209 -1.000000 +88210 -1.000000 +88211 -1.000000 +88212 -1.000000 +88213 -1.000000 +88214 -1.000000 +88215 -1.000000 +88216 -1.000000 +88217 -1.000000 +88218 -1.000000 +88219 -1.000000 +88221 -1.000000 +88222 -1.000000 +88223 -1.000000 +88224 -1.000000 +88225 -1.000000 +88226 -1.000000 +88227 -1.000000 +88228 -1.000000 +88229 -1.000000 +88230 -1.000000 +88231 -1.000000 +88232 -1.000000 +88233 -1.000000 +88234 -1.000000 +88235 -1.000000 +88236 -1.000000 +88237 -1.000000 +88240 -1.000000 +88241 -1.000000 +88242 -1.000000 +88243 -1.000000 +88244 -1.000000 +88245 -1.000000 +88246 -1.000000 +88247 -1.000000 +88248 -1.000000 +88249 -1.000000 +88250 -1.000000 +88251 -1.000000 +88252 -1.000000 +88253 -1.000000 +88254 -1.000000 +88255 -1.000000 +88256 -1.000000 +88257 -1.000000 +88259 -1.000000 +88260 -1.000000 +88261 -1.000000 +88262 -1.000000 +88263 -1.000000 +88264 -1.000000 +88265 -1.000000 +88266 -1.000000 +88267 -1.000000 +88268 -1.000000 +88269 -1.000000 +88270 -1.000000 +88271 -1.000000 +88272 -1.000000 +88273 -1.000000 +88274 -1.000000 +88275 -1.000000 +88276 -1.000000 +88277 -1.000000 +88278 -1.000000 +88279 -1.000000 +88280 -1.000000 +88281 -1.000000 +88282 -1.000000 +88283 -1.000000 +88284 -1.000000 +88285 -1.000000 +88286 -1.000000 +88287 -1.000000 +88288 -1.000000 +88289 -1.000000 +88290 -1.000000 +88291 -1.000000 +88292 -1.000000 +88293 -1.000000 +88294 -1.000000 +88295 -1.000000 +88296 -1.000000 +88297 -1.000000 +88298 -1.000000 +88299 -1.000000 +88300 -1.000000 +88301 -1.000000 +88302 -1.000000 +88303 -1.000000 +88304 -1.000000 +88305 -1.000000 +88306 -1.000000 +88307 -1.000000 +88308 -1.000000 +88309 -1.000000 +88310 -1.000000 +88311 -1.000000 +88312 -1.000000 +88313 -1.000000 +88314 -1.000000 +88315 -1.000000 +88316 -1.000000 +88317 -1.000000 +88318 -1.000000 +88319 -1.000000 +88320 -1.000000 +88321 -1.000000 +88322 -1.000000 +88323 -1.000000 +88324 -1.000000 +88325 -1.000000 +88326 -1.000000 +88327 -1.000000 +88328 -1.000000 +88329 -1.000000 +88332 -1.000000 +88333 -1.000000 +88334 -1.000000 +88335 -1.000000 +88336 -1.000000 +88337 -1.000000 +88338 -1.000000 +88339 -1.000000 +88340 -1.000000 +88341 -1.000000 +88342 -1.000000 +88343 -1.000000 +88344 -1.000000 +88345 -1.000000 +88346 -1.000000 +88347 -1.000000 +88348 -1.000000 +88349 -1.000000 +88350 -1.000000 +88351 -1.000000 +88352 -1.000000 +88353 -1.000000 +88354 -1.000000 +88355 -1.000000 +88356 -1.000000 +88357 -1.000000 +88358 -1.000000 +88359 -1.000000 +88360 -1.000000 +88361 -1.000000 +88362 -1.000000 +88363 -1.000000 +88364 -1.000000 +88365 -1.000000 +88366 -1.000000 +88368 -1.000000 +88369 -1.000000 +88370 -1.000000 +88371 -1.000000 +88372 -1.000000 +88373 -1.000000 +88374 -1.000000 +88375 -1.000000 +88376 -1.000000 +88377 -1.000000 +88378 -1.000000 +88379 -1.000000 +88380 -1.000000 +88381 -1.000000 +88382 -1.000000 +88383 -1.000000 +88384 -1.000000 +88385 -1.000000 +88386 -1.000000 +88387 -1.000000 +88388 -1.000000 +88389 -1.000000 +88391 -1.000000 +88392 -1.000000 +88393 -1.000000 +88394 -1.000000 +88395 -1.000000 +88396 -1.000000 +88397 -1.000000 +88398 -1.000000 +88399 -1.000000 +88400 -1.000000 +88401 -1.000000 +88402 -1.000000 +88403 -1.000000 +88404 -1.000000 +88405 -1.000000 +88406 -1.000000 +88407 -1.000000 +88408 -1.000000 +88409 -1.000000 +88410 -1.000000 +88411 -1.000000 +88412 -1.000000 +88413 -1.000000 +88414 -1.000000 +88415 -1.000000 +88416 -1.000000 +88417 -1.000000 +88418 -1.000000 +88419 -1.000000 +88421 -1.000000 +88422 -1.000000 +88423 -1.000000 +88424 -1.000000 +88425 -1.000000 +88426 -1.000000 +88427 -1.000000 +88428 -1.000000 +88429 -1.000000 +88430 -1.000000 +88431 -1.000000 +88432 -1.000000 +88433 -1.000000 +88434 -1.000000 +88435 -1.000000 +88436 -1.000000 +88437 -1.000000 +88438 -1.000000 +88439 -1.000000 +88440 -1.000000 +88441 -1.000000 +88442 -1.000000 +88443 -1.000000 +88444 -1.000000 +88445 -1.000000 +88446 -1.000000 +88447 -1.000000 +88448 -1.000000 +88449 -1.000000 +88450 -1.000000 +88451 -1.000000 +88452 -1.000000 +88453 -1.000000 +88454 -1.000000 +88455 -1.000000 +88456 -1.000000 +88457 -1.000000 +88458 -1.000000 +88459 -1.000000 +88460 -1.000000 +88461 -1.000000 +88462 -1.000000 +88463 -1.000000 +88464 -1.000000 +88465 -1.000000 +88466 -1.000000 +88467 -1.000000 +88468 -1.000000 +88469 -1.000000 +88472 -1.000000 +88473 -1.000000 +88474 -1.000000 +88475 -1.000000 +88476 -1.000000 +88477 -1.000000 +88478 -1.000000 +88479 -1.000000 +88480 -1.000000 +88481 -1.000000 +88482 -1.000000 +88483 -1.000000 +88485 -1.000000 +88486 -1.000000 +88487 -1.000000 +88488 -1.000000 +88489 -1.000000 +88490 -1.000000 +88491 -1.000000 +88493 -1.000000 +88495 -1.000000 +88496 -1.000000 +88497 -1.000000 +88500 -1.000000 +88501 -1.000000 +88502 -1.000000 +88503 -1.000000 +88504 -1.000000 +88505 -1.000000 +88506 -1.000000 +88507 -1.000000 +88508 -1.000000 +88509 -1.000000 +88510 -1.000000 +88512 -1.000000 +88513 -1.000000 +88514 -1.000000 +88515 -1.000000 +88516 -1.000000 +88517 -1.000000 +88518 -1.000000 +88519 -1.000000 +88521 -1.000000 +88524 -1.000000 +88525 -1.000000 +88526 -1.000000 +88527 -1.000000 +88528 -1.000000 +88529 -1.000000 +88530 -1.000000 +88531 -1.000000 +88532 -1.000000 +88533 -1.000000 +88534 -1.000000 +88536 -1.000000 +88537 -1.000000 +88538 -1.000000 +88539 -1.000000 +88540 -1.000000 +88541 -1.000000 +88542 -1.000000 +88543 -1.000000 +88544 -1.000000 +88545 -1.000000 +88546 -1.000000 +88548 -1.000000 +88549 -1.000000 +88551 -1.000000 +88552 -1.000000 +88553 -1.000000 +88554 -1.000000 +88555 -1.000000 +88556 -1.000000 +88557 -1.000000 +88558 -1.000000 +88559 -1.000000 +88560 -1.000000 +88561 -1.000000 +88562 -1.000000 +88563 -1.000000 +88564 -1.000000 +88565 -1.000000 +88566 -1.000000 +88567 -1.000000 +88568 -1.000000 +88569 -1.000000 +88570 -1.000000 +88571 -1.000000 +88572 -1.000000 +88573 -1.000000 +88574 -1.000000 +88575 -1.000000 +88576 -1.000000 +88578 -1.000000 +88579 -1.000000 +88580 -1.000000 +88581 -1.000000 +88582 -1.000000 +88583 -1.000000 +88584 -1.000000 +88585 -1.000000 +88586 -1.000000 +88587 -1.000000 +88588 -1.000000 +88589 -1.000000 +88590 -1.000000 +88591 -1.000000 +88592 -1.000000 +88593 -1.000000 +88594 -1.000000 +88595 -1.000000 +88596 -1.000000 +88597 -1.000000 +88598 -1.000000 +88599 -1.000000 +88600 -1.000000 +88601 -1.000000 +88602 -1.000000 +88603 -1.000000 +88604 -1.000000 +88605 -1.000000 +88606 -1.000000 +88607 -1.000000 +88608 -1.000000 +88609 -1.000000 +88611 -1.000000 +88612 -1.000000 +88614 -1.000000 +88615 -1.000000 +88616 -1.000000 +88617 -1.000000 +88618 -1.000000 +88619 -1.000000 +88620 -1.000000 +88621 -1.000000 +88622 -1.000000 +88623 -1.000000 +88624 -1.000000 +88625 -1.000000 +88626 -1.000000 +88628 -1.000000 +88629 -1.000000 +88630 -1.000000 +88631 -1.000000 +88633 -1.000000 +88634 -1.000000 +88635 -1.000000 +88636 -1.000000 +88637 -1.000000 +88638 -1.000000 +88639 -1.000000 +88640 -1.000000 +88641 -1.000000 +88642 -1.000000 +88643 -1.000000 +88644 -1.000000 +88645 -1.000000 +88646 -1.000000 +88647 -1.000000 +88648 -1.000000 +88649 -1.000000 +88650 -1.000000 +88651 -1.000000 +88652 -1.000000 +88653 -1.000000 +88654 -1.000000 +88656 -1.000000 +88657 -1.000000 +88658 -1.000000 +88659 -1.000000 +88660 -1.000000 +88661 -1.000000 +88662 -1.000000 +88663 -1.000000 +88665 -1.000000 +88666 -1.000000 +88669 -1.000000 +88670 -1.000000 +88671 -1.000000 +88672 -1.000000 +88673 -1.000000 +88674 -1.000000 +88675 -1.000000 +88676 -1.000000 +88677 -1.000000 +88679 -1.000000 +88680 -1.000000 +88681 -1.000000 +88682 -1.000000 +88685 -1.000000 +88686 -1.000000 +88688 -1.000000 +88689 -1.000000 +88690 -1.000000 +88691 -1.000000 +88693 -1.000000 +88695 -1.000000 +88697 -1.000000 +88699 -1.000000 +88701 -1.000000 +88705 -1.000000 +88706 -1.000000 +88707 -1.000000 +88708 -1.000000 +88712 -1.000000 +88713 -1.000000 +88714 -1.000000 +88715 -1.000000 +88716 -1.000000 +88717 -1.000000 +88718 -1.000000 +88719 -1.000000 +88720 -1.000000 +88721 -1.000000 +88722 -1.000000 +88723 -1.000000 +88724 -1.000000 +88725 -1.000000 +88726 -1.000000 +88727 -1.000000 +88728 -1.000000 +88729 -1.000000 +88730 -1.000000 +88731 -1.000000 +88732 -1.000000 +88734 -1.000000 +88735 -1.000000 +88737 -1.000000 +88738 -1.000000 +88739 -1.000000 +88740 -1.000000 +88741 -1.000000 +88742 -1.000000 +88743 -1.000000 +88744 -1.000000 +88746 -1.000000 +88748 -1.000000 +88749 -1.000000 +88750 -1.000000 +88752 -1.000000 +88753 -1.000000 +88754 -1.000000 +88755 -1.000000 +88756 -1.000000 +88757 -1.000000 +88758 -1.000000 +88759 -1.000000 +88760 -1.000000 +88761 -1.000000 +88762 -1.000000 +88763 -1.000000 +88764 -1.000000 +88765 -1.000000 +88766 -1.000000 +88767 -1.000000 +88768 -1.000000 +88769 -1.000000 +88770 -1.000000 +88771 -1.000000 +88772 -1.000000 +88773 -1.000000 +88774 -1.000000 +88775 -1.000000 +88776 -1.000000 +88777 -1.000000 +88778 -1.000000 +88779 -1.000000 +88780 -1.000000 +88781 -1.000000 +88782 -1.000000 +88783 -1.000000 +88784 -1.000000 +88785 -1.000000 +88786 -1.000000 +88787 -1.000000 +88788 -1.000000 +88789 -1.000000 +88790 -1.000000 +88791 -1.000000 +88792 -1.000000 +88793 -1.000000 +88794 -1.000000 +88795 -1.000000 +88796 -1.000000 +88797 -1.000000 +88798 -1.000000 +88799 -1.000000 +88800 -1.000000 +88801 -1.000000 +88802 -1.000000 +88803 -1.000000 +88804 -1.000000 +88805 -1.000000 +88806 -1.000000 +88807 -1.000000 +88808 -1.000000 +88809 -1.000000 +88810 -1.000000 +88811 -1.000000 +88812 -1.000000 +88814 -1.000000 +88815 -1.000000 +88816 -1.000000 +88817 -1.000000 +88818 -1.000000 +88819 -1.000000 +88820 -1.000000 +88821 -1.000000 +88822 -1.000000 +88823 -1.000000 +88824 -1.000000 +88826 -1.000000 +88827 -1.000000 +88828 -1.000000 +88829 -1.000000 +88830 -1.000000 +88831 -1.000000 +88832 -1.000000 +88833 -1.000000 +88834 -1.000000 +88835 -1.000000 +88836 -1.000000 +88837 -1.000000 +88838 -1.000000 +88839 -1.000000 +88840 -1.000000 +88841 -1.000000 +88842 -1.000000 +88843 -1.000000 +88844 -1.000000 +88845 -1.000000 +88847 -1.000000 +88848 -1.000000 +88850 -1.000000 +88851 -1.000000 +88852 -1.000000 +88853 -1.000000 +88854 -1.000000 +88855 -1.000000 +88856 -1.000000 +88858 -1.000000 +88859 -1.000000 +88860 -1.000000 +88861 -1.000000 +88862 -1.000000 +88863 -1.000000 +88864 -1.000000 +88865 -1.000000 +88866 -1.000000 +88867 -1.000000 +88868 -1.000000 +88869 -1.000000 +88870 -1.000000 +88871 -1.000000 +88872 -1.000000 +88873 -1.000000 +88874 -1.000000 +88876 -1.000000 +88877 -1.000000 +88881 -1.000000 +88882 -1.000000 +88883 -1.000000 +88884 -1.000000 +88889 -1.000000 +88890 -1.000000 +88891 -1.000000 +88892 -1.000000 +88893 -1.000000 +88895 -1.000000 +88896 -1.000000 +88897 -1.000000 +88898 -1.000000 +88899 -1.000000 +88900 -1.000000 +88901 -1.000000 +88902 -1.000000 +88903 -1.000000 +88904 -1.000000 +88905 -1.000000 +88906 -1.000000 +88907 -1.000000 +88909 -1.000000 +88910 -1.000000 +88911 -1.000000 +88912 -1.000000 +88913 -1.000000 +88914 -1.000000 +88915 -1.000000 +88916 -1.000000 +88917 -1.000000 +88918 -1.000000 +88919 -1.000000 +88920 -1.000000 +88921 -1.000000 +88922 -1.000000 +88923 -1.000000 +88924 -1.000000 +88925 -1.000000 +88926 -1.000000 +88927 -1.000000 +88928 -1.000000 +88929 -1.000000 +88930 -1.000000 +88931 -1.000000 +88932 -1.000000 +88933 -1.000000 +88934 -1.000000 +88935 -1.000000 +88936 -1.000000 +88937 -1.000000 +88938 -1.000000 +88939 -1.000000 +88940 -1.000000 +88941 -1.000000 +88942 -1.000000 +88943 -1.000000 +88944 -1.000000 +88945 -1.000000 +88946 -1.000000 +88947 -1.000000 +88948 -1.000000 +88949 -1.000000 +88950 -1.000000 +88951 -1.000000 +88952 -1.000000 +88953 -1.000000 +88954 -1.000000 +88955 -1.000000 +88956 -1.000000 +88957 -1.000000 +88958 -1.000000 +88959 -1.000000 +88960 -1.000000 +88961 -1.000000 +88962 -1.000000 +88963 -1.000000 +88964 -1.000000 +88965 -1.000000 +88966 -1.000000 +88967 -1.000000 +88968 -1.000000 +88969 -1.000000 +88970 -1.000000 +88972 -1.000000 +88973 -1.000000 +88974 -1.000000 +88975 -1.000000 +88976 -1.000000 +88977 -1.000000 +88978 -1.000000 +88979 -1.000000 +88980 -1.000000 +88981 -1.000000 +88982 -1.000000 +88983 -1.000000 +88984 -1.000000 +88985 -1.000000 +88986 -1.000000 +88987 -1.000000 +88988 -1.000000 +88989 -1.000000 +88992 -1.000000 +88993 -1.000000 +88995 -1.000000 +88996 -1.000000 +88997 -1.000000 +88998 -1.000000 +88999 -1.000000 +89000 -1.000000 +89001 -1.000000 +89002 -1.000000 +89003 -1.000000 +89004 -1.000000 +89005 -1.000000 +89006 -1.000000 +89007 -1.000000 +89008 -1.000000 +89012 -1.000000 +89014 -1.000000 +89018 -1.000000 +89019 -1.000000 +89020 -1.000000 +89021 -1.000000 +89022 -1.000000 +89023 -1.000000 +89024 -1.000000 +89025 -1.000000 +89026 -1.000000 +89027 -1.000000 +89028 -1.000000 +89029 -1.000000 +89030 -1.000000 +89031 -1.000000 +89032 -1.000000 +89033 -1.000000 +89034 -1.000000 +89035 -1.000000 +89036 -1.000000 +89037 -1.000000 +89038 -1.000000 +89039 -1.000000 +89040 -1.000000 +89042 -1.000000 +89043 -1.000000 +89044 -1.000000 +89045 -1.000000 +89046 -1.000000 +89047 -1.000000 +89048 -1.000000 +89049 -1.000000 +89050 -1.000000 +89052 -1.000000 +89054 -1.000000 +89055 -1.000000 +89056 -1.000000 +89057 -1.000000 +89058 -1.000000 +89059 -1.000000 +89060 -1.000000 +89061 -1.000000 +89062 -1.000000 +89063 -1.000000 +89066 -1.000000 +89067 -1.000000 +89072 -1.000000 +89073 -1.000000 +89074 -1.000000 +89078 -1.000000 +89079 -1.000000 +89080 -1.000000 +89081 -1.000000 +89082 -1.000000 +89085 -1.000000 +89086 -1.000000 +89087 -1.000000 +89089 -1.000000 +89090 -1.000000 +89091 -1.000000 +89092 -1.000000 +89093 -1.000000 +89094 -1.000000 +89095 -1.000000 +89096 -1.000000 +89097 -1.000000 +89098 -1.000000 +89099 -1.000000 +89100 -1.000000 +89101 -1.000000 +89102 -1.000000 +89103 -1.000000 +89104 -1.000000 +89105 -1.000000 +89106 -1.000000 +89107 -1.000000 +89109 -1.000000 +89110 -1.000000 +89113 -1.000000 +89114 -1.000000 +89115 -1.000000 +89116 -1.000000 +89117 -1.000000 +89118 -1.000000 +89119 -1.000000 +89120 -1.000000 +89121 -1.000000 +89122 -1.000000 +89123 -1.000000 +89124 -1.000000 +89125 -1.000000 +89126 -1.000000 +89128 -1.000000 +89129 -1.000000 +89131 -1.000000 +89132 -1.000000 +89133 -1.000000 +89134 -1.000000 +89135 -1.000000 +89136 -1.000000 +89137 -1.000000 +89138 -1.000000 +89139 -1.000000 +89140 -1.000000 +89141 -1.000000 +89143 -1.000000 +89144 -1.000000 +89145 -1.000000 +89146 -1.000000 +89147 -1.000000 +89148 -1.000000 +89150 -1.000000 +89151 -1.000000 +89152 -1.000000 +89153 -1.000000 +89154 -1.000000 +89155 -1.000000 +89156 -1.000000 +89157 -1.000000 +89158 -1.000000 +89159 -1.000000 +89161 -1.000000 +89162 -1.000000 +89163 -1.000000 +89164 -1.000000 +89165 -1.000000 +89166 -1.000000 +89167 -1.000000 +89168 -1.000000 +89169 -1.000000 +89170 -1.000000 +89171 -1.000000 +89173 -1.000000 +89175 -1.000000 +89176 -1.000000 +89177 -1.000000 +89178 -1.000000 +89179 -1.000000 +89181 -1.000000 +89184 -1.000000 +89188 -1.000000 +89189 -1.000000 +89190 -1.000000 +89193 -1.000000 +89196 -1.000000 +89197 -1.000000 +89198 -1.000000 +89199 -1.000000 +89200 -1.000000 +89201 -1.000000 +89202 -1.000000 +89203 -1.000000 +89204 -1.000000 +89205 -1.000000 +89206 -1.000000 +89207 -1.000000 +89208 -1.000000 +89209 -1.000000 +89215 -1.000000 +89216 -1.000000 +89217 -1.000000 +89218 -1.000000 +89220 -1.000000 +89221 -1.000000 +89222 -1.000000 +89224 -1.000000 +89225 -1.000000 +89227 -1.000000 +89228 -1.000000 +89229 -1.000000 +89230 -1.000000 +89231 -1.000000 +89232 -1.000000 +89233 -1.000000 +89234 -1.000000 +89235 -1.000000 +89236 -1.000000 +89237 -1.000000 +89238 -1.000000 +89240 -1.000000 +89241 -1.000000 +89242 -1.000000 +89243 -1.000000 +89244 -1.000000 +89246 -1.000000 +89249 -1.000000 +89250 -1.000000 +89251 -1.000000 +89252 -1.000000 +89253 -1.000000 +89254 -1.000000 +89255 -1.000000 +89256 -1.000000 +89257 -1.000000 +89259 -1.000000 +89260 -1.000000 +89263 -1.000000 +89264 -1.000000 +89266 -1.000000 +89267 -1.000000 +89268 -1.000000 +89269 -1.000000 +89270 -1.000000 +89271 -1.000000 +89272 -1.000000 +89273 -1.000000 +89276 -1.000000 +89277 -1.000000 +89278 -1.000000 +89279 -1.000000 +89280 -1.000000 +89281 -1.000000 +89282 -1.000000 +89283 -1.000000 +89284 -1.000000 +89285 -1.000000 +89286 -1.000000 +89287 -1.000000 +89289 -1.000000 +89290 -1.000000 +89292 -1.000000 +89293 -1.000000 +89294 -1.000000 +89295 -1.000000 +89296 -1.000000 +89297 -1.000000 +89298 -1.000000 +89299 -1.000000 +89300 -1.000000 +89301 -1.000000 +89302 -1.000000 +89303 -1.000000 +89304 -1.000000 +89305 -1.000000 +89306 -1.000000 +89307 -1.000000 +89308 -1.000000 +89309 -1.000000 +89310 -1.000000 +89311 -1.000000 +89312 -1.000000 +89313 -1.000000 +89314 -1.000000 +89315 -1.000000 +89316 -1.000000 +89318 -1.000000 +89322 -1.000000 +89323 -1.000000 +89324 -1.000000 +89326 -1.000000 +89327 -1.000000 +89328 -1.000000 +89329 -1.000000 +89330 -1.000000 +89331 -1.000000 +89332 -1.000000 +89334 -1.000000 +89335 -1.000000 +89336 -1.000000 +89337 -1.000000 +89339 -1.000000 +89341 -1.000000 +89342 -1.000000 +89343 -1.000000 +89344 -1.000000 +89345 -1.000000 +89346 -1.000000 +89347 -1.000000 +89348 -1.000000 +89349 -1.000000 +89350 -1.000000 +89351 -1.000000 +89352 -1.000000 +89353 -1.000000 +89354 -1.000000 +89355 -1.000000 +89356 -1.000000 +89357 -1.000000 +89358 -1.000000 +89359 -1.000000 +89360 -1.000000 +89361 -1.000000 +89362 -1.000000 +89363 -1.000000 +89364 -1.000000 +89365 -1.000000 +89367 -1.000000 +89368 -1.000000 +89369 -1.000000 +89370 -1.000000 +89372 -1.000000 +89374 -1.000000 +89375 -1.000000 +89376 -1.000000 +89377 -1.000000 +89378 -1.000000 +89379 -1.000000 +89380 -1.000000 +89381 -1.000000 +89382 -1.000000 +89383 -1.000000 +89384 -1.000000 +89385 -1.000000 +89386 -1.000000 +89387 -1.000000 +89388 -1.000000 +89389 -1.000000 +89391 -1.000000 +89392 -1.000000 +89393 -1.000000 +89394 -1.000000 +89395 -1.000000 +89396 -1.000000 +89397 -1.000000 +89398 -1.000000 +89399 -1.000000 +89400 -1.000000 +89401 -1.000000 +89402 -1.000000 +89403 -1.000000 +89404 -1.000000 +89405 -1.000000 +89406 -1.000000 +89407 -1.000000 +89408 -1.000000 +89409 -1.000000 +89410 -1.000000 +89411 -1.000000 +89412 -1.000000 +89413 -1.000000 +89414 -1.000000 +89415 -1.000000 +89416 -1.000000 +89417 -1.000000 +89419 -1.000000 +89423 -1.000000 +89424 -1.000000 +89426 -1.000000 +89427 -1.000000 +89428 -1.000000 +89433 -1.000000 +89435 -1.000000 +89442 -1.000000 +89443 -1.000000 +89444 -1.000000 +89445 -1.000000 +89446 -1.000000 +89447 -1.000000 +89449 -1.000000 +89452 -1.000000 +89454 -1.000000 +89455 -1.000000 +89459 -1.000000 +89460 -1.000000 +89461 -1.000000 +89462 -1.000000 +89464 -1.000000 +89465 -1.000000 +89467 -1.000000 +89468 -1.000000 +89469 -1.000000 +89471 -1.000000 +89473 -1.000000 +89474 -1.000000 +89475 -1.000000 +89476 -1.000000 +89477 -1.000000 +89478 -1.000000 +89479 -1.000000 +89480 -1.000000 +89481 -1.000000 +89482 -1.000000 +89483 -1.000000 +89484 -1.000000 +89485 -1.000000 +89486 -1.000000 +89487 -1.000000 +89488 -1.000000 +89489 -1.000000 +89490 -1.000000 +89491 -1.000000 +89492 -1.000000 +89493 -1.000000 +89494 -1.000000 +89495 -1.000000 +89496 -1.000000 +89497 -1.000000 +89498 -1.000000 +89499 -1.000000 +89500 -1.000000 +89501 -1.000000 +89502 -1.000000 +89503 -1.000000 +89504 -1.000000 +89505 -1.000000 +89506 -1.000000 +89507 -1.000000 +89508 -1.000000 +89509 -1.000000 +89510 -1.000000 +89511 -1.000000 +89512 -1.000000 +89513 -1.000000 +89514 -1.000000 +89516 -1.000000 +89517 -1.000000 +89518 -1.000000 +89520 -1.000000 +89521 -1.000000 +89522 -1.000000 +89523 -1.000000 +89524 -1.000000 +89525 -1.000000 +89526 -1.000000 +89527 -1.000000 +89528 -1.000000 +89529 -1.000000 +89530 -1.000000 +89531 -1.000000 +89532 -1.000000 +89533 -1.000000 +89534 -1.000000 +89535 -1.000000 +89536 -1.000000 +89537 -1.000000 +89538 -1.000000 +89539 -1.000000 +89540 -1.000000 +89541 -1.000000 +89542 -1.000000 +89543 -1.000000 +89544 -1.000000 +89545 -1.000000 +89546 -1.000000 +89547 -1.000000 +89548 -1.000000 +89549 -1.000000 +89550 -1.000000 +89551 -1.000000 +89552 -1.000000 +89553 -1.000000 +89554 -1.000000 +89555 -1.000000 +89556 -1.000000 +89557 -1.000000 +89558 -1.000000 +89559 -1.000000 +89560 -1.000000 +89561 -1.000000 +89562 -1.000000 +89563 -1.000000 +89564 -1.000000 +89565 -1.000000 +89566 -1.000000 +89567 -1.000000 +89568 -1.000000 +89569 -1.000000 +89570 -1.000000 +89571 -1.000000 +89572 -1.000000 +89573 -1.000000 +89574 -1.000000 +89575 -1.000000 +89576 -1.000000 +89577 -1.000000 +89578 -1.000000 +89579 -1.000000 +89580 -1.000000 +89581 -1.000000 +89582 -1.000000 +89583 -1.000000 +89584 -1.000000 +89585 -1.000000 +89586 -1.000000 +89587 -1.000000 +89588 -1.000000 +89589 -1.000000 +89590 -1.000000 +89591 -1.000000 +89592 -1.000000 +89593 -1.000000 +89594 -1.000000 +89595 -1.000000 +89597 -1.000000 +89598 -1.000000 +89599 -1.000000 +89601 -1.000000 +89602 -1.000000 +89605 -1.000000 +89606 -1.000000 +89608 -1.000000 +89609 -1.000000 +89611 -1.000000 +89612 -1.000000 +89613 -1.000000 +89614 -1.000000 +89616 -1.000000 +89617 -1.000000 +89618 -1.000000 +89619 -1.000000 +89620 -1.000000 +89621 -1.000000 +89622 -1.000000 +89623 -1.000000 +89624 -1.000000 +89625 -1.000000 +89626 -1.000000 +89627 -1.000000 +89628 -1.000000 +89629 -1.000000 +89630 -1.000000 +89632 -1.000000 +89633 -1.000000 +89634 -1.000000 +89636 -1.000000 +89637 -1.000000 +89638 -1.000000 +89639 -1.000000 +89641 -1.000000 +89642 -1.000000 +89643 -1.000000 +89644 -1.000000 +89645 -1.000000 +89646 -1.000000 +89647 -1.000000 +89648 -1.000000 +89649 -1.000000 +89650 -1.000000 +89651 -1.000000 +89652 -1.000000 +89653 -1.000000 +89654 -1.000000 +89655 -1.000000 +89656 -1.000000 +89658 -1.000000 +89661 -1.000000 +89662 -1.000000 +89663 -1.000000 +89664 -1.000000 +89665 -1.000000 +89666 -1.000000 +89669 -1.000000 +89670 -1.000000 +89671 -1.000000 +89672 -1.000000 +89673 -1.000000 +89674 -1.000000 +89675 -1.000000 +89676 -1.000000 +89677 -1.000000 +89678 -1.000000 +89679 -1.000000 +89680 -1.000000 +89681 -1.000000 +89682 -1.000000 +89683 -1.000000 +89684 -1.000000 +89685 -1.000000 +89686 -1.000000 +89687 -1.000000 +89688 -1.000000 +89689 -1.000000 +89690 -1.000000 +89691 -1.000000 +89692 -1.000000 +89693 -1.000000 +89694 -1.000000 +89695 -1.000000 +89696 -1.000000 +89697 -1.000000 +89698 -1.000000 +89699 -1.000000 +89700 -1.000000 +89701 -1.000000 +89702 -1.000000 +89703 -1.000000 +89704 -1.000000 +89705 -1.000000 +89707 -1.000000 +89708 -1.000000 +89709 -1.000000 +89710 -1.000000 +89711 -1.000000 +89712 -1.000000 +89713 -1.000000 +89714 -1.000000 +89715 -1.000000 +89716 -1.000000 +89717 -1.000000 +89718 -1.000000 +89719 -1.000000 +89720 -1.000000 +89721 -1.000000 +89722 -1.000000 +89723 -1.000000 +89724 -1.000000 +89725 -1.000000 +89726 -1.000000 +89727 -1.000000 +89728 -1.000000 +89729 -1.000000 +89730 -1.000000 +89731 -1.000000 +89732 -1.000000 +89734 -1.000000 +89735 -1.000000 +89736 -1.000000 +89737 -1.000000 +89738 -1.000000 +89739 -1.000000 +89740 -1.000000 +89741 -1.000000 +89742 -1.000000 +89743 -1.000000 +89746 -1.000000 +89747 -1.000000 +89748 -1.000000 +89749 -1.000000 +89750 -1.000000 +89751 -1.000000 +89752 -1.000000 +89754 -1.000000 +89756 -1.000000 +89757 -1.000000 +89758 -1.000000 +89759 -1.000000 +89767 -1.000000 +89775 -1.000000 +89776 -1.000000 +89777 -1.000000 +89778 -1.000000 +89779 -1.000000 +89781 -1.000000 +89783 -1.000000 +89784 -1.000000 +89786 -1.000000 +89787 -1.000000 +89788 -1.000000 +89789 -1.000000 +89790 -1.000000 +89791 -1.000000 +89792 -1.000000 +89793 -1.000000 +89794 -1.000000 +89807 -1.000000 +89808 -1.000000 +89809 -1.000000 +89810 -1.000000 +89811 -1.000000 +89812 -1.000000 +89813 -1.000000 +89814 -1.000000 +89815 -1.000000 +89818 -1.000000 +89819 -1.000000 +89820 -1.000000 +89821 -1.000000 +89822 -1.000000 +89823 -1.000000 +89824 -1.000000 +89825 -1.000000 +89826 -1.000000 +89827 -1.000000 +89828 -1.000000 +89829 -1.000000 +89830 -1.000000 +89831 -1.000000 +89832 -1.000000 +89833 -1.000000 +89834 -1.000000 +89835 -1.000000 +89836 -1.000000 +89837 -1.000000 +89838 -1.000000 +89839 -1.000000 +89840 -1.000000 +89841 -1.000000 +89842 -1.000000 +89844 -1.000000 +89845 -1.000000 +89846 -1.000000 +89847 -1.000000 +89848 -1.000000 +89849 -1.000000 +89850 -1.000000 +89851 -1.000000 +89852 -1.000000 +89853 -1.000000 +89854 -1.000000 +89855 -1.000000 +89856 -1.000000 +89857 -1.000000 +89858 -1.000000 +89859 -1.000000 +89860 -1.000000 +89861 -1.000000 +89862 -1.000000 +89863 -1.000000 +89865 -1.000000 +89866 -1.000000 +89867 -1.000000 +89868 -1.000000 +89869 -1.000000 +89871 -1.000000 +89872 -1.000000 +89873 -1.000000 +89874 -1.000000 +89875 -1.000000 +89876 -1.000000 +89877 -1.000000 +89878 -1.000000 +89879 -1.000000 +89880 -1.000000 +89881 -1.000000 +89882 -1.000000 +89883 -1.000000 +89884 -1.000000 +89886 -1.000000 +89887 -1.000000 +89888 -1.000000 +89889 -1.000000 +89890 -1.000000 +89891 -1.000000 +89893 -1.000000 +89894 -1.000000 +89895 -1.000000 +89896 -1.000000 +89897 -1.000000 +89898 -1.000000 +89899 -1.000000 +89900 -1.000000 +89901 -1.000000 +89902 -1.000000 +89903 -1.000000 +89905 -1.000000 +89906 -1.000000 +89907 -1.000000 +89908 -1.000000 +89909 -1.000000 +89910 -1.000000 +89911 -1.000000 +89912 -1.000000 +89913 -1.000000 +89914 -1.000000 +89916 -1.000000 +89917 -1.000000 +89918 -1.000000 +89919 -1.000000 +89920 -1.000000 +89921 -1.000000 +89922 -1.000000 +89923 -1.000000 +89924 -1.000000 +89925 -1.000000 +89926 -1.000000 +89927 -1.000000 +89928 -1.000000 +89929 -1.000000 +89930 -1.000000 +89931 -1.000000 +89932 -1.000000 +89933 -1.000000 +89934 -1.000000 +89935 -1.000000 +89936 -1.000000 +89937 -1.000000 +89938 -1.000000 +89939 -1.000000 +89940 -1.000000 +89941 -1.000000 +89942 -1.000000 +89943 -1.000000 +89944 -1.000000 +89945 -1.000000 +89946 -1.000000 +89947 -1.000000 +89948 -1.000000 +89949 -1.000000 +89951 -1.000000 +89952 -1.000000 +89953 -1.000000 +89954 -1.000000 +89955 -1.000000 +89956 -1.000000 +89957 -1.000000 +89958 -1.000000 +89959 -1.000000 +89960 -1.000000 +89961 -1.000000 +89962 -1.000000 +89963 -1.000000 +89964 -1.000000 +89965 -1.000000 +89966 -1.000000 +89967 -1.000000 +89968 -1.000000 +89969 -1.000000 +89970 -1.000000 +89971 -1.000000 +89972 -1.000000 +89973 -1.000000 +89974 -1.000000 +89975 -1.000000 +89976 -1.000000 +89977 -1.000000 +89978 -1.000000 +89979 -1.000000 +89980 -1.000000 +89981 -1.000000 +89982 -1.000000 +89983 -1.000000 +89984 -1.000000 +89985 -1.000000 +89986 -1.000000 +89987 -1.000000 +89988 -1.000000 +89989 -1.000000 +89990 -1.000000 +89991 -1.000000 +89992 -1.000000 +89993 -1.000000 +89994 -1.000000 +89995 -1.000000 +89996 -1.000000 +89997 -1.000000 +89999 -1.000000 +90000 -1.000000 +90001 -1.000000 +90002 -1.000000 +90003 -1.000000 +90004 -1.000000 +90005 -1.000000 +90006 -1.000000 +90007 -1.000000 +90008 -1.000000 +90009 -1.000000 +90010 -1.000000 +90011 -1.000000 +90012 -1.000000 +90013 -1.000000 +90014 -1.000000 +90015 -1.000000 +90016 -1.000000 +90017 -1.000000 +90018 -1.000000 +90019 -1.000000 +90020 -1.000000 +90021 -1.000000 +90022 -1.000000 +90023 -1.000000 +90024 -1.000000 +90025 -1.000000 +90026 -1.000000 +90027 -1.000000 +90028 -1.000000 +90029 -1.000000 +90030 -1.000000 +90032 -1.000000 +90033 -1.000000 +90034 -1.000000 +90035 -1.000000 +90036 -1.000000 +90038 -1.000000 +90039 -1.000000 +90040 -1.000000 +90041 -1.000000 +90042 -1.000000 +90043 -1.000000 +90044 -1.000000 +90045 -1.000000 +90046 -1.000000 +90050 -1.000000 +90051 -1.000000 +90052 -1.000000 +90053 -1.000000 +90054 -1.000000 +90055 -1.000000 +90056 -1.000000 +90057 -1.000000 +90058 -1.000000 +90059 -1.000000 +90060 -1.000000 +90061 -1.000000 +90062 -1.000000 +90063 -1.000000 +90064 -1.000000 +90065 -1.000000 +90066 -1.000000 +90067 -1.000000 +90068 -1.000000 +90069 -1.000000 +90070 -1.000000 +90071 -1.000000 +90072 -1.000000 +90073 -1.000000 +90074 -1.000000 +90075 -1.000000 +90076 -1.000000 +90077 -1.000000 +90078 -1.000000 +90079 -1.000000 +90080 -1.000000 +90081 -1.000000 +90082 -1.000000 +90083 -1.000000 +90084 -1.000000 +90085 -1.000000 +90086 -1.000000 +90087 -1.000000 +90088 -1.000000 +90089 -1.000000 +90090 -1.000000 +90091 -1.000000 +90092 -1.000000 +90093 -1.000000 +90094 -1.000000 +90095 -1.000000 +90096 -1.000000 +90097 -1.000000 +90098 -1.000000 +90099 -1.000000 +90102 -1.000000 +90103 -1.000000 +90104 -1.000000 +90105 -1.000000 +90106 -1.000000 +90107 -1.000000 +90108 -1.000000 +90109 -1.000000 +90110 -1.000000 +90111 -1.000000 +90113 -1.000000 +90115 -1.000000 +90116 -1.000000 +90117 -1.000000 +90118 -1.000000 +90119 -1.000000 +90120 -1.000000 +90127 -1.000000 +90128 -1.000000 +90130 -1.000000 +90131 -1.000000 +90132 -1.000000 +90133 -1.000000 +90134 -1.000000 +90135 -1.000000 +90136 -1.000000 +90137 -1.000000 +90139 -1.000000 +90140 -1.000000 +90141 -1.000000 +90142 -1.000000 +90143 -1.000000 +90144 -1.000000 +90145 -1.000000 +90146 -1.000000 +90147 -1.000000 +90148 -1.000000 +90149 -1.000000 +90151 -1.000000 +90152 -1.000000 +90154 -1.000000 +90155 -1.000000 +90156 -1.000000 +90157 -1.000000 +90158 -1.000000 +90159 -1.000000 +90160 -1.000000 +90161 -1.000000 +90163 -1.000000 +90165 -1.000000 +90166 -1.000000 +90167 -1.000000 +90168 -1.000000 +90169 -1.000000 +90170 -1.000000 +90171 -1.000000 +90172 -1.000000 +90173 -1.000000 +90174 -1.000000 +90175 -1.000000 +90176 -1.000000 +90177 -1.000000 +90178 -1.000000 +90179 -1.000000 +90180 -1.000000 +90181 -1.000000 +90182 -1.000000 +90183 -1.000000 +90184 -1.000000 +90185 -1.000000 +90186 -1.000000 +90187 -1.000000 +90189 -1.000000 +90190 -1.000000 +90192 -1.000000 +90193 -1.000000 +90194 -1.000000 +90195 -1.000000 +90196 -1.000000 +90197 -1.000000 +90198 -1.000000 +90199 -1.000000 +90200 -1.000000 +90201 -1.000000 +90202 -1.000000 +90203 -1.000000 +90204 -1.000000 +90205 -1.000000 +90206 -1.000000 +90207 -1.000000 +90208 -1.000000 +90209 -1.000000 +90211 -1.000000 +90212 -1.000000 +90213 -1.000000 +90214 -1.000000 +90215 -1.000000 +90216 -1.000000 +90217 -1.000000 +90220 -1.000000 +90221 -1.000000 +90222 -1.000000 +90223 -1.000000 +90224 -1.000000 +90227 -1.000000 +90228 -1.000000 +90229 -1.000000 +90230 -1.000000 +90231 -1.000000 +90232 -1.000000 +90233 -1.000000 +90234 -1.000000 +90235 -1.000000 +90236 -1.000000 +90237 -1.000000 +90238 -1.000000 +90239 -1.000000 +90240 -1.000000 +90241 -1.000000 +90242 -1.000000 +90243 -1.000000 +90244 -1.000000 +90245 -1.000000 +90247 -1.000000 +90248 -1.000000 +90249 -1.000000 +90251 -1.000000 +90255 -1.000000 +90256 -1.000000 +90257 -1.000000 +90258 -1.000000 +90259 -1.000000 +90260 -1.000000 +90261 -1.000000 +90262 -1.000000 +90263 -1.000000 +90264 -1.000000 +90265 -1.000000 +90266 -1.000000 +90267 -1.000000 +90270 -1.000000 +90271 -1.000000 +90272 -1.000000 +90273 -1.000000 +90274 -1.000000 +90275 -1.000000 +90277 -1.000000 +90278 -1.000000 +90279 -1.000000 +90280 -1.000000 +90282 -1.000000 +90283 -1.000000 +90284 -1.000000 +90285 -1.000000 +90286 -1.000000 +90287 -1.000000 +90288 -1.000000 +90289 -1.000000 +90291 -1.000000 +90292 -1.000000 +90293 -1.000000 +90294 -1.000000 +90295 -1.000000 +90296 -1.000000 +90297 -1.000000 +90298 -1.000000 +90308 -1.000000 +90313 -1.000000 +90314 -1.000000 +90315 -1.000000 +90316 -1.000000 +90317 -1.000000 +90318 -1.000000 +90321 -1.000000 +90323 -1.000000 +90324 -1.000000 +90325 -1.000000 +90326 -1.000000 +90327 -1.000000 +90328 -1.000000 +90329 -1.000000 +90331 -1.000000 +90332 -1.000000 +90338 -1.000000 +90339 -1.000000 +90340 -1.000000 +90341 -1.000000 +90342 -1.000000 +90343 -1.000000 +90344 -1.000000 +90345 -1.000000 +90348 -1.000000 +90349 -1.000000 +90350 -1.000000 +90351 -1.000000 +90352 -1.000000 +90353 -1.000000 +90354 -1.000000 +90355 -1.000000 +90356 -1.000000 +90360 -1.000000 +90361 -1.000000 +90363 -1.000000 +90364 -1.000000 +90365 -1.000000 +90366 -1.000000 +90372 -1.000000 +90373 -1.000000 +90374 -1.000000 +90375 -1.000000 +90376 -1.000000 +90377 -1.000000 +90379 -1.000000 +90380 -1.000000 +90381 -1.000000 +90382 -1.000000 +90383 -1.000000 +90385 -1.000000 +90398 -1.000000 +90399 -1.000000 +90400 -1.000000 +90401 -1.000000 +90402 -1.000000 +90403 -1.000000 +90405 -1.000000 +90406 -1.000000 +90407 -1.000000 +90408 -1.000000 +90409 -1.000000 +90410 -1.000000 +90411 -1.000000 +90413 -1.000000 +90414 -1.000000 +90415 -1.000000 +90416 -1.000000 +90417 -1.000000 +90418 -1.000000 +90419 -1.000000 +90421 -1.000000 +90422 -1.000000 +90423 -1.000000 +90424 -1.000000 +90425 -1.000000 +90426 -1.000000 +90427 -1.000000 +90428 -1.000000 +90429 -1.000000 +90430 -1.000000 +90431 -1.000000 +90432 -1.000000 +90435 -1.000000 +90436 -1.000000 +90437 -1.000000 +90438 -1.000000 +90439 -1.000000 +90440 -1.000000 +90441 -1.000000 +90442 -1.000000 +90443 -1.000000 +90446 -1.000000 +90447 -1.000000 +90448 -1.000000 +90449 -1.000000 +90450 -1.000000 +90451 -1.000000 +90452 -1.000000 +90453 -1.000000 +90454 -1.000000 +90455 -1.000000 +90456 -1.000000 +90457 -1.000000 +90458 -1.000000 +90459 -1.000000 +90460 -1.000000 +90462 -1.000000 +90463 -1.000000 +90465 -1.000000 +90466 -1.000000 +90468 -1.000000 +90471 -1.000000 +90472 -1.000000 +90473 -1.000000 +90474 -1.000000 +90475 -1.000000 +90476 -1.000000 +90477 -1.000000 +90478 -1.000000 +90479 -1.000000 +90480 -1.000000 +90481 -1.000000 +90482 -1.000000 +90483 -1.000000 +90484 -1.000000 +90485 -1.000000 +90486 -1.000000 +90487 -1.000000 +90488 -1.000000 +90489 -1.000000 +90490 -1.000000 +90491 -1.000000 +90492 -1.000000 +90493 -1.000000 +90494 -1.000000 +90496 -1.000000 +90497 -1.000000 +90498 -1.000000 +90499 -1.000000 +90500 -1.000000 +90501 -1.000000 +90502 -1.000000 +90503 -1.000000 +90504 -1.000000 +90505 -1.000000 +90506 -1.000000 +90507 -1.000000 +90508 -1.000000 +90509 -1.000000 +90510 -1.000000 +90511 -1.000000 +90514 -1.000000 +90515 -1.000000 +90516 -1.000000 +90517 -1.000000 +90518 -1.000000 +90519 -1.000000 +90520 -1.000000 +90521 -1.000000 +90522 -1.000000 +90523 -1.000000 +90524 -1.000000 +90525 -1.000000 +90526 -1.000000 +90527 -1.000000 +90528 -1.000000 +90530 -1.000000 +90531 -1.000000 +90534 -1.000000 +90535 -1.000000 +90536 -1.000000 +90537 -1.000000 +90538 -1.000000 +90539 -1.000000 +90540 -1.000000 +90541 -1.000000 +90542 -1.000000 +90543 -1.000000 +90545 -1.000000 +90546 -1.000000 +90547 -1.000000 +90549 -1.000000 +90550 -1.000000 +90551 -1.000000 +90552 -1.000000 +90553 -1.000000 +90554 -1.000000 +90555 -1.000000 +90556 -1.000000 +90557 -1.000000 +90558 -1.000000 +90559 -1.000000 +90560 -1.000000 +90561 -1.000000 +90562 -1.000000 +90563 -1.000000 +90564 -1.000000 +90565 -1.000000 +90566 -1.000000 +90567 -1.000000 +90568 -1.000000 +90569 -1.000000 +90570 -1.000000 +90571 -1.000000 +90572 -1.000000 +90573 -1.000000 +90574 -1.000000 +90575 -1.000000 +90576 -1.000000 +90577 -1.000000 +90578 -1.000000 +90580 -1.000000 +90581 -1.000000 +90582 -1.000000 +90583 -1.000000 +90584 -1.000000 +90586 -1.000000 +90587 -1.000000 +90588 -1.000000 +90589 -1.000000 +90590 -1.000000 +90591 -1.000000 +90592 -1.000000 +90593 -1.000000 +90597 -1.000000 +90604 -1.000000 +90605 -1.000000 +90608 -1.000000 +90609 -1.000000 +90610 -1.000000 +90622 -1.000000 +90626 -1.000000 +90627 -1.000000 +90628 -1.000000 +90629 -1.000000 +90630 -1.000000 +90631 -1.000000 +90632 -1.000000 +90633 -1.000000 +90634 -1.000000 +90635 -1.000000 +90636 -1.000000 +90637 -1.000000 +90638 -1.000000 +90640 -1.000000 +90642 -1.000000 +90645 -1.000000 +90646 -1.000000 +90647 -1.000000 +90649 -1.000000 +90650 -1.000000 +90651 -1.000000 +90652 -1.000000 +90653 -1.000000 +90654 -1.000000 +90655 -1.000000 +90656 -1.000000 +90657 -1.000000 +90658 -1.000000 +90660 -1.000000 +90661 -1.000000 +90662 -1.000000 +90663 -1.000000 +90664 -1.000000 +90665 -1.000000 +90666 -1.000000 +90667 -1.000000 +90668 -1.000000 +90669 -1.000000 +90670 -1.000000 +90671 -1.000000 +90672 -1.000000 +90674 -1.000000 +90677 -1.000000 +90678 -1.000000 +90682 -1.000000 +90683 -1.000000 +90697 -1.000000 +90698 -1.000000 +90702 -1.000000 +90703 -1.000000 +90704 -1.000000 +90705 -1.000000 +90708 -1.000000 +90711 -1.000000 +90712 -1.000000 +90713 -1.000000 +90714 -1.000000 +90715 -1.000000 +90716 -1.000000 +90717 -1.000000 +90718 -1.000000 +90719 -1.000000 +90722 -1.000000 +90725 -1.000000 +90726 -1.000000 +90728 -1.000000 +90730 -1.000000 +90731 -1.000000 +90732 -1.000000 +90733 -1.000000 +90734 -1.000000 +90735 -1.000000 +90736 -1.000000 +90737 -1.000000 +90738 -1.000000 +90739 -1.000000 +90740 -1.000000 +90741 -1.000000 +90742 -1.000000 +90743 -1.000000 +90744 -1.000000 +90745 -1.000000 +90746 -1.000000 +90747 -1.000000 +90748 -1.000000 +90749 -1.000000 +90750 -1.000000 +90751 -1.000000 +90752 -1.000000 +90753 -1.000000 +90754 -1.000000 +90755 -1.000000 +90757 -1.000000 +90758 -1.000000 +90759 -1.000000 +90760 -1.000000 +90761 -1.000000 +90762 -1.000000 +90763 -1.000000 +90764 -1.000000 +90765 -1.000000 +90766 -1.000000 +90767 -1.000000 +90768 -1.000000 +90769 -1.000000 +90770 -1.000000 +90771 -1.000000 +90772 -1.000000 +90773 -1.000000 +90774 -1.000000 +90775 -1.000000 +90776 -1.000000 +90777 -1.000000 +90778 -1.000000 +90779 -1.000000 +90780 -1.000000 +90781 -1.000000 +90782 -1.000000 +90783 -1.000000 +90784 -1.000000 +90785 -1.000000 +90786 -1.000000 +90788 -1.000000 +90789 -1.000000 +90790 -1.000000 +90791 -1.000000 +90793 -1.000000 +90794 -1.000000 +90795 -1.000000 +90796 -1.000000 +90797 -1.000000 +90798 -1.000000 +90799 -1.000000 +90800 -1.000000 +90801 -1.000000 +90802 -1.000000 +90803 -1.000000 +90804 -1.000000 +90805 -1.000000 +90806 -1.000000 +90807 -1.000000 +90808 -1.000000 +90809 -1.000000 +90810 -1.000000 +90811 -1.000000 +90812 -1.000000 +90813 -1.000000 +90814 -1.000000 +90815 -1.000000 +90816 -1.000000 +90817 -1.000000 +90818 -1.000000 +90819 -1.000000 +90820 -1.000000 +90821 -1.000000 +90822 -1.000000 +90823 -1.000000 +90824 -1.000000 +90825 -1.000000 +90826 -1.000000 +90827 -1.000000 +90828 -1.000000 +90829 -1.000000 +90830 -1.000000 +90831 -1.000000 +90832 -1.000000 +90833 -1.000000 +90834 -1.000000 +90835 -1.000000 +90836 -1.000000 +90837 -1.000000 +90838 -1.000000 +90839 -1.000000 +90840 -1.000000 +90841 -1.000000 +90842 -1.000000 +90843 -1.000000 +90844 -1.000000 +90845 -1.000000 +90846 -1.000000 +90847 -1.000000 +90848 -1.000000 +90849 -1.000000 +90850 -1.000000 +90851 -1.000000 +90852 -1.000000 +90853 -1.000000 +90854 -1.000000 +90855 -1.000000 +90856 -1.000000 +90857 -1.000000 +90858 -1.000000 +90859 -1.000000 +90860 -1.000000 +90861 -1.000000 +90862 -1.000000 +90863 -1.000000 +90864 -1.000000 +90865 -1.000000 +90866 -1.000000 +90867 -1.000000 +90868 -1.000000 +90869 -1.000000 +90870 -1.000000 +90871 -1.000000 +90872 -1.000000 +90873 -1.000000 +90874 -1.000000 +90875 -1.000000 +90876 -1.000000 +90877 -1.000000 +90878 -1.000000 +90879 -1.000000 +90880 -1.000000 +90881 -1.000000 +90882 -1.000000 +90883 -1.000000 +90884 -1.000000 +90885 -1.000000 +90886 -1.000000 +90888 -1.000000 +90889 -1.000000 +90890 -1.000000 +90891 -1.000000 +90892 -1.000000 +90893 -1.000000 +90894 -1.000000 +90895 -1.000000 +90896 -1.000000 +90897 -1.000000 +90898 -1.000000 +90899 -1.000000 +90901 -1.000000 +90902 -1.000000 +90903 -1.000000 +90905 -1.000000 +90906 -1.000000 +90907 -1.000000 +90908 -1.000000 +90909 -1.000000 +90910 -1.000000 +90911 -1.000000 +90912 -1.000000 +90913 -1.000000 +90914 -1.000000 +90915 -1.000000 +90916 -1.000000 +90917 -1.000000 +90918 -1.000000 +90919 -1.000000 +90920 -1.000000 +90921 -1.000000 +90922 -1.000000 +90923 -1.000000 +90925 -1.000000 +90926 -1.000000 +90927 -1.000000 +90928 -1.000000 +90929 -1.000000 +90930 -1.000000 +90931 -1.000000 +90932 -1.000000 +90933 -1.000000 +90934 -1.000000 +90935 -1.000000 +90936 -1.000000 +90937 -1.000000 +90938 -1.000000 +90939 -1.000000 +90940 -1.000000 +90941 -1.000000 +90942 -1.000000 +90943 -1.000000 +90944 -1.000000 +90945 -1.000000 +90946 -1.000000 +90947 -1.000000 +90948 -1.000000 +90949 -1.000000 +90950 -1.000000 +90952 -1.000000 +90954 -1.000000 +90955 -1.000000 +90956 -1.000000 +90957 -1.000000 +90958 -1.000000 +90959 -1.000000 +90960 -1.000000 +90961 -1.000000 +90962 -1.000000 +90963 -1.000000 +90964 -1.000000 +90967 -1.000000 +90968 -1.000000 +90969 -1.000000 +90970 -1.000000 +90971 -1.000000 +90972 -1.000000 +90973 -1.000000 +90974 -1.000000 +90975 -1.000000 +90976 -1.000000 +90977 -1.000000 +90978 -1.000000 +90979 -1.000000 +90980 -1.000000 +90981 -1.000000 +90983 -1.000000 +90984 -1.000000 +90986 -1.000000 +90987 -1.000000 +90988 -1.000000 +90989 -1.000000 +90990 -1.000000 +90991 -1.000000 +90992 -1.000000 +90993 -1.000000 +90994 -1.000000 +90995 -1.000000 +90996 -1.000000 +90997 -1.000000 +90998 -1.000000 +90999 -1.000000 +91000 -1.000000 +91001 -1.000000 +91002 -1.000000 +91003 -1.000000 +91004 -1.000000 +91005 -1.000000 +91007 -1.000000 +91008 -1.000000 +91009 -1.000000 +91010 -1.000000 +91011 -1.000000 +91012 -1.000000 +91014 -1.000000 +91015 -1.000000 +91016 -1.000000 +91017 -1.000000 +91018 -1.000000 +91019 -1.000000 +91020 -1.000000 +91021 -1.000000 +91022 -1.000000 +91023 -1.000000 +91027 -1.000000 +91034 -1.000000 +91041 -1.000000 +91043 -1.000000 +91044 -1.000000 +91045 -1.000000 +91046 -1.000000 +91047 -1.000000 +91048 -1.000000 +91049 -1.000000 +91050 -1.000000 +91051 -1.000000 +91052 -1.000000 +91053 -1.000000 +91054 -1.000000 +91055 -1.000000 +91056 -1.000000 +91057 -1.000000 +91058 -1.000000 +91059 -1.000000 +91060 -1.000000 +91061 -1.000000 +91062 -1.000000 +91063 -1.000000 +91064 -1.000000 +91065 -1.000000 +91066 -1.000000 +91070 -1.000000 +91075 -1.000000 +91083 -1.000000 +91084 -1.000000 +91085 -1.000000 +91086 -1.000000 +91087 -1.000000 +91089 -1.000000 +91097 -1.000000 +91099 -1.000000 +91100 -1.000000 +91101 -1.000000 +91102 -1.000000 +91104 -1.000000 +91105 -1.000000 +91106 -1.000000 +91107 -1.000000 +91108 -1.000000 +91109 -1.000000 +91110 -1.000000 +91111 -1.000000 +91112 -1.000000 +91113 -1.000000 +91114 -1.000000 +91115 -1.000000 +91116 -1.000000 +91117 -1.000000 +91118 -1.000000 +91119 -1.000000 +91120 -1.000000 +91121 -1.000000 +91122 -1.000000 +91123 -1.000000 +91124 -1.000000 +91125 -1.000000 +91126 -1.000000 +91127 -1.000000 +91128 -1.000000 +91129 -1.000000 +91130 -1.000000 +91131 -1.000000 +91133 -1.000000 +91134 -1.000000 +91135 -1.000000 +91136 -1.000000 +91137 -1.000000 +91138 -1.000000 +91139 -1.000000 +91140 -1.000000 +91141 -1.000000 +91142 -1.000000 +91143 -1.000000 +91144 -1.000000 +91145 -1.000000 +91146 -1.000000 +91147 -1.000000 +91148 -1.000000 +91149 -1.000000 +91150 -1.000000 +91151 -1.000000 +91152 -1.000000 +91153 -1.000000 +91154 -1.000000 +91155 -1.000000 +91156 -1.000000 +91157 -1.000000 +91158 -1.000000 +91159 -1.000000 +91160 -1.000000 +91161 -1.000000 +91162 -1.000000 +91163 -1.000000 +91164 -1.000000 +91165 -1.000000 +91166 -1.000000 +91167 -1.000000 +91168 -1.000000 +91169 -1.000000 +91170 -1.000000 +91171 -1.000000 +91172 -1.000000 +91173 -1.000000 +91174 -1.000000 +91175 -1.000000 +91176 -1.000000 +91177 -1.000000 +91178 -1.000000 +91179 -1.000000 +91180 -1.000000 +91181 -1.000000 +91182 -1.000000 +91183 -1.000000 +91184 -1.000000 +91185 -1.000000 +91186 -1.000000 +91187 -1.000000 +91188 -1.000000 +91189 -1.000000 +91190 -1.000000 +91191 -1.000000 +91192 -1.000000 +91193 -1.000000 +91194 -1.000000 +91195 -1.000000 +91196 -1.000000 +91197 -1.000000 +91198 -1.000000 +91199 -1.000000 +91200 -1.000000 +91201 -1.000000 +91202 -1.000000 +91203 -1.000000 +91204 -1.000000 +91205 -1.000000 +91206 -1.000000 +91207 -1.000000 +91208 -1.000000 +91209 -1.000000 +91210 -1.000000 +91211 -1.000000 +91212 -1.000000 +91213 -1.000000 +91214 -1.000000 +91215 -1.000000 +91216 -1.000000 +91217 -1.000000 +91218 -1.000000 +91219 -1.000000 +91220 -1.000000 +91221 -1.000000 +91222 -1.000000 +91223 -1.000000 +91224 -1.000000 +91225 -1.000000 +91226 -1.000000 +91227 -1.000000 +91228 -1.000000 +91229 -1.000000 +91230 -1.000000 +91231 -1.000000 +91232 -1.000000 +91233 -1.000000 +91234 -1.000000 +91235 -1.000000 +91236 -1.000000 +91237 -1.000000 +91238 -1.000000 +91239 -1.000000 +91240 -1.000000 +91241 -1.000000 +91242 -1.000000 +91244 -1.000000 +91245 -1.000000 +91246 -1.000000 +91247 -1.000000 +91248 -1.000000 +91249 -1.000000 +91250 -1.000000 +91251 -1.000000 +91252 -1.000000 +91253 -1.000000 +91254 -1.000000 +91255 -1.000000 +91256 -1.000000 +91257 -1.000000 +91258 -1.000000 +91259 -1.000000 +91260 -1.000000 +91261 -1.000000 +91262 -1.000000 +91263 -1.000000 +91264 -1.000000 +91265 -1.000000 +91267 -1.000000 +91268 -1.000000 +91270 -1.000000 +91271 -1.000000 +91272 -1.000000 +91273 -1.000000 +91274 -1.000000 +91275 -1.000000 +91276 -1.000000 +91277 -1.000000 +91279 -1.000000 +91280 -1.000000 +91281 -1.000000 +91282 -1.000000 +91283 -1.000000 +91286 -1.000000 +91287 -1.000000 +91288 -1.000000 +91289 -1.000000 +91290 -1.000000 +91295 -1.000000 +91296 -1.000000 +91297 -1.000000 +91298 -1.000000 +91299 -1.000000 +91301 -1.000000 +91303 -1.000000 +91304 -1.000000 +91305 -1.000000 +91306 -1.000000 +91307 -1.000000 +91308 -1.000000 +91309 -1.000000 +91310 -1.000000 +91311 -1.000000 +91312 -1.000000 +91313 -1.000000 +91314 -1.000000 +91315 -1.000000 +91316 -1.000000 +91317 -1.000000 +91318 -1.000000 +91319 -1.000000 +91320 -1.000000 +91321 -1.000000 +91322 -1.000000 +91323 -1.000000 +91324 -1.000000 +91325 -1.000000 +91326 -1.000000 +91327 -1.000000 +91328 -1.000000 +91329 -1.000000 +91330 -1.000000 +91331 -1.000000 +91332 -1.000000 +91333 -1.000000 +91335 -1.000000 +91336 -1.000000 +91337 -1.000000 +91341 -1.000000 +91343 -1.000000 +91344 -1.000000 +91345 -1.000000 +91346 -1.000000 +91347 -1.000000 +91348 -1.000000 +91349 -1.000000 +91350 -1.000000 +91351 -1.000000 +91352 -1.000000 +91353 -1.000000 +91354 -1.000000 +91356 -1.000000 +91357 -1.000000 +91359 -1.000000 +91360 -1.000000 +91361 -1.000000 +91362 -1.000000 +91363 -1.000000 +91364 -1.000000 +91365 -1.000000 +91366 -1.000000 +91367 -1.000000 +91368 -1.000000 +91369 -1.000000 +91370 -1.000000 +91371 -1.000000 +91372 -1.000000 +91373 -1.000000 +91374 -1.000000 +91375 -1.000000 +91376 -1.000000 +91377 -1.000000 +91378 -1.000000 +91379 -1.000000 +91380 -1.000000 +91381 -1.000000 +91382 -1.000000 +91383 -1.000000 +91384 -1.000000 +91385 -1.000000 +91386 -1.000000 +91387 -1.000000 +91389 -1.000000 +91390 -1.000000 +91391 -1.000000 +91392 -1.000000 +91393 -1.000000 +91394 -1.000000 +91395 -1.000000 +91396 -1.000000 +91397 -1.000000 +91398 -1.000000 +91399 -1.000000 +91400 -1.000000 +91401 -1.000000 +91402 -1.000000 +91403 -1.000000 +91404 -1.000000 +91405 -1.000000 +91406 -1.000000 +91408 -1.000000 +91409 -1.000000 +91410 -1.000000 +91411 -1.000000 +91412 -1.000000 +91413 -1.000000 +91414 -1.000000 +91415 -1.000000 +91416 -1.000000 +91417 -1.000000 +91418 -1.000000 +91419 -1.000000 +91420 -1.000000 +91421 -1.000000 +91422 -1.000000 +91423 -1.000000 +91424 -1.000000 +91425 -1.000000 +91426 -1.000000 +91428 -1.000000 +91430 -1.000000 +91431 -1.000000 +91432 -1.000000 +91433 -1.000000 +91434 -1.000000 +91435 -1.000000 +91436 -1.000000 +91437 -1.000000 +91438 -1.000000 +91439 -1.000000 +91440 -1.000000 +91441 -1.000000 +91442 -1.000000 +91444 -1.000000 +91447 -1.000000 +91448 -1.000000 +91449 -1.000000 +91450 -1.000000 +91451 -1.000000 +91452 -1.000000 +91453 -1.000000 +91454 -1.000000 +91455 -1.000000 +91456 -1.000000 +91457 -1.000000 +91458 -1.000000 +91459 -1.000000 +91460 -1.000000 +91461 -1.000000 +91462 -1.000000 +91464 -1.000000 +91465 -1.000000 +91466 -1.000000 +91467 -1.000000 +91468 -1.000000 +91469 -1.000000 +91470 -1.000000 +91471 -1.000000 +91472 -1.000000 +91473 -1.000000 +91474 -1.000000 +91475 -1.000000 +91476 -1.000000 +91477 -1.000000 +91478 -1.000000 +91479 -1.000000 +91480 -1.000000 +91481 -1.000000 +91482 -1.000000 +91483 -1.000000 +91484 -1.000000 +91485 -1.000000 +91486 -1.000000 +91487 -1.000000 +91488 -1.000000 +91489 -1.000000 +91490 -1.000000 +91491 -1.000000 +91492 -1.000000 +91493 -1.000000 +91494 -1.000000 +91495 -1.000000 +91496 -1.000000 +91498 -1.000000 +91499 -1.000000 +91500 -1.000000 +91501 -1.000000 +91502 -1.000000 +91503 -1.000000 +91504 -1.000000 +91505 -1.000000 +91507 -1.000000 +91508 -1.000000 +91510 -1.000000 +91511 -1.000000 +91512 -1.000000 +91513 -1.000000 +91514 -1.000000 +91515 -1.000000 +91516 -1.000000 +91517 -1.000000 +91518 -1.000000 +91519 -1.000000 +91520 -1.000000 +91521 -1.000000 +91523 -1.000000 +91524 -1.000000 +91525 -1.000000 +91526 -1.000000 +91527 -1.000000 +91528 -1.000000 +91529 -1.000000 +91530 -1.000000 +91531 -1.000000 +91532 -1.000000 +91533 -1.000000 +91535 -1.000000 +91536 -1.000000 +91537 -1.000000 +91538 -1.000000 +91539 -1.000000 +91540 -1.000000 +91542 -1.000000 +91543 -1.000000 +91544 -1.000000 +91545 -1.000000 +91546 -1.000000 +91547 -1.000000 +91548 -1.000000 +91549 -1.000000 +91550 -1.000000 +91551 -1.000000 +91552 -1.000000 +91553 -1.000000 +91554 -1.000000 +91555 -1.000000 +91556 -1.000000 +91557 -1.000000 +91558 -1.000000 +91559 -1.000000 +91560 -1.000000 +91561 -1.000000 +91562 -1.000000 +91563 -1.000000 +91564 -1.000000 +91565 -1.000000 +91566 -1.000000 +91567 -1.000000 +91568 -1.000000 +91569 -1.000000 +91571 -1.000000 +91573 -1.000000 +91574 -1.000000 +91575 -1.000000 +91576 -1.000000 +91577 -1.000000 +91578 -1.000000 +91579 -1.000000 +91580 -1.000000 +91581 -1.000000 +91582 -1.000000 +91583 -1.000000 +91584 -1.000000 +91585 -1.000000 +91587 -1.000000 +91588 -1.000000 +91590 -1.000000 +91591 -1.000000 +91592 -1.000000 +91593 -1.000000 +91594 -1.000000 +91595 -1.000000 +91596 -1.000000 +91597 -1.000000 +91599 -1.000000 +91600 -1.000000 +91601 -1.000000 +91602 -1.000000 +91603 -1.000000 +91604 -1.000000 +91605 -1.000000 +91606 -1.000000 +91607 -1.000000 +91608 -1.000000 +91609 -1.000000 +91610 -1.000000 +91611 -1.000000 +91612 -1.000000 +91613 -1.000000 +91614 -1.000000 +91616 -1.000000 +91617 -1.000000 +91618 -1.000000 +91619 -1.000000 +91620 -1.000000 +91621 -1.000000 +91622 -1.000000 +91623 -1.000000 +91624 -1.000000 +91625 -1.000000 +91626 -1.000000 +91630 -1.000000 +91631 -1.000000 +91632 -1.000000 +91633 -1.000000 +91634 -1.000000 +91635 -1.000000 +91637 -1.000000 +91638 -1.000000 +91639 -1.000000 +91640 -1.000000 +91643 -1.000000 +91644 -1.000000 +91645 -1.000000 +91646 -1.000000 +91647 -1.000000 +91648 -1.000000 +91649 -1.000000 +91650 -1.000000 +91652 -1.000000 +91653 -1.000000 +91654 -1.000000 +91655 -1.000000 +91656 -1.000000 +91658 -1.000000 +91659 -1.000000 +91660 -1.000000 +91661 -1.000000 +91662 -1.000000 +91663 -1.000000 +91664 -1.000000 +91665 -1.000000 +91666 -1.000000 +91667 -1.000000 +91668 -1.000000 +91669 -1.000000 +91670 -1.000000 +91671 -1.000000 +91672 -1.000000 +91673 -1.000000 +91674 -1.000000 +91675 -1.000000 +91676 -1.000000 +91679 -1.000000 +91680 -1.000000 +91682 -1.000000 +91683 -1.000000 +91684 -1.000000 +91685 -1.000000 +91686 -1.000000 +91687 -1.000000 +91688 -1.000000 +91689 -1.000000 +91690 -1.000000 +91691 -1.000000 +91692 -1.000000 +91693 -1.000000 +91694 -1.000000 +91695 -1.000000 +91697 -1.000000 +91698 -1.000000 +91699 -1.000000 +91700 -1.000000 +91701 -1.000000 +91702 -1.000000 +91703 -1.000000 +91705 -1.000000 +91706 -1.000000 +91708 -1.000000 +91710 -1.000000 +91711 -1.000000 +91712 -1.000000 +91713 -1.000000 +91714 -1.000000 +91715 -1.000000 +91716 -1.000000 +91717 -1.000000 +91718 -1.000000 +91719 -1.000000 +91720 -1.000000 +91721 -1.000000 +91722 -1.000000 +91723 -1.000000 +91724 -1.000000 +91725 -1.000000 +91726 -1.000000 +91729 -1.000000 +91730 -1.000000 +91731 -1.000000 +91732 -1.000000 +91733 -1.000000 +91734 -1.000000 +91735 -1.000000 +91736 -1.000000 +91737 -1.000000 +91738 -1.000000 +91739 -1.000000 +91740 -1.000000 +91741 -1.000000 +91742 -1.000000 +91743 -1.000000 +91744 -1.000000 +91745 -1.000000 +91746 -1.000000 +91747 -1.000000 +91748 -1.000000 +91749 -1.000000 +91750 -1.000000 +91751 -1.000000 +91752 -1.000000 +91753 -1.000000 +91754 -1.000000 +91755 -1.000000 +91756 -1.000000 +91757 -1.000000 +91758 -1.000000 +91759 -1.000000 +91760 -1.000000 +91761 -1.000000 +91762 -1.000000 +91763 -1.000000 +91764 -1.000000 +91765 -1.000000 +91766 -1.000000 +91767 -1.000000 +91768 -1.000000 +91769 -1.000000 +91770 -1.000000 +91771 -1.000000 +91772 -1.000000 +91773 -1.000000 +91774 -1.000000 +91775 -1.000000 +91776 -1.000000 +91777 -1.000000 +91778 -1.000000 +91779 -1.000000 +91780 -1.000000 +91781 -1.000000 +91782 -1.000000 +91783 -1.000000 +91784 -1.000000 +91785 -1.000000 +91786 -1.000000 +91787 -1.000000 +91788 -1.000000 +91789 -1.000000 +91790 -1.000000 +91791 -1.000000 +91792 -1.000000 +91794 -1.000000 +91795 -1.000000 +91797 -1.000000 +91798 -1.000000 +91799 -1.000000 +91800 -1.000000 +91801 -1.000000 +91802 -1.000000 +91803 -1.000000 +91804 -1.000000 +91805 -1.000000 +91806 -1.000000 +91807 -1.000000 +91808 -1.000000 +91809 -1.000000 +91810 -1.000000 +91811 -1.000000 +91812 -1.000000 +91813 -1.000000 +91814 -1.000000 +91815 -1.000000 +91816 -1.000000 +91817 -1.000000 +91818 -1.000000 +91819 -1.000000 +91821 -1.000000 +91822 -1.000000 +91823 -1.000000 +91824 -1.000000 +91825 -1.000000 +91827 -1.000000 +91828 -1.000000 +91829 -1.000000 +91830 -1.000000 +91831 -1.000000 +91833 -1.000000 +91834 -1.000000 +91835 -1.000000 +91836 -1.000000 +91837 -1.000000 +91838 -1.000000 +91839 -1.000000 +91840 -1.000000 +91841 -1.000000 +91842 -1.000000 +91844 -1.000000 +91845 -1.000000 +91846 -1.000000 +91847 -1.000000 +91848 -1.000000 +91849 -1.000000 +91850 -1.000000 +91851 -1.000000 +91852 -1.000000 +91853 -1.000000 +91854 -1.000000 +91855 -1.000000 +91856 -1.000000 +91857 -1.000000 +91858 -1.000000 +91859 -1.000000 +91860 -1.000000 +91861 -1.000000 +91862 -1.000000 +91863 -1.000000 +91864 -1.000000 +91865 -1.000000 +91866 -1.000000 +91867 -1.000000 +91868 -1.000000 +91869 -1.000000 +91870 -1.000000 +91871 -1.000000 +91872 -1.000000 +91873 -1.000000 +91874 -1.000000 +91875 -1.000000 +91876 -1.000000 +91877 -1.000000 +91878 -1.000000 +91879 -1.000000 +91880 -1.000000 +91881 -1.000000 +91882 -1.000000 +91883 -1.000000 +91884 -1.000000 +91885 -1.000000 +91886 -1.000000 +91887 -1.000000 +91890 -1.000000 +91891 -1.000000 +91892 -1.000000 +91893 -1.000000 +91894 -1.000000 +91895 -1.000000 +91896 -1.000000 +91897 -1.000000 +91898 -1.000000 +91899 -1.000000 +91900 -1.000000 +91901 -1.000000 +91902 -1.000000 +91903 -1.000000 +91904 -1.000000 +91905 -1.000000 +91906 -1.000000 +91908 -1.000000 +91909 -1.000000 +91910 -1.000000 +91911 -1.000000 +91912 -1.000000 +91913 -1.000000 +91914 -1.000000 +91915 -1.000000 +91916 -1.000000 +91917 -1.000000 +91918 -1.000000 +91919 -1.000000 +91920 -1.000000 +91921 -1.000000 +91922 -1.000000 +91923 -1.000000 +91925 -1.000000 +91926 -1.000000 +91927 -1.000000 +91928 -1.000000 +91929 -1.000000 +91930 -1.000000 +91931 -1.000000 +91932 -1.000000 +91933 -1.000000 +91934 -1.000000 +91935 -1.000000 +91936 -1.000000 +91937 -1.000000 +91938 -1.000000 +91939 -1.000000 +91940 -1.000000 +91941 -1.000000 +91942 -1.000000 +91943 -1.000000 +91944 -1.000000 +91945 -1.000000 +91946 -1.000000 +91947 -1.000000 +91948 -1.000000 +91949 -1.000000 +91950 -1.000000 +91951 -1.000000 +91952 -1.000000 +91953 -1.000000 +91954 -1.000000 +91955 -1.000000 +91956 -1.000000 +91957 -1.000000 +91958 -1.000000 +91959 -1.000000 +91960 -1.000000 +91961 -1.000000 +91962 -1.000000 +91964 -1.000000 +91965 -1.000000 +91966 -1.000000 +91967 -1.000000 +91968 -1.000000 +91969 -1.000000 +91973 -1.000000 +91974 -1.000000 +91975 -1.000000 +91976 -1.000000 +91978 -1.000000 +91979 -1.000000 +91980 -1.000000 +91982 -1.000000 +91984 -1.000000 +91985 -1.000000 +91986 -1.000000 +91987 -1.000000 +91988 -1.000000 +91996 -1.000000 +91999 -1.000000 +92000 -1.000000 +92002 -1.000000 +92004 -1.000000 +92005 -1.000000 +92007 -1.000000 +92011 -1.000000 +92023 -1.000000 +92024 -1.000000 +92025 -1.000000 +92026 -1.000000 +92027 -1.000000 +92028 -1.000000 +92029 -1.000000 +92030 -1.000000 +92032 -1.000000 +92039 -1.000000 +92041 -1.000000 +92042 -1.000000 +92046 -1.000000 +92048 -1.000000 +92051 -1.000000 +92052 -1.000000 +92053 -1.000000 +92055 -1.000000 +92058 -1.000000 +92060 -1.000000 +92061 -1.000000 +92062 -1.000000 +92063 -1.000000 +92064 -1.000000 +92065 -1.000000 +92066 -1.000000 +92067 -1.000000 +92068 -1.000000 +92071 -1.000000 +92073 -1.000000 +92074 -1.000000 +92075 -1.000000 +92076 -1.000000 +92078 -1.000000 +92079 -1.000000 +92083 -1.000000 +92084 -1.000000 +92085 -1.000000 +92086 -1.000000 +92089 -1.000000 +92090 -1.000000 +92091 -1.000000 +92092 -1.000000 +92093 -1.000000 +92094 -1.000000 +92095 -1.000000 +92096 -1.000000 +92097 -1.000000 +92102 -1.000000 +92103 -1.000000 +92104 -1.000000 +92106 -1.000000 +92108 -1.000000 +92109 -1.000000 +92110 -1.000000 +92111 -1.000000 +92112 -1.000000 +92114 -1.000000 +92115 -1.000000 +92117 -1.000000 +92118 -1.000000 +92119 -1.000000 +92121 -1.000000 +92122 -1.000000 +92126 -1.000000 +92127 -1.000000 +92128 -1.000000 +92130 -1.000000 +92131 -1.000000 +92133 -1.000000 +92137 -1.000000 +92138 -1.000000 +92139 -1.000000 +92141 -1.000000 +92142 -1.000000 +92143 -1.000000 +92145 -1.000000 +92146 -1.000000 +92147 -1.000000 +92148 -1.000000 +92149 -1.000000 +92150 -1.000000 +92151 -1.000000 +92154 -1.000000 +92155 -1.000000 +92158 -1.000000 +92159 -1.000000 +92160 -1.000000 +92161 -1.000000 +92162 -1.000000 +92163 -1.000000 +92164 -1.000000 +92166 -1.000000 +92167 -1.000000 +92168 -1.000000 +92169 -1.000000 +92170 -1.000000 +92171 -1.000000 +92172 -1.000000 +92173 -1.000000 +92175 -1.000000 +92177 -1.000000 +92178 -1.000000 +92179 -1.000000 +92180 -1.000000 +92183 -1.000000 +92184 -1.000000 +92187 -1.000000 +92191 -1.000000 +92193 -1.000000 +92194 -1.000000 +92195 -1.000000 +92196 -1.000000 +92198 -1.000000 +92200 -1.000000 +92201 -1.000000 +92202 -1.000000 +92203 -1.000000 +92204 -1.000000 +92206 -1.000000 +92207 -1.000000 +92209 -1.000000 +92216 -1.000000 +92218 -1.000000 +92219 -1.000000 +92222 -1.000000 +92223 -1.000000 +92224 -1.000000 +92228 -1.000000 +92232 -1.000000 +92234 -1.000000 +92237 -1.000000 +92242 -1.000000 +92243 -1.000000 +92244 -1.000000 +92245 -1.000000 +92247 -1.000000 +92248 -1.000000 +92249 -1.000000 +92250 -1.000000 +92251 -1.000000 +92252 -1.000000 +92254 -1.000000 +92255 -1.000000 +92256 -1.000000 +92257 -1.000000 +92259 -1.000000 +92260 -1.000000 +92261 -1.000000 +92262 -1.000000 +92263 -1.000000 +92265 -1.000000 +92266 -1.000000 +92268 -1.000000 +92272 -1.000000 +92273 -1.000000 +92274 -1.000000 +92275 -1.000000 +92276 -1.000000 +92277 -1.000000 +92279 -1.000000 +92280 -1.000000 +92281 -1.000000 +92282 -1.000000 +92283 -1.000000 +92284 -1.000000 +92285 -1.000000 +92286 -1.000000 +92290 -1.000000 +92291 -1.000000 +92292 -1.000000 +92293 -1.000000 +92294 -1.000000 +92295 -1.000000 +92298 -1.000000 +92301 -1.000000 +92302 -1.000000 +92303 -1.000000 +92304 -1.000000 +92306 -1.000000 +92307 -1.000000 +92310 -1.000000 +92314 -1.000000 +92315 -1.000000 +92316 -1.000000 +92318 -1.000000 +92319 -1.000000 +92320 -1.000000 +92321 -1.000000 +92322 -1.000000 +92323 -1.000000 +92324 -1.000000 +92326 -1.000000 +92327 -1.000000 +92328 -1.000000 +92329 -1.000000 +92330 -1.000000 +92331 -1.000000 +92332 -1.000000 +92333 -1.000000 +92334 -1.000000 +92335 -1.000000 +92336 -1.000000 +92337 -1.000000 +92338 -1.000000 +92339 -1.000000 +92340 -1.000000 +92341 -1.000000 +92342 -1.000000 +92343 -1.000000 +92344 -1.000000 +92345 -1.000000 +92346 -1.000000 +92347 -1.000000 +92348 -1.000000 +92350 -1.000000 +92351 -1.000000 +92352 -1.000000 +92353 -1.000000 +92354 -1.000000 +92355 -1.000000 +92369 -1.000000 +92370 -1.000000 +92371 -1.000000 +92373 -1.000000 +92374 -1.000000 +92379 -1.000000 +92380 -1.000000 +92382 -1.000000 +92383 -1.000000 +92385 -1.000000 +92386 -1.000000 +92387 -1.000000 +92388 -1.000000 +92390 -1.000000 +92391 -1.000000 +92392 -1.000000 +92396 -1.000000 +92397 -1.000000 +92400 -1.000000 +92401 -1.000000 +92403 -1.000000 +92405 -1.000000 +92406 -1.000000 +92407 -1.000000 +92409 -1.000000 +92411 -1.000000 +92412 -1.000000 +92416 -1.000000 +92417 -1.000000 +92418 -1.000000 +92419 -1.000000 +92420 -1.000000 +92421 -1.000000 +92423 -1.000000 +92425 -1.000000 +92426 -1.000000 +92428 -1.000000 +92429 -1.000000 +92430 -1.000000 +92431 -1.000000 +92434 -1.000000 +92436 -1.000000 +92438 -1.000000 +92439 -1.000000 +92440 -1.000000 +92441 -1.000000 +92442 -1.000000 +92443 -1.000000 +92444 -1.000000 +92445 -1.000000 +92446 -1.000000 +92450 -1.000000 +92451 -1.000000 +92455 -1.000000 +92463 -1.000000 +92464 -1.000000 +92465 -1.000000 +92467 -1.000000 +92468 -1.000000 +92469 -1.000000 +92470 -1.000000 +92471 -1.000000 +92472 -1.000000 +92473 -1.000000 +92474 -1.000000 +92475 -1.000000 +92476 -1.000000 +92478 -1.000000 +92479 -1.000000 +92480 -1.000000 +92481 -1.000000 +92482 -1.000000 +92483 -1.000000 +92484 -1.000000 +92485 -1.000000 +92486 -1.000000 +92487 -1.000000 +92488 -1.000000 +92489 -1.000000 +92490 -1.000000 +92491 -1.000000 +92492 -1.000000 +92493 -1.000000 +92494 -1.000000 +92495 -1.000000 +92496 -1.000000 +92498 -1.000000 +92499 -1.000000 +92501 -1.000000 +92502 -1.000000 +92503 -1.000000 diff --git a/data/shot_lists/jet/ILW_clear_late.txt b/data/shot_lists/jet/ILW_clear_late.txt new file mode 100644 index 00000000..6f66467a --- /dev/null +++ b/data/shot_lists/jet/ILW_clear_late.txt @@ -0,0 +1,7201 @@ +83794 -1.000000 +83795 -1.000000 +83796 -1.000000 +83797 -1.000000 +83798 -1.000000 +83799 -1.000000 +83800 -1.000000 +83801 -1.000000 +83802 -1.000000 +83803 -1.000000 +83804 -1.000000 +83805 -1.000000 +83806 -1.000000 +83807 -1.000000 +83808 -1.000000 +83809 -1.000000 +83810 -1.000000 +83811 -1.000000 +83812 -1.000000 +83813 -1.000000 +83814 -1.000000 +83815 -1.000000 +83816 -1.000000 +83817 -1.000000 +83818 -1.000000 +83819 -1.000000 +83820 -1.000000 +83821 -1.000000 +83822 -1.000000 +83823 -1.000000 +83824 -1.000000 +83825 -1.000000 +83826 -1.000000 +83827 -1.000000 +83828 -1.000000 +83829 -1.000000 +83830 -1.000000 +83831 -1.000000 +83832 -1.000000 +83833 -1.000000 +83834 -1.000000 +83835 -1.000000 +83836 -1.000000 +83837 -1.000000 +83838 -1.000000 +83839 -1.000000 +83840 -1.000000 +83841 -1.000000 +83842 -1.000000 +83843 -1.000000 +83844 -1.000000 +83845 -1.000000 +83846 -1.000000 +83847 -1.000000 +83848 -1.000000 +83849 -1.000000 +83850 -1.000000 +83851 -1.000000 +83852 -1.000000 +83853 -1.000000 +83854 -1.000000 +83855 -1.000000 +83856 -1.000000 +83857 -1.000000 +83858 -1.000000 +83859 -1.000000 +83860 -1.000000 +83861 -1.000000 +83862 -1.000000 +83863 -1.000000 +83864 -1.000000 +83865 -1.000000 +83866 -1.000000 +83867 -1.000000 +83868 -1.000000 +83869 -1.000000 +83870 -1.000000 +83871 -1.000000 +83872 -1.000000 +83873 -1.000000 +83874 -1.000000 +83875 -1.000000 +83876 -1.000000 +83877 -1.000000 +83878 -1.000000 +83879 -1.000000 +83880 -1.000000 +83881 -1.000000 +83882 -1.000000 +83883 -1.000000 +83884 -1.000000 +83885 -1.000000 +83886 -1.000000 +83887 -1.000000 +83888 -1.000000 +83889 -1.000000 +83890 -1.000000 +83891 -1.000000 +83892 -1.000000 +83893 -1.000000 +83894 -1.000000 +83895 -1.000000 +83896 -1.000000 +83897 -1.000000 +83898 -1.000000 +83899 -1.000000 +83900 -1.000000 +83901 -1.000000 +83902 -1.000000 +83903 -1.000000 +83904 -1.000000 +83905 -1.000000 +83906 -1.000000 +83907 -1.000000 +83908 -1.000000 +83909 -1.000000 +83910 -1.000000 +83911 -1.000000 +83912 -1.000000 +83913 -1.000000 +83914 -1.000000 +83915 -1.000000 +83916 -1.000000 +83917 -1.000000 +83918 -1.000000 +83919 -1.000000 +83920 -1.000000 +83921 -1.000000 +83922 -1.000000 +83923 -1.000000 +83924 -1.000000 +83925 -1.000000 +83926 -1.000000 +83927 -1.000000 +83928 -1.000000 +83929 -1.000000 +83930 -1.000000 +83931 -1.000000 +83932 -1.000000 +83933 -1.000000 +83934 -1.000000 +83935 -1.000000 +83936 -1.000000 +83937 -1.000000 +83938 -1.000000 +83939 -1.000000 +83940 -1.000000 +83941 -1.000000 +83942 -1.000000 +83943 -1.000000 +83944 -1.000000 +83945 -1.000000 +83946 -1.000000 +83947 -1.000000 +83948 -1.000000 +83949 -1.000000 +83950 -1.000000 +83951 -1.000000 +83952 -1.000000 +83953 -1.000000 +83954 -1.000000 +83955 -1.000000 +83956 -1.000000 +83957 -1.000000 +83958 -1.000000 +83959 -1.000000 +83960 -1.000000 +83961 -1.000000 +83962 -1.000000 +83963 -1.000000 +83964 -1.000000 +83965 -1.000000 +83966 -1.000000 +83967 -1.000000 +83968 -1.000000 +83969 -1.000000 +83970 -1.000000 +83971 -1.000000 +83972 -1.000000 +83973 -1.000000 +83974 -1.000000 +83975 -1.000000 +83976 -1.000000 +83977 -1.000000 +83978 -1.000000 +83979 -1.000000 +83981 -1.000000 +83983 -1.000000 +83984 -1.000000 +83985 -1.000000 +83986 -1.000000 +83988 -1.000000 +83989 -1.000000 +83990 -1.000000 +83991 -1.000000 +83992 -1.000000 +83993 -1.000000 +83994 -1.000000 +83995 -1.000000 +83996 -1.000000 +83997 -1.000000 +83998 -1.000000 +84000 -1.000000 +84001 -1.000000 +84002 -1.000000 +84003 -1.000000 +84004 -1.000000 +84005 -1.000000 +84006 -1.000000 +84007 -1.000000 +84008 -1.000000 +84009 -1.000000 +84010 -1.000000 +84011 -1.000000 +84012 -1.000000 +84013 -1.000000 +84014 -1.000000 +84015 -1.000000 +84016 -1.000000 +84017 -1.000000 +84018 -1.000000 +84019 -1.000000 +84020 -1.000000 +84021 -1.000000 +84022 -1.000000 +84023 -1.000000 +84024 -1.000000 +84025 -1.000000 +84026 -1.000000 +84027 -1.000000 +84028 -1.000000 +84029 -1.000000 +84030 -1.000000 +84032 -1.000000 +84033 -1.000000 +84034 -1.000000 +84035 -1.000000 +84037 -1.000000 +84038 -1.000000 +84039 -1.000000 +84040 -1.000000 +84041 -1.000000 +84042 -1.000000 +84043 -1.000000 +84044 -1.000000 +84045 -1.000000 +84046 -1.000000 +84047 -1.000000 +84048 -1.000000 +84049 -1.000000 +84050 -1.000000 +84052 -1.000000 +84053 -1.000000 +84055 -1.000000 +84056 -1.000000 +84057 -1.000000 +84058 -1.000000 +84059 -1.000000 +84060 -1.000000 +84061 -1.000000 +84062 -1.000000 +84063 -1.000000 +84064 -1.000000 +84065 -1.000000 +84066 -1.000000 +84067 -1.000000 +84068 -1.000000 +84069 -1.000000 +84070 -1.000000 +84071 -1.000000 +84072 -1.000000 +84073 -1.000000 +84074 -1.000000 +84075 -1.000000 +84076 -1.000000 +84078 -1.000000 +84080 -1.000000 +84081 -1.000000 +84082 -1.000000 +84083 -1.000000 +84084 -1.000000 +84085 -1.000000 +84086 -1.000000 +84087 -1.000000 +84088 -1.000000 +84089 -1.000000 +84090 -1.000000 +84091 -1.000000 +84092 -1.000000 +84093 -1.000000 +84094 -1.000000 +84095 -1.000000 +84096 -1.000000 +84097 -1.000000 +84098 -1.000000 +84099 -1.000000 +84100 -1.000000 +84101 -1.000000 +84102 -1.000000 +84103 -1.000000 +84104 -1.000000 +84105 -1.000000 +84106 -1.000000 +84107 -1.000000 +84108 -1.000000 +84109 -1.000000 +84110 -1.000000 +84111 -1.000000 +84112 -1.000000 +84113 -1.000000 +84114 -1.000000 +84116 -1.000000 +84117 -1.000000 +84118 -1.000000 +84119 -1.000000 +84120 -1.000000 +84121 -1.000000 +84122 -1.000000 +84123 -1.000000 +84124 -1.000000 +84125 -1.000000 +84126 -1.000000 +84127 -1.000000 +84128 -1.000000 +84129 -1.000000 +84130 -1.000000 +84131 -1.000000 +84132 -1.000000 +84133 -1.000000 +84134 -1.000000 +84135 -1.000000 +84136 -1.000000 +84137 -1.000000 +84138 -1.000000 +84139 -1.000000 +84140 -1.000000 +84141 -1.000000 +84144 -1.000000 +84145 -1.000000 +84146 -1.000000 +84147 -1.000000 +84148 -1.000000 +84149 -1.000000 +84150 -1.000000 +84151 -1.000000 +84152 -1.000000 +84153 -1.000000 +84154 -1.000000 +84155 -1.000000 +84156 -1.000000 +84157 -1.000000 +84159 -1.000000 +84174 -1.000000 +84175 -1.000000 +84176 -1.000000 +84177 -1.000000 +84178 -1.000000 +84179 -1.000000 +84181 -1.000000 +84182 -1.000000 +84183 -1.000000 +84184 -1.000000 +84185 -1.000000 +84186 -1.000000 +84187 -1.000000 +84188 -1.000000 +84189 -1.000000 +84190 -1.000000 +84191 -1.000000 +84192 -1.000000 +84193 -1.000000 +84194 -1.000000 +84195 -1.000000 +84196 -1.000000 +84197 -1.000000 +84198 -1.000000 +84199 -1.000000 +84200 -1.000000 +84201 -1.000000 +84202 -1.000000 +84203 -1.000000 +84204 -1.000000 +84205 -1.000000 +84206 -1.000000 +84207 -1.000000 +84208 -1.000000 +84209 -1.000000 +84210 -1.000000 +84211 -1.000000 +84212 -1.000000 +84213 -1.000000 +84214 -1.000000 +84215 -1.000000 +84216 -1.000000 +84217 -1.000000 +84218 -1.000000 +84219 -1.000000 +84220 -1.000000 +84221 -1.000000 +84222 -1.000000 +84223 -1.000000 +84224 -1.000000 +84225 -1.000000 +84226 -1.000000 +84227 -1.000000 +84228 -1.000000 +84229 -1.000000 +84230 -1.000000 +84231 -1.000000 +84232 -1.000000 +84233 -1.000000 +84234 -1.000000 +84235 -1.000000 +84236 -1.000000 +84237 -1.000000 +84238 -1.000000 +84239 -1.000000 +84240 -1.000000 +84241 -1.000000 +84242 -1.000000 +84243 -1.000000 +84244 -1.000000 +84245 -1.000000 +84246 -1.000000 +84247 -1.000000 +84248 -1.000000 +84249 -1.000000 +84250 -1.000000 +84251 -1.000000 +84252 -1.000000 +84253 -1.000000 +84254 -1.000000 +84255 -1.000000 +84256 -1.000000 +84257 -1.000000 +84258 -1.000000 +84259 -1.000000 +84260 -1.000000 +84261 -1.000000 +84262 -1.000000 +84263 -1.000000 +84264 -1.000000 +84265 -1.000000 +84266 -1.000000 +84267 -1.000000 +84268 -1.000000 +84269 -1.000000 +84270 -1.000000 +84272 -1.000000 +84273 -1.000000 +84274 -1.000000 +84276 -1.000000 +84277 -1.000000 +84278 -1.000000 +84279 -1.000000 +84280 -1.000000 +84281 -1.000000 +84282 -1.000000 +84283 -1.000000 +84284 -1.000000 +84285 -1.000000 +84286 -1.000000 +84287 -1.000000 +84288 -1.000000 +84289 -1.000000 +84290 -1.000000 +84291 -1.000000 +84292 -1.000000 +84293 -1.000000 +84294 -1.000000 +84295 -1.000000 +84296 -1.000000 +84297 -1.000000 +84298 -1.000000 +84299 -1.000000 +84300 -1.000000 +84301 -1.000000 +84302 -1.000000 +84303 -1.000000 +84304 -1.000000 +84305 -1.000000 +84306 -1.000000 +84307 -1.000000 +84309 -1.000000 +84310 -1.000000 +84311 -1.000000 +84312 -1.000000 +84313 -1.000000 +84314 -1.000000 +84316 -1.000000 +84317 -1.000000 +84318 -1.000000 +84319 -1.000000 +84320 -1.000000 +84321 -1.000000 +84325 -1.000000 +84326 -1.000000 +84327 -1.000000 +84329 -1.000000 +84330 -1.000000 +84331 -1.000000 +84332 -1.000000 +84333 -1.000000 +84334 -1.000000 +84335 -1.000000 +84336 -1.000000 +84337 -1.000000 +84338 -1.000000 +84339 -1.000000 +84340 -1.000000 +84341 -1.000000 +84342 -1.000000 +84343 -1.000000 +84344 -1.000000 +84345 -1.000000 +84346 -1.000000 +84347 -1.000000 +84348 -1.000000 +84349 -1.000000 +84350 -1.000000 +84351 -1.000000 +84353 -1.000000 +84354 -1.000000 +84355 -1.000000 +84356 -1.000000 +84357 -1.000000 +84358 -1.000000 +84359 -1.000000 +84360 -1.000000 +84361 -1.000000 +84362 -1.000000 +84363 -1.000000 +84364 -1.000000 +84365 -1.000000 +84366 -1.000000 +84367 -1.000000 +84368 -1.000000 +84369 -1.000000 +84370 -1.000000 +84371 -1.000000 +84372 -1.000000 +84373 -1.000000 +84374 -1.000000 +84375 -1.000000 +84376 -1.000000 +84377 -1.000000 +84378 -1.000000 +84379 -1.000000 +84380 -1.000000 +84381 -1.000000 +84382 -1.000000 +84383 -1.000000 +84384 -1.000000 +84385 -1.000000 +84386 -1.000000 +84387 -1.000000 +84388 -1.000000 +84389 -1.000000 +84390 -1.000000 +84391 -1.000000 +84392 -1.000000 +84393 -1.000000 +84394 -1.000000 +84395 -1.000000 +84396 -1.000000 +84397 -1.000000 +84398 -1.000000 +84399 -1.000000 +84400 -1.000000 +84401 -1.000000 +84402 -1.000000 +84403 -1.000000 +84404 -1.000000 +84405 -1.000000 +84406 -1.000000 +84407 -1.000000 +84409 -1.000000 +84410 -1.000000 +84411 -1.000000 +84412 -1.000000 +84413 -1.000000 +84414 -1.000000 +84415 -1.000000 +84416 -1.000000 +84417 -1.000000 +84418 -1.000000 +84419 -1.000000 +84420 -1.000000 +84421 -1.000000 +84422 -1.000000 +84423 -1.000000 +84424 -1.000000 +84425 -1.000000 +84426 -1.000000 +84427 -1.000000 +84429 -1.000000 +84430 -1.000000 +84431 -1.000000 +84432 -1.000000 +84434 -1.000000 +84436 -1.000000 +84437 -1.000000 +84438 -1.000000 +84439 -1.000000 +84440 -1.000000 +84441 -1.000000 +84442 -1.000000 +84443 -1.000000 +84444 -1.000000 +84445 -1.000000 +84446 -1.000000 +84447 -1.000000 +84448 -1.000000 +84449 -1.000000 +84450 -1.000000 +84451 -1.000000 +84452 -1.000000 +84453 -1.000000 +84454 -1.000000 +84455 -1.000000 +84456 -1.000000 +84457 -1.000000 +84458 -1.000000 +84459 -1.000000 +84460 -1.000000 +84461 -1.000000 +84463 -1.000000 +84466 -1.000000 +84468 -1.000000 +84469 -1.000000 +84470 -1.000000 +84471 -1.000000 +84472 -1.000000 +84473 -1.000000 +84474 -1.000000 +84475 -1.000000 +84476 -1.000000 +84477 -1.000000 +84478 -1.000000 +84479 -1.000000 +84480 -1.000000 +84481 -1.000000 +84482 -1.000000 +84483 -1.000000 +84484 -1.000000 +84485 -1.000000 +84486 -1.000000 +84487 -1.000000 +84488 -1.000000 +84489 -1.000000 +84490 -1.000000 +84491 -1.000000 +84492 -1.000000 +84493 -1.000000 +84495 -1.000000 +84496 -1.000000 +84497 -1.000000 +84498 -1.000000 +84499 -1.000000 +84500 -1.000000 +84501 -1.000000 +84502 -1.000000 +84503 -1.000000 +84504 -1.000000 +84505 -1.000000 +84506 -1.000000 +84507 -1.000000 +84508 -1.000000 +84509 -1.000000 +84510 -1.000000 +84511 -1.000000 +84512 -1.000000 +84513 -1.000000 +84514 -1.000000 +84515 -1.000000 +84516 -1.000000 +84518 -1.000000 +84521 -1.000000 +84522 -1.000000 +84523 -1.000000 +84525 -1.000000 +84526 -1.000000 +84527 -1.000000 +84528 -1.000000 +84529 -1.000000 +84530 -1.000000 +84531 -1.000000 +84532 -1.000000 +84534 -1.000000 +84535 -1.000000 +84536 -1.000000 +84537 -1.000000 +84538 -1.000000 +84539 -1.000000 +84540 -1.000000 +84541 -1.000000 +84542 -1.000000 +84543 -1.000000 +84544 -1.000000 +84545 -1.000000 +84546 -1.000000 +84547 -1.000000 +84548 -1.000000 +84549 -1.000000 +84550 -1.000000 +84551 -1.000000 +84552 -1.000000 +84555 -1.000000 +84556 -1.000000 +84558 -1.000000 +84559 -1.000000 +84560 -1.000000 +84561 -1.000000 +84562 -1.000000 +84563 -1.000000 +84564 -1.000000 +84565 -1.000000 +84566 -1.000000 +84567 -1.000000 +84568 -1.000000 +84569 -1.000000 +84570 -1.000000 +84571 -1.000000 +84572 -1.000000 +84573 -1.000000 +84574 -1.000000 +84575 -1.000000 +84576 -1.000000 +84577 -1.000000 +84578 -1.000000 +84579 -1.000000 +84580 -1.000000 +84582 -1.000000 +84583 -1.000000 +84584 -1.000000 +84585 -1.000000 +84586 -1.000000 +84587 -1.000000 +84589 -1.000000 +84590 -1.000000 +84591 -1.000000 +84592 -1.000000 +84596 -1.000000 +84597 -1.000000 +84598 -1.000000 +84599 -1.000000 +84601 -1.000000 +84602 -1.000000 +84603 -1.000000 +84604 -1.000000 +84605 -1.000000 +84606 -1.000000 +84607 -1.000000 +84608 -1.000000 +84609 -1.000000 +84610 -1.000000 +84611 -1.000000 +84615 -1.000000 +84616 -1.000000 +84617 -1.000000 +84618 -1.000000 +84619 -1.000000 +84621 -1.000000 +84622 -1.000000 +84623 -1.000000 +84624 -1.000000 +84625 -1.000000 +84626 -1.000000 +84627 -1.000000 +84629 -1.000000 +84630 -1.000000 +84631 -1.000000 +84632 -1.000000 +84633 -1.000000 +84634 -1.000000 +84635 -1.000000 +84636 -1.000000 +84637 -1.000000 +84638 -1.000000 +84641 -1.000000 +84642 -1.000000 +84643 -1.000000 +84644 -1.000000 +84645 -1.000000 +84646 -1.000000 +84647 -1.000000 +84649 -1.000000 +84650 -1.000000 +84651 -1.000000 +84652 -1.000000 +84653 -1.000000 +84654 -1.000000 +84659 -1.000000 +84660 -1.000000 +84661 -1.000000 +84662 -1.000000 +84663 -1.000000 +84664 -1.000000 +84665 -1.000000 +84666 -1.000000 +84667 -1.000000 +84668 -1.000000 +84669 -1.000000 +84670 -1.000000 +84672 -1.000000 +84673 -1.000000 +84678 -1.000000 +84680 -1.000000 +84682 -1.000000 +84683 -1.000000 +84684 -1.000000 +84685 -1.000000 +84686 -1.000000 +84687 -1.000000 +84688 -1.000000 +84689 -1.000000 +84692 -1.000000 +84693 -1.000000 +84694 -1.000000 +84697 -1.000000 +84698 -1.000000 +84699 -1.000000 +84700 -1.000000 +84701 -1.000000 +84702 -1.000000 +84703 -1.000000 +84704 -1.000000 +84705 -1.000000 +84706 -1.000000 +84708 -1.000000 +84709 -1.000000 +84712 -1.000000 +84713 -1.000000 +84714 -1.000000 +84717 -1.000000 +84718 -1.000000 +84719 -1.000000 +84720 -1.000000 +84721 -1.000000 +84722 -1.000000 +84723 -1.000000 +84724 -1.000000 +84725 -1.000000 +84726 -1.000000 +84727 -1.000000 +84728 -1.000000 +84729 -1.000000 +84730 -1.000000 +84731 -1.000000 +84732 -1.000000 +84733 -1.000000 +84734 -1.000000 +84735 -1.000000 +84736 -1.000000 +84737 -1.000000 +84738 -1.000000 +84739 -1.000000 +84740 -1.000000 +84741 -1.000000 +84742 -1.000000 +84743 -1.000000 +84744 -1.000000 +84745 -1.000000 +84746 -1.000000 +84749 -1.000000 +84750 -1.000000 +84751 -1.000000 +84753 -1.000000 +84755 -1.000000 +84756 -1.000000 +84757 -1.000000 +84758 -1.000000 +84759 -1.000000 +84760 -1.000000 +84761 -1.000000 +84762 -1.000000 +84765 -1.000000 +84767 -1.000000 +84768 -1.000000 +84769 -1.000000 +84770 -1.000000 +84772 -1.000000 +84773 -1.000000 +84774 -1.000000 +84776 -1.000000 +84777 -1.000000 +84778 -1.000000 +84779 -1.000000 +84780 -1.000000 +84781 -1.000000 +84782 -1.000000 +84784 -1.000000 +84785 -1.000000 +84786 -1.000000 +84787 -1.000000 +84788 -1.000000 +84789 -1.000000 +84790 -1.000000 +84791 -1.000000 +84792 -1.000000 +84793 -1.000000 +84794 -1.000000 +84795 -1.000000 +84796 -1.000000 +84797 -1.000000 +84798 -1.000000 +84799 -1.000000 +84800 -1.000000 +84801 -1.000000 +84802 -1.000000 +84803 -1.000000 +84804 -1.000000 +84805 -1.000000 +84806 -1.000000 +84807 -1.000000 +84809 -1.000000 +84810 -1.000000 +84811 -1.000000 +84812 -1.000000 +84813 -1.000000 +84814 -1.000000 +84815 -1.000000 +84816 -1.000000 +84817 -1.000000 +84821 -1.000000 +84822 -1.000000 +84823 -1.000000 +84824 -1.000000 +84825 -1.000000 +84830 -1.000000 +84831 -1.000000 +84833 -1.000000 +84834 -1.000000 +84835 -1.000000 +84836 -1.000000 +84840 -1.000000 +84842 -1.000000 +84843 -1.000000 +84844 -1.000000 +84845 -1.000000 +84847 -1.000000 +84848 -1.000000 +84849 -1.000000 +84850 -1.000000 +84851 -1.000000 +84852 -1.000000 +84854 -1.000000 +84855 -1.000000 +84856 -1.000000 +84859 -1.000000 +84863 -1.000000 +84864 -1.000000 +84866 -1.000000 +84868 -1.000000 +84871 -1.000000 +84872 -1.000000 +84873 -1.000000 +84874 -1.000000 +84875 -1.000000 +84876 -1.000000 +84877 -1.000000 +84879 -1.000000 +84880 -1.000000 +84881 -1.000000 +84882 -1.000000 +84883 -1.000000 +84884 -1.000000 +84889 -1.000000 +84890 -1.000000 +84892 -1.000000 +84895 -1.000000 +84896 -1.000000 +84897 -1.000000 +84900 -1.000000 +84902 -1.000000 +84903 -1.000000 +84904 -1.000000 +84905 -1.000000 +84906 -1.000000 +84907 -1.000000 +84908 -1.000000 +84909 -1.000000 +84910 -1.000000 +84911 -1.000000 +84913 -1.000000 +84914 -1.000000 +84915 -1.000000 +84916 -1.000000 +84917 -1.000000 +84918 -1.000000 +84919 -1.000000 +84920 -1.000000 +84921 -1.000000 +84922 -1.000000 +84923 -1.000000 +84924 -1.000000 +84925 -1.000000 +84926 -1.000000 +84927 -1.000000 +84928 -1.000000 +84929 -1.000000 +84930 -1.000000 +84931 -1.000000 +84932 -1.000000 +84934 -1.000000 +84936 -1.000000 +84938 -1.000000 +84939 -1.000000 +84940 -1.000000 +84941 -1.000000 +84944 -1.000000 +84945 -1.000000 +84948 -1.000000 +84949 -1.000000 +84951 -1.000000 +84952 -1.000000 +84953 -1.000000 +84954 -1.000000 +84956 -1.000000 +84958 -1.000000 +84959 -1.000000 +84961 -1.000000 +84963 -1.000000 +84964 -1.000000 +84966 -1.000000 +84967 -1.000000 +84968 -1.000000 +84969 -1.000000 +84970 -1.000000 +84971 -1.000000 +84972 -1.000000 +84973 -1.000000 +84974 -1.000000 +84975 -1.000000 +84976 -1.000000 +84977 -1.000000 +84978 -1.000000 +84979 -1.000000 +84980 -1.000000 +84981 -1.000000 +84982 -1.000000 +84992 -1.000000 +84997 -1.000000 +85001 -1.000000 +85002 -1.000000 +85003 -1.000000 +85004 -1.000000 +85006 -1.000000 +85007 -1.000000 +85008 -1.000000 +85009 -1.000000 +85010 -1.000000 +85011 -1.000000 +85012 -1.000000 +85014 -1.000000 +85015 -1.000000 +85016 -1.000000 +85017 -1.000000 +85018 -1.000000 +85022 -1.000000 +85023 -1.000000 +85025 -1.000000 +85026 -1.000000 +85028 -1.000000 +85029 -1.000000 +85030 -1.000000 +85031 -1.000000 +85032 -1.000000 +85034 -1.000000 +85035 -1.000000 +85036 -1.000000 +85037 -1.000000 +85038 -1.000000 +85039 -1.000000 +85040 -1.000000 +85041 -1.000000 +85042 -1.000000 +85043 -1.000000 +85044 -1.000000 +85045 -1.000000 +85046 -1.000000 +85047 -1.000000 +85049 -1.000000 +85050 -1.000000 +85051 -1.000000 +85052 -1.000000 +85054 -1.000000 +85056 -1.000000 +85057 -1.000000 +85059 -1.000000 +85063 -1.000000 +85064 -1.000000 +85065 -1.000000 +85066 -1.000000 +85068 -1.000000 +85069 -1.000000 +85075 -1.000000 +85079 -1.000000 +85080 -1.000000 +85081 -1.000000 +85082 -1.000000 +85083 -1.000000 +85084 -1.000000 +85085 -1.000000 +85086 -1.000000 +85087 -1.000000 +85088 -1.000000 +85089 -1.000000 +85090 -1.000000 +85091 -1.000000 +85092 -1.000000 +85093 -1.000000 +85094 -1.000000 +85095 -1.000000 +85096 -1.000000 +85097 -1.000000 +85098 -1.000000 +85099 -1.000000 +85100 -1.000000 +85101 -1.000000 +85102 -1.000000 +85105 -1.000000 +85107 -1.000000 +85109 -1.000000 +85110 -1.000000 +85111 -1.000000 +85112 -1.000000 +85113 -1.000000 +85114 -1.000000 +85115 -1.000000 +85116 -1.000000 +85117 -1.000000 +85118 -1.000000 +85119 -1.000000 +85120 -1.000000 +85121 -1.000000 +85122 -1.000000 +85123 -1.000000 +85124 -1.000000 +85125 -1.000000 +85126 -1.000000 +85127 -1.000000 +85128 -1.000000 +85129 -1.000000 +85130 -1.000000 +85131 -1.000000 +85132 -1.000000 +85133 -1.000000 +85134 -1.000000 +85135 -1.000000 +85136 -1.000000 +85137 -1.000000 +85138 -1.000000 +85139 -1.000000 +85140 -1.000000 +85141 -1.000000 +85142 -1.000000 +85143 -1.000000 +85144 -1.000000 +85145 -1.000000 +85146 -1.000000 +85147 -1.000000 +85148 -1.000000 +85149 -1.000000 +85150 -1.000000 +85151 -1.000000 +85152 -1.000000 +85153 -1.000000 +85154 -1.000000 +85155 -1.000000 +85156 -1.000000 +85157 -1.000000 +85158 -1.000000 +85159 -1.000000 +85160 -1.000000 +85161 -1.000000 +85163 -1.000000 +85166 -1.000000 +85167 -1.000000 +85169 -1.000000 +85172 -1.000000 +85173 -1.000000 +85174 -1.000000 +85175 -1.000000 +85178 -1.000000 +85180 -1.000000 +85181 -1.000000 +85183 -1.000000 +85184 -1.000000 +85186 -1.000000 +85194 -1.000000 +85195 -1.000000 +85196 -1.000000 +85197 -1.000000 +85198 -1.000000 +85199 -1.000000 +85201 -1.000000 +85203 -1.000000 +85205 -1.000000 +85206 -1.000000 +85207 -1.000000 +85208 -1.000000 +85209 -1.000000 +85213 -1.000000 +85214 -1.000000 +85215 -1.000000 +85216 -1.000000 +85217 -1.000000 +85218 -1.000000 +85219 -1.000000 +85220 -1.000000 +85221 -1.000000 +85223 -1.000000 +85224 -1.000000 +85225 -1.000000 +85226 -1.000000 +85227 -1.000000 +85228 -1.000000 +85230 -1.000000 +85231 -1.000000 +85232 -1.000000 +85233 -1.000000 +85234 -1.000000 +85235 -1.000000 +85237 -1.000000 +85242 -1.000000 +85243 -1.000000 +85244 -1.000000 +85245 -1.000000 +85246 -1.000000 +85247 -1.000000 +85248 -1.000000 +85249 -1.000000 +85250 -1.000000 +85254 -1.000000 +85255 -1.000000 +85256 -1.000000 +85260 -1.000000 +85261 -1.000000 +85262 -1.000000 +85263 -1.000000 +85264 -1.000000 +85265 -1.000000 +85266 -1.000000 +85268 -1.000000 +85269 -1.000000 +85270 -1.000000 +85271 -1.000000 +85272 -1.000000 +85273 -1.000000 +85274 -1.000000 +85276 -1.000000 +85277 -1.000000 +85278 -1.000000 +85279 -1.000000 +85280 -1.000000 +85281 -1.000000 +85282 -1.000000 +85283 -1.000000 +85284 -1.000000 +85288 -1.000000 +85289 -1.000000 +85296 -1.000000 +85297 -1.000000 +85298 -1.000000 +85299 -1.000000 +85301 -1.000000 +85302 -1.000000 +85303 -1.000000 +85304 -1.000000 +85305 -1.000000 +85306 -1.000000 +85307 -1.000000 +85308 -1.000000 +85309 -1.000000 +85310 -1.000000 +85311 -1.000000 +85312 -1.000000 +85313 -1.000000 +85314 -1.000000 +85315 -1.000000 +85316 -1.000000 +85319 -1.000000 +85320 -1.000000 +85321 -1.000000 +85322 -1.000000 +85323 -1.000000 +85324 -1.000000 +85325 -1.000000 +85326 -1.000000 +85327 -1.000000 +85328 -1.000000 +85329 -1.000000 +85330 -1.000000 +85331 -1.000000 +85332 -1.000000 +85333 -1.000000 +85334 -1.000000 +85335 -1.000000 +85336 -1.000000 +85337 -1.000000 +85338 -1.000000 +85339 -1.000000 +85340 -1.000000 +85341 -1.000000 +85342 -1.000000 +85343 -1.000000 +85344 -1.000000 +85345 -1.000000 +85346 -1.000000 +85347 -1.000000 +85348 -1.000000 +85349 -1.000000 +85350 -1.000000 +85351 -1.000000 +85352 -1.000000 +85353 -1.000000 +85354 -1.000000 +85355 -1.000000 +85356 -1.000000 +85357 -1.000000 +85358 -1.000000 +85361 -1.000000 +85364 -1.000000 +85365 -1.000000 +85366 -1.000000 +85367 -1.000000 +85368 -1.000000 +85369 -1.000000 +85370 -1.000000 +85371 -1.000000 +85372 -1.000000 +85373 -1.000000 +85374 -1.000000 +85375 -1.000000 +85376 -1.000000 +85378 -1.000000 +85380 -1.000000 +85381 -1.000000 +85388 -1.000000 +85389 -1.000000 +85391 -1.000000 +85392 -1.000000 +85393 -1.000000 +85394 -1.000000 +85398 -1.000000 +85399 -1.000000 +85400 -1.000000 +85402 -1.000000 +85403 -1.000000 +85405 -1.000000 +85406 -1.000000 +85407 -1.000000 +85412 -1.000000 +85415 -1.000000 +85416 -1.000000 +85418 -1.000000 +85419 -1.000000 +85420 -1.000000 +85423 -1.000000 +85425 -1.000000 +85428 -1.000000 +85431 -1.000000 +85432 -1.000000 +85433 -1.000000 +85435 -1.000000 +85436 -1.000000 +85437 -1.000000 +85438 -1.000000 +85439 -1.000000 +85441 -1.000000 +85444 -1.000000 +85458 -1.000000 +85459 -1.000000 +85461 -1.000000 +85462 -1.000000 +85463 -1.000000 +85464 -1.000000 +85465 -1.000000 +85466 -1.000000 +85468 -1.000000 +85469 -1.000000 +85470 -1.000000 +85471 -1.000000 +85474 -1.000000 +85475 -1.000000 +85476 -1.000000 +85477 -1.000000 +85478 -1.000000 +85479 -1.000000 +85480 -1.000000 +85481 -1.000000 +85485 -1.000000 +85486 -1.000000 +85487 -1.000000 +85488 -1.000000 +85489 -1.000000 +85490 -1.000000 +85491 -1.000000 +85492 -1.000000 +85493 -1.000000 +85494 -1.000000 +85495 -1.000000 +85496 -1.000000 +85497 -1.000000 +85498 -1.000000 +85499 -1.000000 +85500 -1.000000 +85501 -1.000000 +85502 -1.000000 +85503 -1.000000 +85504 -1.000000 +85505 -1.000000 +85506 -1.000000 +85507 -1.000000 +85508 -1.000000 +85509 -1.000000 +85510 -1.000000 +85511 -1.000000 +85512 -1.000000 +85513 -1.000000 +85514 -1.000000 +85515 -1.000000 +85516 -1.000000 +85517 -1.000000 +85518 -1.000000 +85519 -1.000000 +85520 -1.000000 +85521 -1.000000 +85522 -1.000000 +85523 -1.000000 +85524 -1.000000 +85525 -1.000000 +85526 -1.000000 +85527 -1.000000 +85528 -1.000000 +85529 -1.000000 +85530 -1.000000 +85531 -1.000000 +85532 -1.000000 +85533 -1.000000 +85534 -1.000000 +85535 -1.000000 +85536 -1.000000 +85537 -1.000000 +85538 -1.000000 +85539 -1.000000 +85540 -1.000000 +85541 -1.000000 +85542 -1.000000 +85543 -1.000000 +85544 -1.000000 +85545 -1.000000 +85546 -1.000000 +85547 -1.000000 +85548 -1.000000 +85549 -1.000000 +85550 -1.000000 +85552 -1.000000 +85553 -1.000000 +85555 -1.000000 +85556 -1.000000 +85557 -1.000000 +85558 -1.000000 +85559 -1.000000 +85560 -1.000000 +85561 -1.000000 +85562 -1.000000 +85563 -1.000000 +85564 -1.000000 +85565 -1.000000 +85566 -1.000000 +85567 -1.000000 +85568 -1.000000 +85569 -1.000000 +85570 -1.000000 +85571 -1.000000 +85572 -1.000000 +85573 -1.000000 +85574 -1.000000 +85575 -1.000000 +85576 -1.000000 +85577 -1.000000 +85578 -1.000000 +85579 -1.000000 +85580 -1.000000 +85581 -1.000000 +85582 -1.000000 +85583 -1.000000 +85584 -1.000000 +85585 -1.000000 +85586 -1.000000 +85587 -1.000000 +85588 -1.000000 +85589 -1.000000 +85590 -1.000000 +85591 -1.000000 +85592 -1.000000 +85594 -1.000000 +85595 -1.000000 +85596 -1.000000 +85597 -1.000000 +85598 -1.000000 +85599 -1.000000 +85600 -1.000000 +85601 -1.000000 +85602 -1.000000 +85604 -1.000000 +85606 -1.000000 +85607 -1.000000 +85608 -1.000000 +85609 -1.000000 +85610 -1.000000 +85611 -1.000000 +85612 -1.000000 +85613 -1.000000 +85614 -1.000000 +85615 -1.000000 +85616 -1.000000 +85617 -1.000000 +85618 -1.000000 +85619 -1.000000 +85620 -1.000000 +85621 -1.000000 +85622 -1.000000 +85623 -1.000000 +85624 -1.000000 +85625 -1.000000 +85626 -1.000000 +85627 -1.000000 +85628 -1.000000 +85629 -1.000000 +85630 -1.000000 +85631 -1.000000 +85632 -1.000000 +85633 -1.000000 +85634 -1.000000 +85635 -1.000000 +85636 -1.000000 +85637 -1.000000 +85638 -1.000000 +85639 -1.000000 +85640 -1.000000 +85641 -1.000000 +85642 -1.000000 +85644 -1.000000 +85645 -1.000000 +85646 -1.000000 +85647 -1.000000 +85648 -1.000000 +85649 -1.000000 +85650 -1.000000 +85651 -1.000000 +85652 -1.000000 +85653 -1.000000 +85654 -1.000000 +85655 -1.000000 +85656 -1.000000 +85657 -1.000000 +85658 -1.000000 +85659 -1.000000 +85660 -1.000000 +85661 -1.000000 +85662 -1.000000 +85663 -1.000000 +85664 -1.000000 +85665 -1.000000 +85666 -1.000000 +85667 -1.000000 +85668 -1.000000 +85670 -1.000000 +85671 -1.000000 +85672 -1.000000 +85673 -1.000000 +85676 -1.000000 +85677 -1.000000 +85678 -1.000000 +85679 -1.000000 +85680 -1.000000 +85681 -1.000000 +85682 -1.000000 +85683 -1.000000 +85684 -1.000000 +85685 -1.000000 +85686 -1.000000 +85687 -1.000000 +85688 -1.000000 +85689 -1.000000 +85690 -1.000000 +85691 -1.000000 +85692 -1.000000 +85693 -1.000000 +85694 -1.000000 +85695 -1.000000 +85696 -1.000000 +85697 -1.000000 +85698 -1.000000 +85699 -1.000000 +85700 -1.000000 +85701 -1.000000 +85702 -1.000000 +85703 -1.000000 +85704 -1.000000 +85705 -1.000000 +85706 -1.000000 +85707 -1.000000 +85708 -1.000000 +85709 -1.000000 +85710 -1.000000 +85711 -1.000000 +85712 -1.000000 +85713 -1.000000 +85714 -1.000000 +85715 -1.000000 +85716 -1.000000 +85717 -1.000000 +85718 -1.000000 +85719 -1.000000 +85721 -1.000000 +85723 -1.000000 +85724 -1.000000 +85725 -1.000000 +85726 -1.000000 +85728 -1.000000 +85729 -1.000000 +85730 -1.000000 +85731 -1.000000 +85732 -1.000000 +85733 -1.000000 +85734 -1.000000 +85735 -1.000000 +85736 -1.000000 +85737 -1.000000 +85738 -1.000000 +85739 -1.000000 +85740 -1.000000 +85741 -1.000000 +85742 -1.000000 +85743 -1.000000 +85744 -1.000000 +85745 -1.000000 +85746 -1.000000 +85747 -1.000000 +85748 -1.000000 +85749 -1.000000 +85750 -1.000000 +85751 -1.000000 +85752 -1.000000 +85754 -1.000000 +85755 -1.000000 +85756 -1.000000 +85757 -1.000000 +85758 -1.000000 +85759 -1.000000 +85760 -1.000000 +85761 -1.000000 +85762 -1.000000 +85764 -1.000000 +85765 -1.000000 +85766 -1.000000 +85768 -1.000000 +85769 -1.000000 +85770 -1.000000 +85772 -1.000000 +85773 -1.000000 +85775 -1.000000 +85776 -1.000000 +85777 -1.000000 +85781 -1.000000 +85782 -1.000000 +85783 -1.000000 +85784 -1.000000 +85785 -1.000000 +85786 -1.000000 +85787 -1.000000 +85788 -1.000000 +85789 -1.000000 +85790 -1.000000 +85791 -1.000000 +85793 -1.000000 +85794 -1.000000 +85795 -1.000000 +85796 -1.000000 +85797 -1.000000 +85798 -1.000000 +85799 -1.000000 +85800 -1.000000 +85802 -1.000000 +85810 -1.000000 +85811 -1.000000 +85812 -1.000000 +85813 -1.000000 +85814 -1.000000 +85815 -1.000000 +85816 -1.000000 +85817 -1.000000 +85818 -1.000000 +85819 -1.000000 +85820 -1.000000 +85821 -1.000000 +85822 -1.000000 +85826 -1.000000 +85827 -1.000000 +85828 -1.000000 +85829 -1.000000 +85830 -1.000000 +85831 -1.000000 +85832 -1.000000 +85833 -1.000000 +85834 -1.000000 +85835 -1.000000 +85836 -1.000000 +85837 -1.000000 +85838 -1.000000 +85839 -1.000000 +85840 -1.000000 +85841 -1.000000 +85842 -1.000000 +85844 -1.000000 +85845 -1.000000 +85846 -1.000000 +85847 -1.000000 +85849 -1.000000 +85850 -1.000000 +85851 -1.000000 +85852 -1.000000 +85853 -1.000000 +85854 -1.000000 +85855 -1.000000 +85856 -1.000000 +85859 -1.000000 +85860 -1.000000 +85862 -1.000000 +85863 -1.000000 +85864 -1.000000 +85866 -1.000000 +85868 -1.000000 +85870 -1.000000 +85871 -1.000000 +85872 -1.000000 +85873 -1.000000 +85874 -1.000000 +85875 -1.000000 +85876 -1.000000 +85877 -1.000000 +85878 -1.000000 +85879 -1.000000 +85880 -1.000000 +85881 -1.000000 +85884 -1.000000 +85885 -1.000000 +85886 -1.000000 +85887 -1.000000 +85889 -1.000000 +85890 -1.000000 +85891 -1.000000 +85892 -1.000000 +85893 -1.000000 +85894 -1.000000 +85895 -1.000000 +85896 -1.000000 +85897 -1.000000 +85898 -1.000000 +85899 -1.000000 +85900 -1.000000 +85901 -1.000000 +85902 -1.000000 +85903 -1.000000 +85904 -1.000000 +85905 -1.000000 +85906 -1.000000 +85907 -1.000000 +85908 -1.000000 +85909 -1.000000 +85910 -1.000000 +85911 -1.000000 +85912 -1.000000 +85913 -1.000000 +85914 -1.000000 +85915 -1.000000 +85916 -1.000000 +85917 -1.000000 +85918 -1.000000 +85919 -1.000000 +85920 -1.000000 +85921 -1.000000 +85922 -1.000000 +85923 -1.000000 +85924 -1.000000 +85926 -1.000000 +85927 -1.000000 +85928 -1.000000 +85929 -1.000000 +85930 -1.000000 +85931 -1.000000 +85932 -1.000000 +85933 -1.000000 +85934 -1.000000 +85935 -1.000000 +85936 -1.000000 +85937 -1.000000 +85945 -1.000000 +85949 -1.000000 +85950 -1.000000 +85951 -1.000000 +85952 -1.000000 +85953 -1.000000 +85954 -1.000000 +85955 -1.000000 +85956 -1.000000 +85957 -1.000000 +85960 -1.000000 +85961 -1.000000 +85962 -1.000000 +85963 -1.000000 +85965 -1.000000 +85966 -1.000000 +85979 -1.000000 +85980 -1.000000 +85981 -1.000000 +85982 -1.000000 +85983 -1.000000 +85984 -1.000000 +85985 -1.000000 +85986 -1.000000 +85987 -1.000000 +85988 -1.000000 +85989 -1.000000 +85990 -1.000000 +85991 -1.000000 +85992 -1.000000 +85993 -1.000000 +85994 -1.000000 +85995 -1.000000 +85996 -1.000000 +85997 -1.000000 +85998 -1.000000 +85999 -1.000000 +86000 -1.000000 +86001 -1.000000 +86002 -1.000000 +86003 -1.000000 +86004 -1.000000 +86005 -1.000000 +86006 -1.000000 +86007 -1.000000 +86008 -1.000000 +86009 -1.000000 +86010 -1.000000 +86011 -1.000000 +86012 -1.000000 +86013 -1.000000 +86014 -1.000000 +86015 -1.000000 +86016 -1.000000 +86017 -1.000000 +86018 -1.000000 +86019 -1.000000 +86020 -1.000000 +86021 -1.000000 +86022 -1.000000 +86023 -1.000000 +86024 -1.000000 +86025 -1.000000 +86026 -1.000000 +86027 -1.000000 +86028 -1.000000 +86029 -1.000000 +86030 -1.000000 +86031 -1.000000 +86032 -1.000000 +86033 -1.000000 +86034 -1.000000 +86035 -1.000000 +86036 -1.000000 +86037 -1.000000 +86038 -1.000000 +86039 -1.000000 +86040 -1.000000 +86041 -1.000000 +86042 -1.000000 +86043 -1.000000 +86044 -1.000000 +86045 -1.000000 +86046 -1.000000 +86047 -1.000000 +86048 -1.000000 +86049 -1.000000 +86050 -1.000000 +86051 -1.000000 +86052 -1.000000 +86053 -1.000000 +86054 -1.000000 +86055 -1.000000 +86056 -1.000000 +86057 -1.000000 +86058 -1.000000 +86059 -1.000000 +86060 -1.000000 +86061 -1.000000 +86062 -1.000000 +86063 -1.000000 +86064 -1.000000 +86065 -1.000000 +86066 -1.000000 +86067 -1.000000 +86068 -1.000000 +86069 -1.000000 +86070 -1.000000 +86071 -1.000000 +86072 -1.000000 +86073 -1.000000 +86074 -1.000000 +86075 -1.000000 +86076 -1.000000 +86077 -1.000000 +86078 -1.000000 +86079 -1.000000 +86080 -1.000000 +86081 -1.000000 +86082 -1.000000 +86083 -1.000000 +86084 -1.000000 +86085 -1.000000 +86086 -1.000000 +86087 -1.000000 +86088 -1.000000 +86089 -1.000000 +86090 -1.000000 +86091 -1.000000 +86092 -1.000000 +86093 -1.000000 +86094 -1.000000 +86095 -1.000000 +86096 -1.000000 +86097 -1.000000 +86098 -1.000000 +86099 -1.000000 +86100 -1.000000 +86101 -1.000000 +86103 -1.000000 +86104 -1.000000 +86105 -1.000000 +86106 -1.000000 +86107 -1.000000 +86108 -1.000000 +86109 -1.000000 +86110 -1.000000 +86111 -1.000000 +86112 -1.000000 +86113 -1.000000 +86114 -1.000000 +86115 -1.000000 +86116 -1.000000 +86117 -1.000000 +86118 -1.000000 +86119 -1.000000 +86120 -1.000000 +86121 -1.000000 +86122 -1.000000 +86123 -1.000000 +86124 -1.000000 +86125 -1.000000 +86126 -1.000000 +86127 -1.000000 +86128 -1.000000 +86130 -1.000000 +86132 -1.000000 +86133 -1.000000 +86134 -1.000000 +86135 -1.000000 +86136 -1.000000 +86137 -1.000000 +86138 -1.000000 +86139 -1.000000 +86141 -1.000000 +86142 -1.000000 +86143 -1.000000 +86144 -1.000000 +86145 -1.000000 +86146 -1.000000 +86147 -1.000000 +86148 -1.000000 +86149 -1.000000 +86150 -1.000000 +86151 -1.000000 +86152 -1.000000 +86154 -1.000000 +86155 -1.000000 +86156 -1.000000 +86157 -1.000000 +86158 -1.000000 +86159 -1.000000 +86160 -1.000000 +86161 -1.000000 +86162 -1.000000 +86163 -1.000000 +86164 -1.000000 +86165 -1.000000 +86166 -1.000000 +86167 -1.000000 +86168 -1.000000 +86169 -1.000000 +86170 -1.000000 +86171 -1.000000 +86172 -1.000000 +86173 -1.000000 +86174 -1.000000 +86175 -1.000000 +86176 -1.000000 +86177 -1.000000 +86178 -1.000000 +86179 -1.000000 +86180 -1.000000 +86181 -1.000000 +86182 -1.000000 +86183 -1.000000 +86184 -1.000000 +86185 -1.000000 +86186 -1.000000 +86187 -1.000000 +86188 -1.000000 +86189 -1.000000 +86190 -1.000000 +86191 -1.000000 +86192 -1.000000 +86193 -1.000000 +86194 -1.000000 +86195 -1.000000 +86196 -1.000000 +86197 -1.000000 +86198 -1.000000 +86199 -1.000000 +86200 -1.000000 +86201 -1.000000 +86202 -1.000000 +86203 -1.000000 +86204 -1.000000 +86205 -1.000000 +86206 -1.000000 +86207 -1.000000 +86208 -1.000000 +86209 -1.000000 +86210 -1.000000 +86211 -1.000000 +86212 -1.000000 +86213 -1.000000 +86214 -1.000000 +86215 -1.000000 +86216 -1.000000 +86217 -1.000000 +86218 -1.000000 +86219 -1.000000 +86220 -1.000000 +86221 -1.000000 +86222 -1.000000 +86223 -1.000000 +86224 -1.000000 +86225 -1.000000 +86226 -1.000000 +86227 -1.000000 +86228 -1.000000 +86229 -1.000000 +86230 -1.000000 +86231 -1.000000 +86232 -1.000000 +86233 -1.000000 +86234 -1.000000 +86235 -1.000000 +86236 -1.000000 +86237 -1.000000 +86238 -1.000000 +86239 -1.000000 +86240 -1.000000 +86241 -1.000000 +86242 -1.000000 +86243 -1.000000 +86244 -1.000000 +86245 -1.000000 +86246 -1.000000 +86247 -1.000000 +86248 -1.000000 +86249 -1.000000 +86250 -1.000000 +86251 -1.000000 +86252 -1.000000 +86253 -1.000000 +86254 -1.000000 +86255 -1.000000 +86256 -1.000000 +86257 -1.000000 +86258 -1.000000 +86259 -1.000000 +86260 -1.000000 +86261 -1.000000 +86262 -1.000000 +86263 -1.000000 +86264 -1.000000 +86265 -1.000000 +86267 -1.000000 +86269 -1.000000 +86270 -1.000000 +86271 -1.000000 +86272 -1.000000 +86273 -1.000000 +86274 -1.000000 +86275 -1.000000 +86276 -1.000000 +86277 -1.000000 +86279 -1.000000 +86281 -1.000000 +86282 -1.000000 +86283 -1.000000 +86284 -1.000000 +86285 -1.000000 +86286 -1.000000 +86287 -1.000000 +86294 -1.000000 +86295 -1.000000 +86296 -1.000000 +86297 -1.000000 +86298 -1.000000 +86299 -1.000000 +86301 -1.000000 +86302 -1.000000 +86303 -1.000000 +86304 -1.000000 +86305 -1.000000 +86307 -1.000000 +86309 -1.000000 +86310 -1.000000 +86311 -1.000000 +86312 -1.000000 +86314 -1.000000 +86316 -1.000000 +86318 -1.000000 +86320 -1.000000 +86322 -1.000000 +86323 -1.000000 +86324 -1.000000 +86325 -1.000000 +86326 -1.000000 +86327 -1.000000 +86328 -1.000000 +86329 -1.000000 +86330 -1.000000 +86331 -1.000000 +86332 -1.000000 +86333 -1.000000 +86334 -1.000000 +86335 -1.000000 +86336 -1.000000 +86338 -1.000000 +86339 -1.000000 +86340 -1.000000 +86341 -1.000000 +86342 -1.000000 +86343 -1.000000 +86344 -1.000000 +86345 -1.000000 +86346 -1.000000 +86347 -1.000000 +86349 -1.000000 +86351 -1.000000 +86352 -1.000000 +86353 -1.000000 +86354 -1.000000 +86355 -1.000000 +86356 -1.000000 +86357 -1.000000 +86358 -1.000000 +86359 -1.000000 +86361 -1.000000 +86364 -1.000000 +86365 -1.000000 +86366 -1.000000 +86367 -1.000000 +86368 -1.000000 +86369 -1.000000 +86370 -1.000000 +86371 -1.000000 +86372 -1.000000 +86373 -1.000000 +86374 -1.000000 +86375 -1.000000 +86376 -1.000000 +86377 -1.000000 +86379 -1.000000 +86380 -1.000000 +86382 -1.000000 +86384 -1.000000 +86385 -1.000000 +86386 -1.000000 +86387 -1.000000 +86388 -1.000000 +86389 -1.000000 +86390 -1.000000 +86391 -1.000000 +86392 -1.000000 +86393 -1.000000 +86394 -1.000000 +86395 -1.000000 +86396 -1.000000 +86397 -1.000000 +86398 -1.000000 +86399 -1.000000 +86400 -1.000000 +86401 -1.000000 +86402 -1.000000 +86403 -1.000000 +86404 -1.000000 +86405 -1.000000 +86406 -1.000000 +86407 -1.000000 +86409 -1.000000 +86410 -1.000000 +86411 -1.000000 +86412 -1.000000 +86413 -1.000000 +86414 -1.000000 +86416 -1.000000 +86417 -1.000000 +86418 -1.000000 +86419 -1.000000 +86420 -1.000000 +86421 -1.000000 +86422 -1.000000 +86423 -1.000000 +86424 -1.000000 +86425 -1.000000 +86427 -1.000000 +86428 -1.000000 +86429 -1.000000 +86431 -1.000000 +86432 -1.000000 +86433 -1.000000 +86434 -1.000000 +86435 -1.000000 +86436 -1.000000 +86437 -1.000000 +86438 -1.000000 +86439 -1.000000 +86440 -1.000000 +86441 -1.000000 +86442 -1.000000 +86443 -1.000000 +86444 -1.000000 +86445 -1.000000 +86447 -1.000000 +86450 -1.000000 +86451 -1.000000 +86452 -1.000000 +86453 -1.000000 +86454 -1.000000 +86456 -1.000000 +86457 -1.000000 +86458 -1.000000 +86459 -1.000000 +86460 -1.000000 +86461 -1.000000 +86462 -1.000000 +86463 -1.000000 +86464 -1.000000 +86465 -1.000000 +86466 -1.000000 +86469 -1.000000 +86470 -1.000000 +86471 -1.000000 +86472 -1.000000 +86473 -1.000000 +86474 -1.000000 +86475 -1.000000 +86476 -1.000000 +86477 -1.000000 +86478 -1.000000 +86479 -1.000000 +86480 -1.000000 +86481 -1.000000 +86482 -1.000000 +86483 -1.000000 +86484 -1.000000 +86485 -1.000000 +86486 -1.000000 +86487 -1.000000 +86488 -1.000000 +86489 -1.000000 +86490 -1.000000 +86491 -1.000000 +86492 -1.000000 +86493 -1.000000 +86494 -1.000000 +86495 -1.000000 +86496 -1.000000 +86497 -1.000000 +86498 -1.000000 +86499 -1.000000 +86500 -1.000000 +86501 -1.000000 +86502 -1.000000 +86503 -1.000000 +86504 -1.000000 +86505 -1.000000 +86506 -1.000000 +86507 -1.000000 +86508 -1.000000 +86509 -1.000000 +86510 -1.000000 +86511 -1.000000 +86512 -1.000000 +86513 -1.000000 +86514 -1.000000 +86515 -1.000000 +86516 -1.000000 +86517 -1.000000 +86518 -1.000000 +86519 -1.000000 +86520 -1.000000 +86521 -1.000000 +86522 -1.000000 +86526 -1.000000 +86528 -1.000000 +86529 -1.000000 +86530 -1.000000 +86531 -1.000000 +86532 -1.000000 +86533 -1.000000 +86535 -1.000000 +86536 -1.000000 +86537 -1.000000 +86538 -1.000000 +86539 -1.000000 +86540 -1.000000 +86541 -1.000000 +86543 -1.000000 +86546 -1.000000 +86547 -1.000000 +86548 -1.000000 +86549 -1.000000 +86551 -1.000000 +86552 -1.000000 +86555 -1.000000 +86556 -1.000000 +86557 -1.000000 +86560 -1.000000 +86561 -1.000000 +86563 -1.000000 +86564 -1.000000 +86565 -1.000000 +86566 -1.000000 +86567 -1.000000 +86568 -1.000000 +86569 -1.000000 +86570 -1.000000 +86571 -1.000000 +86572 -1.000000 +86573 -1.000000 +86574 -1.000000 +86575 -1.000000 +86576 -1.000000 +86577 -1.000000 +86578 -1.000000 +86579 -1.000000 +86581 -1.000000 +86584 -1.000000 +86589 -1.000000 +86590 -1.000000 +86593 -1.000000 +86594 -1.000000 +86595 -1.000000 +86596 -1.000000 +86597 -1.000000 +86598 -1.000000 +86599 -1.000000 +86600 -1.000000 +86605 -1.000000 +86606 -1.000000 +86607 -1.000000 +86609 -1.000000 +86610 -1.000000 +86611 -1.000000 +86612 -1.000000 +86613 -1.000000 +86616 -1.000000 +86618 -1.000000 +86621 -1.000000 +86623 -1.000000 +86625 -1.000000 +86631 -1.000000 +86633 -1.000000 +86634 -1.000000 +86637 -1.000000 +86638 -1.000000 +86639 -1.000000 +86640 -1.000000 +86641 -1.000000 +86642 -1.000000 +86643 -1.000000 +86645 -1.000000 +86647 -1.000000 +86648 -1.000000 +86649 -1.000000 +86650 -1.000000 +86651 -1.000000 +86652 -1.000000 +86653 -1.000000 +86654 -1.000000 +86656 -1.000000 +86657 -1.000000 +86662 -1.000000 +86663 -1.000000 +86665 -1.000000 +86666 -1.000000 +86667 -1.000000 +86668 -1.000000 +86669 -1.000000 +86671 -1.000000 +86672 -1.000000 +86673 -1.000000 +86675 -1.000000 +86676 -1.000000 +86677 -1.000000 +86679 -1.000000 +86680 -1.000000 +86681 -1.000000 +86682 -1.000000 +86683 -1.000000 +86684 -1.000000 +86685 -1.000000 +86687 -1.000000 +86689 -1.000000 +86690 -1.000000 +86697 -1.000000 +86700 -1.000000 +86703 -1.000000 +86704 -1.000000 +86705 -1.000000 +86706 -1.000000 +86707 -1.000000 +86708 -1.000000 +86709 -1.000000 +86710 -1.000000 +86711 -1.000000 +86713 -1.000000 +86715 -1.000000 +86717 -1.000000 +86718 -1.000000 +86719 -1.000000 +86720 -1.000000 +86723 -1.000000 +86724 -1.000000 +86725 -1.000000 +86727 -1.000000 +86728 -1.000000 +86729 -1.000000 +86730 -1.000000 +86731 -1.000000 +86732 -1.000000 +86733 -1.000000 +86734 -1.000000 +86735 -1.000000 +86736 -1.000000 +86737 -1.000000 +86738 -1.000000 +86739 -1.000000 +86740 -1.000000 +86741 -1.000000 +86742 -1.000000 +86743 -1.000000 +86744 -1.000000 +86745 -1.000000 +86746 -1.000000 +86747 -1.000000 +86748 -1.000000 +86749 -1.000000 +86751 -1.000000 +86752 -1.000000 +86753 -1.000000 +86754 -1.000000 +86755 -1.000000 +86756 -1.000000 +86757 -1.000000 +86758 -1.000000 +86759 -1.000000 +86760 -1.000000 +86761 -1.000000 +86762 -1.000000 +86763 -1.000000 +86764 -1.000000 +86765 -1.000000 +86766 -1.000000 +86767 -1.000000 +86768 -1.000000 +86769 -1.000000 +86770 -1.000000 +86771 -1.000000 +86772 -1.000000 +86773 -1.000000 +86774 -1.000000 +86775 -1.000000 +86778 -1.000000 +86779 -1.000000 +86780 -1.000000 +86781 -1.000000 +86782 -1.000000 +86783 -1.000000 +86784 -1.000000 +86785 -1.000000 +86786 -1.000000 +86787 -1.000000 +86788 -1.000000 +86789 -1.000000 +86790 -1.000000 +86791 -1.000000 +86792 -1.000000 +86794 -1.000000 +86795 -1.000000 +86797 -1.000000 +86800 -1.000000 +86803 -1.000000 +86804 -1.000000 +86805 -1.000000 +86806 -1.000000 +86807 -1.000000 +86808 -1.000000 +86809 -1.000000 +86810 -1.000000 +86811 -1.000000 +86812 -1.000000 +86813 -1.000000 +86814 -1.000000 +86815 -1.000000 +86816 -1.000000 +86817 -1.000000 +86818 -1.000000 +86820 -1.000000 +86821 -1.000000 +86822 -1.000000 +86823 -1.000000 +86824 -1.000000 +86825 -1.000000 +86827 -1.000000 +86828 -1.000000 +86829 -1.000000 +86830 -1.000000 +86831 -1.000000 +86833 -1.000000 +86834 -1.000000 +86835 -1.000000 +86837 -1.000000 +86838 -1.000000 +86839 -1.000000 +86840 -1.000000 +86841 -1.000000 +86842 -1.000000 +86843 -1.000000 +86844 -1.000000 +86846 -1.000000 +86847 -1.000000 +86849 -1.000000 +86850 -1.000000 +86851 -1.000000 +86852 -1.000000 +86853 -1.000000 +86854 -1.000000 +86862 -1.000000 +86868 -1.000000 +86869 -1.000000 +86870 -1.000000 +86872 -1.000000 +86873 -1.000000 +86876 -1.000000 +86888 -1.000000 +86890 -1.000000 +86891 -1.000000 +86892 -1.000000 +86893 -1.000000 +86895 -1.000000 +86897 -1.000000 +86900 -1.000000 +86901 -1.000000 +86902 -1.000000 +86903 -1.000000 +86905 -1.000000 +86906 -1.000000 +86907 -1.000000 +86908 -1.000000 +86909 -1.000000 +86910 -1.000000 +86912 -1.000000 +86913 -1.000000 +86915 -1.000000 +86917 -1.000000 +86918 -1.000000 +86919 -1.000000 +86921 -1.000000 +86922 -1.000000 +86923 -1.000000 +86928 -1.000000 +86930 -1.000000 +86931 -1.000000 +86932 -1.000000 +86934 -1.000000 +86936 -1.000000 +86941 -1.000000 +86942 -1.000000 +86943 -1.000000 +86944 -1.000000 +86945 -1.000000 +86946 -1.000000 +86947 -1.000000 +86948 -1.000000 +86950 -1.000000 +86952 -1.000000 +86954 -1.000000 +86955 -1.000000 +86957 -1.000000 +86958 -1.000000 +86960 -1.000000 +86962 -1.000000 +86963 -1.000000 +86964 -1.000000 +86965 -1.000000 +86966 -1.000000 +86967 -1.000000 +86968 -1.000000 +86970 -1.000000 +86971 -1.000000 +86972 -1.000000 +86973 -1.000000 +86974 -1.000000 +86975 -1.000000 +86976 -1.000000 +86977 -1.000000 +86978 -1.000000 +86979 -1.000000 +86980 -1.000000 +86981 -1.000000 +86982 -1.000000 +86983 -1.000000 +86984 -1.000000 +86985 -1.000000 +86987 -1.000000 +86988 -1.000000 +86989 -1.000000 +86990 -1.000000 +86992 -1.000000 +86994 -1.000000 +86995 -1.000000 +86996 -1.000000 +86997 -1.000000 +86998 -1.000000 +86999 -1.000000 +87000 -1.000000 +87001 -1.000000 +87002 -1.000000 +87003 -1.000000 +87004 -1.000000 +87005 -1.000000 +87006 -1.000000 +87007 -1.000000 +87009 -1.000000 +87010 -1.000000 +87012 -1.000000 +87013 -1.000000 +87016 -1.000000 +87017 -1.000000 +87018 -1.000000 +87019 -1.000000 +87020 -1.000000 +87021 -1.000000 +87022 -1.000000 +87024 -1.000000 +87028 -1.000000 +87029 -1.000000 +87030 -1.000000 +87031 -1.000000 +87032 -1.000000 +87033 -1.000000 +87034 -1.000000 +87035 -1.000000 +87036 -1.000000 +87037 -1.000000 +87038 -1.000000 +87039 -1.000000 +87041 -1.000000 +87042 -1.000000 +87044 -1.000000 +87045 -1.000000 +87046 -1.000000 +87047 -1.000000 +87048 -1.000000 +87049 -1.000000 +87052 -1.000000 +87053 -1.000000 +87054 -1.000000 +87055 -1.000000 +87057 -1.000000 +87058 -1.000000 +87059 -1.000000 +87060 -1.000000 +87062 -1.000000 +87063 -1.000000 +87064 -1.000000 +87067 -1.000000 +87069 -1.000000 +87070 -1.000000 +87071 -1.000000 +87072 -1.000000 +87073 -1.000000 +87075 -1.000000 +87076 -1.000000 +87077 -1.000000 +87078 -1.000000 +87079 -1.000000 +87080 -1.000000 +87081 -1.000000 +87082 -1.000000 +87083 -1.000000 +87084 -1.000000 +87085 -1.000000 +87086 -1.000000 +87087 -1.000000 +87088 -1.000000 +87089 -1.000000 +87090 -1.000000 +87091 -1.000000 +87092 -1.000000 +87093 -1.000000 +87094 -1.000000 +87095 -1.000000 +87096 -1.000000 +87097 -1.000000 +87098 -1.000000 +87099 -1.000000 +87100 -1.000000 +87102 -1.000000 +87103 -1.000000 +87104 -1.000000 +87105 -1.000000 +87106 -1.000000 +87107 -1.000000 +87108 -1.000000 +87109 -1.000000 +87110 -1.000000 +87111 -1.000000 +87112 -1.000000 +87113 -1.000000 +87114 -1.000000 +87115 -1.000000 +87116 -1.000000 +87117 -1.000000 +87118 -1.000000 +87120 -1.000000 +87121 -1.000000 +87122 -1.000000 +87123 -1.000000 +87124 -1.000000 +87137 -1.000000 +87139 -1.000000 +87140 -1.000000 +87141 -1.000000 +87143 -1.000000 +87144 -1.000000 +87145 -1.000000 +87157 -1.000000 +87158 -1.000000 +87159 -1.000000 +87160 -1.000000 +87161 -1.000000 +87162 -1.000000 +87163 -1.000000 +87164 -1.000000 +87165 -1.000000 +87166 -1.000000 +87167 -1.000000 +87168 -1.000000 +87169 -1.000000 +87170 -1.000000 +87172 -1.000000 +87173 -1.000000 +87174 -1.000000 +87176 -1.000000 +87177 -1.000000 +87178 -1.000000 +87179 -1.000000 +87180 -1.000000 +87181 -1.000000 +87182 -1.000000 +87183 -1.000000 +87185 -1.000000 +87186 -1.000000 +87187 -1.000000 +87188 -1.000000 +87190 -1.000000 +87191 -1.000000 +87192 -1.000000 +87193 -1.000000 +87194 -1.000000 +87195 -1.000000 +87196 -1.000000 +87199 -1.000000 +87200 -1.000000 +87206 -1.000000 +87207 -1.000000 +87208 -1.000000 +87215 -1.000000 +87217 -1.000000 +87218 -1.000000 +87220 -1.000000 +87221 -1.000000 +87224 -1.000000 +87225 -1.000000 +87227 -1.000000 +87229 -1.000000 +87230 -1.000000 +87231 -1.000000 +87233 -1.000000 +87234 -1.000000 +87235 -1.000000 +87236 -1.000000 +87237 -1.000000 +87238 -1.000000 +87239 -1.000000 +87240 -1.000000 +87241 -1.000000 +87242 -1.000000 +87245 -1.000000 +87246 -1.000000 +87247 -1.000000 +87249 -1.000000 +87250 -1.000000 +87251 -1.000000 +87252 -1.000000 +87256 -1.000000 +87257 -1.000000 +87258 -1.000000 +87259 -1.000000 +87260 -1.000000 +87261 -1.000000 +87262 -1.000000 +87263 -1.000000 +87264 -1.000000 +87265 -1.000000 +87266 -1.000000 +87267 -1.000000 +87269 -1.000000 +87270 -1.000000 +87271 -1.000000 +87272 -1.000000 +87273 -1.000000 +87274 -1.000000 +87275 -1.000000 +87276 -1.000000 +87277 -1.000000 +87278 -1.000000 +87279 -1.000000 +87280 -1.000000 +87281 -1.000000 +87282 -1.000000 +87283 -1.000000 +87284 -1.000000 +87285 -1.000000 +87286 -1.000000 +87288 -1.000000 +87289 -1.000000 +87290 -1.000000 +87291 -1.000000 +87292 -1.000000 +87294 -1.000000 +87295 -1.000000 +87297 -1.000000 +87298 -1.000000 +87301 -1.000000 +87302 -1.000000 +87303 -1.000000 +87304 -1.000000 +87307 -1.000000 +87308 -1.000000 +87309 -1.000000 +87310 -1.000000 +87311 -1.000000 +87312 -1.000000 +87313 -1.000000 +87314 -1.000000 +87315 -1.000000 +87316 -1.000000 +87317 -1.000000 +87319 -1.000000 +87320 -1.000000 +87321 -1.000000 +87322 -1.000000 +87323 -1.000000 +87324 -1.000000 +87327 -1.000000 +87330 -1.000000 +87331 -1.000000 +87332 -1.000000 +87333 -1.000000 +87335 -1.000000 +87336 -1.000000 +87337 -1.000000 +87338 -1.000000 +87339 -1.000000 +87340 -1.000000 +87341 -1.000000 +87342 -1.000000 +87343 -1.000000 +87344 -1.000000 +87345 -1.000000 +87346 -1.000000 +87347 -1.000000 +87348 -1.000000 +87349 -1.000000 +87350 -1.000000 +87351 -1.000000 +87352 -1.000000 +87353 -1.000000 +87354 -1.000000 +87357 -1.000000 +87359 -1.000000 +87360 -1.000000 +87361 -1.000000 +87362 -1.000000 +87363 -1.000000 +87364 -1.000000 +87365 -1.000000 +87366 -1.000000 +87367 -1.000000 +87368 -1.000000 +87369 -1.000000 +87370 -1.000000 +87371 -1.000000 +87372 -1.000000 +87373 -1.000000 +87374 -1.000000 +87375 -1.000000 +87376 -1.000000 +87378 -1.000000 +87379 -1.000000 +87380 -1.000000 +87381 -1.000000 +87382 -1.000000 +87383 -1.000000 +87384 -1.000000 +87386 -1.000000 +87388 -1.000000 +87389 -1.000000 +87390 -1.000000 +87391 -1.000000 +87392 -1.000000 +87393 -1.000000 +87394 -1.000000 +87395 -1.000000 +87396 -1.000000 +87397 -1.000000 +87398 -1.000000 +87399 -1.000000 +87400 -1.000000 +87401 -1.000000 +87403 -1.000000 +87404 -1.000000 +87405 -1.000000 +87406 -1.000000 +87407 -1.000000 +87408 -1.000000 +87409 -1.000000 +87410 -1.000000 +87411 -1.000000 +87413 -1.000000 +87414 -1.000000 +87415 -1.000000 +87416 -1.000000 +87417 -1.000000 +87418 -1.000000 +87419 -1.000000 +87420 -1.000000 +87421 -1.000000 +87422 -1.000000 +87423 -1.000000 +87424 -1.000000 +87425 -1.000000 +87426 -1.000000 +87427 -1.000000 +87428 -1.000000 +87429 -1.000000 +87430 -1.000000 +87431 -1.000000 +87432 -1.000000 +87433 -1.000000 +87436 -1.000000 +87438 -1.000000 +87440 -1.000000 +87442 -1.000000 +87445 -1.000000 +87446 -1.000000 +87447 -1.000000 +87448 -1.000000 +87449 -1.000000 +87450 -1.000000 +87451 -1.000000 +87452 -1.000000 +87453 -1.000000 +87454 -1.000000 +87455 -1.000000 +87456 -1.000000 +87457 -1.000000 +87458 -1.000000 +87459 -1.000000 +87460 -1.000000 +87461 -1.000000 +87462 -1.000000 +87463 -1.000000 +87464 -1.000000 +87465 -1.000000 +87466 -1.000000 +87467 -1.000000 +87468 -1.000000 +87469 -1.000000 +87470 -1.000000 +87471 -1.000000 +87472 -1.000000 +87474 -1.000000 +87476 -1.000000 +87479 -1.000000 +87481 -1.000000 +87482 -1.000000 +87483 -1.000000 +87485 -1.000000 +87486 -1.000000 +87488 -1.000000 +87489 -1.000000 +87492 -1.000000 +87497 -1.000000 +87499 -1.000000 +87500 -1.000000 +87501 -1.000000 +87504 -1.000000 +87505 -1.000000 +87506 -1.000000 +87508 -1.000000 +87511 -1.000000 +87512 -1.000000 +87513 -1.000000 +87514 -1.000000 +87517 -1.000000 +87519 -1.000000 +87520 -1.000000 +87523 -1.000000 +87524 -1.000000 +87527 -1.000000 +87528 -1.000000 +87529 -1.000000 +87533 -1.000000 +87535 -1.000000 +87536 -1.000000 +87537 -1.000000 +87538 -1.000000 +87539 -1.000000 +87551 -1.000000 +87554 -1.000000 +87556 -1.000000 +87557 -1.000000 +87558 -1.000000 +87559 -1.000000 +87560 -1.000000 +87561 -1.000000 +87562 -1.000000 +87563 -1.000000 +87564 -1.000000 +87565 -1.000000 +87566 -1.000000 +87567 -1.000000 +87568 -1.000000 +87570 -1.000000 +87584 -1.000000 +87585 -1.000000 +87586 -1.000000 +87587 -1.000000 +87588 -1.000000 +87590 -1.000000 +87591 -1.000000 +87592 -1.000000 +87593 -1.000000 +87594 -1.000000 +87595 -1.000000 +87596 -1.000000 +87597 -1.000000 +87598 -1.000000 +87603 -1.000000 +87604 -1.000000 +87605 -1.000000 +87606 -1.000000 +87607 -1.000000 +87608 -1.000000 +87609 -1.000000 +87610 -1.000000 +87611 -1.000000 +87612 -1.000000 +87613 -1.000000 +87614 -1.000000 +87615 -1.000000 +87616 -1.000000 +87617 -1.000000 +87618 -1.000000 +87619 -1.000000 +87620 -1.000000 +87621 -1.000000 +87622 -1.000000 +87623 -1.000000 +87624 -1.000000 +87625 -1.000000 +87626 -1.000000 +87627 -1.000000 +87628 -1.000000 +87629 -1.000000 +87630 -1.000000 +87631 -1.000000 +87632 -1.000000 +87633 -1.000000 +87634 -1.000000 +87635 -1.000000 +87636 -1.000000 +87637 -1.000000 +87638 -1.000000 +87639 -1.000000 +87641 -1.000000 +87642 -1.000000 +87645 -1.000000 +87646 -1.000000 +87648 -1.000000 +87650 -1.000000 +87651 -1.000000 +87652 -1.000000 +87653 -1.000000 +87654 -1.000000 +87655 -1.000000 +87656 -1.000000 +87657 -1.000000 +87658 -1.000000 +87659 -1.000000 +87660 -1.000000 +87661 -1.000000 +87662 -1.000000 +87663 -1.000000 +87664 -1.000000 +87665 -1.000000 +87666 -1.000000 +87667 -1.000000 +87668 -1.000000 +87669 -1.000000 +87670 -1.000000 +87671 -1.000000 +87672 -1.000000 +87673 -1.000000 +87674 -1.000000 +87675 -1.000000 +87676 -1.000000 +87677 -1.000000 +87678 -1.000000 +87679 -1.000000 +87680 -1.000000 +87681 -1.000000 +87682 -1.000000 +87683 -1.000000 +87684 -1.000000 +87685 -1.000000 +87686 -1.000000 +87687 -1.000000 +87688 -1.000000 +87689 -1.000000 +87690 -1.000000 +87691 -1.000000 +87692 -1.000000 +87693 -1.000000 +87694 -1.000000 +87695 -1.000000 +87696 -1.000000 +87698 -1.000000 +87699 -1.000000 +87700 -1.000000 +87701 -1.000000 +87703 -1.000000 +87704 -1.000000 +87705 -1.000000 +87706 -1.000000 +87707 -1.000000 +87708 -1.000000 +87709 -1.000000 +87710 -1.000000 +87711 -1.000000 +87712 -1.000000 +87715 -1.000000 +87716 -1.000000 +87717 -1.000000 +87718 -1.000000 +87721 -1.000000 +87723 -1.000000 +87724 -1.000000 +87725 -1.000000 +87727 -1.000000 +87729 -1.000000 +87730 -1.000000 +87731 -1.000000 +87732 -1.000000 +87733 -1.000000 +87734 -1.000000 +87735 -1.000000 +87736 -1.000000 +87737 -1.000000 +87738 -1.000000 +87739 -1.000000 +87741 -1.000000 +87742 -1.000000 +87744 -1.000000 +87745 -1.000000 +87747 -1.000000 +87748 -1.000000 +87749 -1.000000 +87750 -1.000000 +87752 -1.000000 +87753 -1.000000 +87754 -1.000000 +87755 -1.000000 +87756 -1.000000 +87758 -1.000000 +87759 -1.000000 +87760 -1.000000 +87761 -1.000000 +87762 -1.000000 +87763 -1.000000 +87764 -1.000000 +87765 -1.000000 +87766 -1.000000 +87767 -1.000000 +87768 -1.000000 +87769 -1.000000 +87770 -1.000000 +87771 -1.000000 +87772 -1.000000 +87773 -1.000000 +87774 -1.000000 +87775 -1.000000 +87776 -1.000000 +87777 -1.000000 +87778 -1.000000 +87779 -1.000000 +87780 -1.000000 +87781 -1.000000 +87783 -1.000000 +87784 -1.000000 +87785 -1.000000 +87786 -1.000000 +87788 -1.000000 +87789 -1.000000 +87792 -1.000000 +87793 -1.000000 +87794 -1.000000 +87795 -1.000000 +87799 -1.000000 +87800 -1.000000 +87801 -1.000000 +87802 -1.000000 +87803 -1.000000 +87805 -1.000000 +87806 -1.000000 +87807 -1.000000 +87808 -1.000000 +87809 -1.000000 +87810 -1.000000 +87814 -1.000000 +87815 -1.000000 +87816 -1.000000 +87817 -1.000000 +87818 -1.000000 +87819 -1.000000 +87823 -1.000000 +87825 -1.000000 +87829 -1.000000 +87830 -1.000000 +87831 -1.000000 +87833 -1.000000 +87834 -1.000000 +87835 -1.000000 +87836 -1.000000 +87837 -1.000000 +87838 -1.000000 +87839 -1.000000 +87840 -1.000000 +87841 -1.000000 +87842 -1.000000 +87843 -1.000000 +87844 -1.000000 +87845 -1.000000 +87846 -1.000000 +87847 -1.000000 +87849 -1.000000 +87850 -1.000000 +87851 -1.000000 +87852 -1.000000 +87853 -1.000000 +87854 -1.000000 +87855 -1.000000 +87858 -1.000000 +87860 -1.000000 +87862 -1.000000 +87865 -1.000000 +87866 -1.000000 +87867 -1.000000 +87868 -1.000000 +87869 -1.000000 +87870 -1.000000 +87871 -1.000000 +87872 -1.000000 +87873 -1.000000 +87874 -1.000000 +87875 -1.000000 +87876 -1.000000 +87877 -1.000000 +87879 -1.000000 +87880 -1.000000 +87881 -1.000000 +87882 -1.000000 +87883 -1.000000 +87884 -1.000000 +87885 -1.000000 +87886 -1.000000 +87887 -1.000000 +87888 -1.000000 +87889 -1.000000 +87890 -1.000000 +87891 -1.000000 +87892 -1.000000 +87893 -1.000000 +87894 -1.000000 +87895 -1.000000 +87896 -1.000000 +87897 -1.000000 +87898 -1.000000 +87899 -1.000000 +87900 -1.000000 +87901 -1.000000 +87902 -1.000000 +87903 -1.000000 +87904 -1.000000 +87905 -1.000000 +87906 -1.000000 +87907 -1.000000 +87908 -1.000000 +87909 -1.000000 +87910 -1.000000 +87911 -1.000000 +87912 -1.000000 +87913 -1.000000 +87914 -1.000000 +87915 -1.000000 +87916 -1.000000 +87917 -1.000000 +87918 -1.000000 +87919 -1.000000 +87920 -1.000000 +87921 -1.000000 +87922 -1.000000 +87923 -1.000000 +87924 -1.000000 +87925 -1.000000 +87926 -1.000000 +87927 -1.000000 +87928 -1.000000 +87929 -1.000000 +87930 -1.000000 +87931 -1.000000 +87932 -1.000000 +87933 -1.000000 +87934 -1.000000 +87935 -1.000000 +87936 -1.000000 +87943 -1.000000 +87944 -1.000000 +87945 -1.000000 +87946 -1.000000 +87947 -1.000000 +87948 -1.000000 +87949 -1.000000 +87950 -1.000000 +87951 -1.000000 +87952 -1.000000 +87953 -1.000000 +87954 -1.000000 +87955 -1.000000 +87956 -1.000000 +87957 -1.000000 +87958 -1.000000 +87959 -1.000000 +87960 -1.000000 +87961 -1.000000 +87962 -1.000000 +87963 -1.000000 +87964 -1.000000 +87965 -1.000000 +87966 -1.000000 +87967 -1.000000 +87968 -1.000000 +87969 -1.000000 +87970 -1.000000 +87971 -1.000000 +87972 -1.000000 +87973 -1.000000 +87974 -1.000000 +87975 -1.000000 +87976 -1.000000 +87977 -1.000000 +87978 -1.000000 +87979 -1.000000 +87980 -1.000000 +87981 -1.000000 +87982 -1.000000 +87983 -1.000000 +87984 -1.000000 +87985 -1.000000 +87986 -1.000000 +87987 -1.000000 +87988 -1.000000 +87989 -1.000000 +87990 -1.000000 +87991 -1.000000 +87992 -1.000000 +87993 -1.000000 +87994 -1.000000 +87995 -1.000000 +87996 -1.000000 +87997 -1.000000 +87998 -1.000000 +87999 -1.000000 +88000 -1.000000 +88001 -1.000000 +88002 -1.000000 +88003 -1.000000 +88004 -1.000000 +88005 -1.000000 +88006 -1.000000 +88007 -1.000000 +88008 -1.000000 +88009 -1.000000 +88010 -1.000000 +88011 -1.000000 +88012 -1.000000 +88013 -1.000000 +88014 -1.000000 +88015 -1.000000 +88016 -1.000000 +88017 -1.000000 +88018 -1.000000 +88019 -1.000000 +88020 -1.000000 +88021 -1.000000 +88022 -1.000000 +88023 -1.000000 +88024 -1.000000 +88025 -1.000000 +88026 -1.000000 +88027 -1.000000 +88028 -1.000000 +88029 -1.000000 +88030 -1.000000 +88031 -1.000000 +88032 -1.000000 +88033 -1.000000 +88034 -1.000000 +88035 -1.000000 +88036 -1.000000 +88037 -1.000000 +88038 -1.000000 +88039 -1.000000 +88040 -1.000000 +88041 -1.000000 +88042 -1.000000 +88043 -1.000000 +88044 -1.000000 +88045 -1.000000 +88046 -1.000000 +88047 -1.000000 +88048 -1.000000 +88049 -1.000000 +88050 -1.000000 +88051 -1.000000 +88052 -1.000000 +88053 -1.000000 +88054 -1.000000 +88055 -1.000000 +88056 -1.000000 +88057 -1.000000 +88058 -1.000000 +88059 -1.000000 +88060 -1.000000 +88061 -1.000000 +88062 -1.000000 +88063 -1.000000 +88064 -1.000000 +88065 -1.000000 +88066 -1.000000 +88067 -1.000000 +88068 -1.000000 +88069 -1.000000 +88070 -1.000000 +88071 -1.000000 +88072 -1.000000 +88073 -1.000000 +88074 -1.000000 +88075 -1.000000 +88076 -1.000000 +88077 -1.000000 +88078 -1.000000 +88079 -1.000000 +88080 -1.000000 +88081 -1.000000 +88082 -1.000000 +88083 -1.000000 +88084 -1.000000 +88085 -1.000000 +88086 -1.000000 +88087 -1.000000 +88088 -1.000000 +88089 -1.000000 +88090 -1.000000 +88091 -1.000000 +88092 -1.000000 +88093 -1.000000 +88094 -1.000000 +88095 -1.000000 +88096 -1.000000 +88097 -1.000000 +88099 -1.000000 +88100 -1.000000 +88101 -1.000000 +88102 -1.000000 +88103 -1.000000 +88104 -1.000000 +88105 -1.000000 +88106 -1.000000 +88107 -1.000000 +88108 -1.000000 +88109 -1.000000 +88110 -1.000000 +88111 -1.000000 +88112 -1.000000 +88113 -1.000000 +88114 -1.000000 +88115 -1.000000 +88116 -1.000000 +88117 -1.000000 +88118 -1.000000 +88119 -1.000000 +88120 -1.000000 +88121 -1.000000 +88122 -1.000000 +88123 -1.000000 +88124 -1.000000 +88125 -1.000000 +88126 -1.000000 +88127 -1.000000 +88128 -1.000000 +88129 -1.000000 +88130 -1.000000 +88131 -1.000000 +88132 -1.000000 +88133 -1.000000 +88134 -1.000000 +88135 -1.000000 +88136 -1.000000 +88137 -1.000000 +88138 -1.000000 +88139 -1.000000 +88140 -1.000000 +88141 -1.000000 +88142 -1.000000 +88143 -1.000000 +88144 -1.000000 +88145 -1.000000 +88146 -1.000000 +88147 -1.000000 +88148 -1.000000 +88149 -1.000000 +88150 -1.000000 +88151 -1.000000 +88152 -1.000000 +88153 -1.000000 +88154 -1.000000 +88155 -1.000000 +88156 -1.000000 +88157 -1.000000 +88158 -1.000000 +88159 -1.000000 +88160 -1.000000 +88161 -1.000000 +88162 -1.000000 +88163 -1.000000 +88164 -1.000000 +88165 -1.000000 +88166 -1.000000 +88167 -1.000000 +88168 -1.000000 +88169 -1.000000 +88170 -1.000000 +88171 -1.000000 +88172 -1.000000 +88173 -1.000000 +88174 -1.000000 +88175 -1.000000 +88176 -1.000000 +88177 -1.000000 +88178 -1.000000 +88179 -1.000000 +88180 -1.000000 +88181 -1.000000 +88183 -1.000000 +88184 -1.000000 +88185 -1.000000 +88186 -1.000000 +88187 -1.000000 +88188 -1.000000 +88193 -1.000000 +88194 -1.000000 +88195 -1.000000 +88196 -1.000000 +88198 -1.000000 +88199 -1.000000 +88200 -1.000000 +88201 -1.000000 +88202 -1.000000 +88203 -1.000000 +88204 -1.000000 +88205 -1.000000 +88206 -1.000000 +88207 -1.000000 +88208 -1.000000 +88209 -1.000000 +88210 -1.000000 +88211 -1.000000 +88212 -1.000000 +88213 -1.000000 +88214 -1.000000 +88215 -1.000000 +88216 -1.000000 +88217 -1.000000 +88218 -1.000000 +88219 -1.000000 +88221 -1.000000 +88222 -1.000000 +88223 -1.000000 +88224 -1.000000 +88225 -1.000000 +88226 -1.000000 +88227 -1.000000 +88228 -1.000000 +88229 -1.000000 +88230 -1.000000 +88231 -1.000000 +88232 -1.000000 +88233 -1.000000 +88234 -1.000000 +88235 -1.000000 +88236 -1.000000 +88237 -1.000000 +88240 -1.000000 +88241 -1.000000 +88242 -1.000000 +88243 -1.000000 +88244 -1.000000 +88245 -1.000000 +88246 -1.000000 +88247 -1.000000 +88248 -1.000000 +88249 -1.000000 +88250 -1.000000 +88251 -1.000000 +88252 -1.000000 +88253 -1.000000 +88254 -1.000000 +88255 -1.000000 +88256 -1.000000 +88257 -1.000000 +88259 -1.000000 +88260 -1.000000 +88261 -1.000000 +88262 -1.000000 +88263 -1.000000 +88264 -1.000000 +88265 -1.000000 +88266 -1.000000 +88267 -1.000000 +88268 -1.000000 +88269 -1.000000 +88270 -1.000000 +88271 -1.000000 +88272 -1.000000 +88273 -1.000000 +88274 -1.000000 +88275 -1.000000 +88276 -1.000000 +88277 -1.000000 +88278 -1.000000 +88279 -1.000000 +88280 -1.000000 +88281 -1.000000 +88282 -1.000000 +88283 -1.000000 +88284 -1.000000 +88285 -1.000000 +88286 -1.000000 +88287 -1.000000 +88288 -1.000000 +88289 -1.000000 +88290 -1.000000 +88291 -1.000000 +88292 -1.000000 +88293 -1.000000 +88294 -1.000000 +88295 -1.000000 +88296 -1.000000 +88297 -1.000000 +88298 -1.000000 +88299 -1.000000 +88300 -1.000000 +88301 -1.000000 +88302 -1.000000 +88303 -1.000000 +88304 -1.000000 +88305 -1.000000 +88306 -1.000000 +88307 -1.000000 +88308 -1.000000 +88309 -1.000000 +88310 -1.000000 +88311 -1.000000 +88312 -1.000000 +88313 -1.000000 +88314 -1.000000 +88315 -1.000000 +88316 -1.000000 +88317 -1.000000 +88318 -1.000000 +88319 -1.000000 +88320 -1.000000 +88321 -1.000000 +88322 -1.000000 +88323 -1.000000 +88324 -1.000000 +88325 -1.000000 +88326 -1.000000 +88327 -1.000000 +88328 -1.000000 +88329 -1.000000 +88332 -1.000000 +88333 -1.000000 +88334 -1.000000 +88335 -1.000000 +88336 -1.000000 +88337 -1.000000 +88338 -1.000000 +88339 -1.000000 +88340 -1.000000 +88341 -1.000000 +88342 -1.000000 +88343 -1.000000 +88344 -1.000000 +88345 -1.000000 +88346 -1.000000 +88347 -1.000000 +88348 -1.000000 +88349 -1.000000 +88350 -1.000000 +88351 -1.000000 +88352 -1.000000 +88353 -1.000000 +88354 -1.000000 +88355 -1.000000 +88356 -1.000000 +88357 -1.000000 +88358 -1.000000 +88359 -1.000000 +88360 -1.000000 +88361 -1.000000 +88362 -1.000000 +88363 -1.000000 +88364 -1.000000 +88365 -1.000000 +88366 -1.000000 +88368 -1.000000 +88369 -1.000000 +88370 -1.000000 +88371 -1.000000 +88372 -1.000000 +88373 -1.000000 +88374 -1.000000 +88375 -1.000000 +88376 -1.000000 +88377 -1.000000 +88378 -1.000000 +88379 -1.000000 +88380 -1.000000 +88381 -1.000000 +88382 -1.000000 +88383 -1.000000 +88384 -1.000000 +88385 -1.000000 +88386 -1.000000 +88387 -1.000000 +88388 -1.000000 +88389 -1.000000 +88391 -1.000000 +88392 -1.000000 +88393 -1.000000 +88394 -1.000000 +88395 -1.000000 +88396 -1.000000 +88397 -1.000000 +88398 -1.000000 +88399 -1.000000 +88400 -1.000000 +88401 -1.000000 +88402 -1.000000 +88403 -1.000000 +88404 -1.000000 +88405 -1.000000 +88406 -1.000000 +88407 -1.000000 +88408 -1.000000 +88409 -1.000000 +88410 -1.000000 +88411 -1.000000 +88412 -1.000000 +88413 -1.000000 +88414 -1.000000 +88415 -1.000000 +88416 -1.000000 +88417 -1.000000 +88418 -1.000000 +88419 -1.000000 +88421 -1.000000 +88422 -1.000000 +88423 -1.000000 +88424 -1.000000 +88425 -1.000000 +88426 -1.000000 +88427 -1.000000 +88428 -1.000000 +88429 -1.000000 +88430 -1.000000 +88431 -1.000000 +88432 -1.000000 +88433 -1.000000 +88434 -1.000000 +88435 -1.000000 +88436 -1.000000 +88437 -1.000000 +88438 -1.000000 +88439 -1.000000 +88440 -1.000000 +88441 -1.000000 +88442 -1.000000 +88443 -1.000000 +88444 -1.000000 +88445 -1.000000 +88446 -1.000000 +88447 -1.000000 +88448 -1.000000 +88449 -1.000000 +88450 -1.000000 +88451 -1.000000 +88452 -1.000000 +88453 -1.000000 +88454 -1.000000 +88455 -1.000000 +88456 -1.000000 +88457 -1.000000 +88458 -1.000000 +88459 -1.000000 +88460 -1.000000 +88461 -1.000000 +88462 -1.000000 +88463 -1.000000 +88464 -1.000000 +88465 -1.000000 +88466 -1.000000 +88467 -1.000000 +88468 -1.000000 +88469 -1.000000 +88472 -1.000000 +88473 -1.000000 +88474 -1.000000 +88475 -1.000000 +88476 -1.000000 +88477 -1.000000 +88478 -1.000000 +88479 -1.000000 +88480 -1.000000 +88481 -1.000000 +88482 -1.000000 +88483 -1.000000 +88485 -1.000000 +88486 -1.000000 +88487 -1.000000 +88488 -1.000000 +88489 -1.000000 +88490 -1.000000 +88491 -1.000000 +88493 -1.000000 +88495 -1.000000 +88496 -1.000000 +88497 -1.000000 +88500 -1.000000 +88501 -1.000000 +88502 -1.000000 +88503 -1.000000 +88504 -1.000000 +88505 -1.000000 +88506 -1.000000 +88507 -1.000000 +88508 -1.000000 +88509 -1.000000 +88510 -1.000000 +88512 -1.000000 +88513 -1.000000 +88514 -1.000000 +88515 -1.000000 +88516 -1.000000 +88517 -1.000000 +88518 -1.000000 +88519 -1.000000 +88521 -1.000000 +88524 -1.000000 +88525 -1.000000 +88526 -1.000000 +88527 -1.000000 +88528 -1.000000 +88529 -1.000000 +88530 -1.000000 +88531 -1.000000 +88532 -1.000000 +88533 -1.000000 +88534 -1.000000 +88536 -1.000000 +88537 -1.000000 +88538 -1.000000 +88539 -1.000000 +88540 -1.000000 +88541 -1.000000 +88542 -1.000000 +88543 -1.000000 +88544 -1.000000 +88545 -1.000000 +88546 -1.000000 +88548 -1.000000 +88549 -1.000000 +88551 -1.000000 +88552 -1.000000 +88553 -1.000000 +88554 -1.000000 +88555 -1.000000 +88556 -1.000000 +88557 -1.000000 +88558 -1.000000 +88559 -1.000000 +88560 -1.000000 +88561 -1.000000 +88562 -1.000000 +88563 -1.000000 +88564 -1.000000 +88565 -1.000000 +88566 -1.000000 +88567 -1.000000 +88568 -1.000000 +88569 -1.000000 +88570 -1.000000 +88571 -1.000000 +88572 -1.000000 +88573 -1.000000 +88574 -1.000000 +88575 -1.000000 +88576 -1.000000 +88578 -1.000000 +88579 -1.000000 +88580 -1.000000 +88581 -1.000000 +88582 -1.000000 +88583 -1.000000 +88584 -1.000000 +88585 -1.000000 +88586 -1.000000 +88587 -1.000000 +88588 -1.000000 +88589 -1.000000 +88590 -1.000000 +88591 -1.000000 +88592 -1.000000 +88593 -1.000000 +88594 -1.000000 +88595 -1.000000 +88596 -1.000000 +88597 -1.000000 +88598 -1.000000 +88599 -1.000000 +88600 -1.000000 +88601 -1.000000 +88602 -1.000000 +88603 -1.000000 +88604 -1.000000 +88605 -1.000000 +88606 -1.000000 +88607 -1.000000 +88608 -1.000000 +88609 -1.000000 +88611 -1.000000 +88612 -1.000000 +88614 -1.000000 +88615 -1.000000 +88616 -1.000000 +88617 -1.000000 +88618 -1.000000 +88619 -1.000000 +88620 -1.000000 +88621 -1.000000 +88622 -1.000000 +88623 -1.000000 +88624 -1.000000 +88625 -1.000000 +88626 -1.000000 +88628 -1.000000 +88629 -1.000000 +88630 -1.000000 +88631 -1.000000 +88633 -1.000000 +88634 -1.000000 +88635 -1.000000 +88636 -1.000000 +88637 -1.000000 +88638 -1.000000 +88639 -1.000000 +88640 -1.000000 +88641 -1.000000 +88642 -1.000000 +88643 -1.000000 +88644 -1.000000 +88645 -1.000000 +88646 -1.000000 +88647 -1.000000 +88648 -1.000000 +88649 -1.000000 +88650 -1.000000 +88651 -1.000000 +88652 -1.000000 +88653 -1.000000 +88654 -1.000000 +88656 -1.000000 +88657 -1.000000 +88658 -1.000000 +88659 -1.000000 +88660 -1.000000 +88661 -1.000000 +88662 -1.000000 +88663 -1.000000 +88665 -1.000000 +88666 -1.000000 +88669 -1.000000 +88670 -1.000000 +88671 -1.000000 +88672 -1.000000 +88673 -1.000000 +88674 -1.000000 +88675 -1.000000 +88676 -1.000000 +88677 -1.000000 +88679 -1.000000 +88680 -1.000000 +88681 -1.000000 +88682 -1.000000 +88685 -1.000000 +88686 -1.000000 +88688 -1.000000 +88689 -1.000000 +88690 -1.000000 +88691 -1.000000 +88693 -1.000000 +88695 -1.000000 +88697 -1.000000 +88699 -1.000000 +88701 -1.000000 +88705 -1.000000 +88706 -1.000000 +88707 -1.000000 +88708 -1.000000 +88712 -1.000000 +88713 -1.000000 +88714 -1.000000 +88715 -1.000000 +88716 -1.000000 +88717 -1.000000 +88718 -1.000000 +88719 -1.000000 +88720 -1.000000 +88721 -1.000000 +88722 -1.000000 +88723 -1.000000 +88724 -1.000000 +88725 -1.000000 +88726 -1.000000 +88727 -1.000000 +88728 -1.000000 +88729 -1.000000 +88730 -1.000000 +88731 -1.000000 +88732 -1.000000 +88734 -1.000000 +88735 -1.000000 +88737 -1.000000 +88738 -1.000000 +88739 -1.000000 +88740 -1.000000 +88741 -1.000000 +88742 -1.000000 +88743 -1.000000 +88744 -1.000000 +88746 -1.000000 +88748 -1.000000 +88749 -1.000000 +88750 -1.000000 +88752 -1.000000 +88753 -1.000000 +88754 -1.000000 +88755 -1.000000 +88756 -1.000000 +88757 -1.000000 +88758 -1.000000 +88759 -1.000000 +88760 -1.000000 +88761 -1.000000 +88762 -1.000000 +88763 -1.000000 +88764 -1.000000 +88765 -1.000000 +88766 -1.000000 +88767 -1.000000 +88768 -1.000000 +88769 -1.000000 +88770 -1.000000 +88771 -1.000000 +88772 -1.000000 +88773 -1.000000 +88774 -1.000000 +88775 -1.000000 +88776 -1.000000 +88777 -1.000000 +88778 -1.000000 +88779 -1.000000 +88780 -1.000000 +88781 -1.000000 +88782 -1.000000 +88783 -1.000000 +88784 -1.000000 +88785 -1.000000 +88786 -1.000000 +88787 -1.000000 +88788 -1.000000 +88789 -1.000000 +88790 -1.000000 +88791 -1.000000 +88792 -1.000000 +88793 -1.000000 +88794 -1.000000 +88795 -1.000000 +88796 -1.000000 +88797 -1.000000 +88798 -1.000000 +88799 -1.000000 +88800 -1.000000 +88801 -1.000000 +88802 -1.000000 +88803 -1.000000 +88804 -1.000000 +88805 -1.000000 +88806 -1.000000 +88807 -1.000000 +88808 -1.000000 +88809 -1.000000 +88810 -1.000000 +88811 -1.000000 +88812 -1.000000 +88814 -1.000000 +88815 -1.000000 +88816 -1.000000 +88817 -1.000000 +88818 -1.000000 +88819 -1.000000 +88820 -1.000000 +88821 -1.000000 +88822 -1.000000 +88823 -1.000000 +88824 -1.000000 +88826 -1.000000 +88827 -1.000000 +88828 -1.000000 +88829 -1.000000 +88830 -1.000000 +88831 -1.000000 +88832 -1.000000 +88833 -1.000000 +88834 -1.000000 +88835 -1.000000 +88836 -1.000000 +88837 -1.000000 +88838 -1.000000 +88839 -1.000000 +88840 -1.000000 +88841 -1.000000 +88842 -1.000000 +88843 -1.000000 +88844 -1.000000 +88845 -1.000000 +88847 -1.000000 +88848 -1.000000 +88850 -1.000000 +88851 -1.000000 +88852 -1.000000 +88853 -1.000000 +88854 -1.000000 +88855 -1.000000 +88856 -1.000000 +88858 -1.000000 +88859 -1.000000 +88860 -1.000000 +88861 -1.000000 +88862 -1.000000 +88863 -1.000000 +88864 -1.000000 +88865 -1.000000 +88866 -1.000000 +88867 -1.000000 +88868 -1.000000 +88869 -1.000000 +88870 -1.000000 +88871 -1.000000 +88872 -1.000000 +88873 -1.000000 +88874 -1.000000 +88876 -1.000000 +88877 -1.000000 +88881 -1.000000 +88882 -1.000000 +88883 -1.000000 +88884 -1.000000 +88889 -1.000000 +88890 -1.000000 +88891 -1.000000 +88892 -1.000000 +88893 -1.000000 +88895 -1.000000 +88896 -1.000000 +88897 -1.000000 +88898 -1.000000 +88899 -1.000000 +88900 -1.000000 +88901 -1.000000 +88902 -1.000000 +88903 -1.000000 +88904 -1.000000 +88905 -1.000000 +88906 -1.000000 +88907 -1.000000 +88909 -1.000000 +88910 -1.000000 +88911 -1.000000 +88912 -1.000000 +88913 -1.000000 +88914 -1.000000 +88915 -1.000000 +88916 -1.000000 +88917 -1.000000 +88918 -1.000000 +88919 -1.000000 +88920 -1.000000 +88921 -1.000000 +88922 -1.000000 +88923 -1.000000 +88924 -1.000000 +88925 -1.000000 +88926 -1.000000 +88927 -1.000000 +88928 -1.000000 +88929 -1.000000 +88930 -1.000000 +88931 -1.000000 +88932 -1.000000 +88933 -1.000000 +88934 -1.000000 +88935 -1.000000 +88936 -1.000000 +88937 -1.000000 +88938 -1.000000 +88939 -1.000000 +88940 -1.000000 +88941 -1.000000 +88942 -1.000000 +88943 -1.000000 +88944 -1.000000 +88945 -1.000000 +88946 -1.000000 +88947 -1.000000 +88948 -1.000000 +88949 -1.000000 +88950 -1.000000 +88951 -1.000000 +88952 -1.000000 +88953 -1.000000 +88954 -1.000000 +88955 -1.000000 +88956 -1.000000 +88957 -1.000000 +88958 -1.000000 +88959 -1.000000 +88960 -1.000000 +88961 -1.000000 +88962 -1.000000 +88963 -1.000000 +88964 -1.000000 +88965 -1.000000 +88966 -1.000000 +88967 -1.000000 +88968 -1.000000 +88969 -1.000000 +88970 -1.000000 +88972 -1.000000 +88973 -1.000000 +88974 -1.000000 +88975 -1.000000 +88976 -1.000000 +88977 -1.000000 +88978 -1.000000 +88979 -1.000000 +88980 -1.000000 +88981 -1.000000 +88982 -1.000000 +88983 -1.000000 +88984 -1.000000 +88985 -1.000000 +88986 -1.000000 +88987 -1.000000 +88988 -1.000000 +88989 -1.000000 +88992 -1.000000 +88993 -1.000000 +88995 -1.000000 +88996 -1.000000 +88997 -1.000000 +88998 -1.000000 +88999 -1.000000 +89000 -1.000000 +89001 -1.000000 +89002 -1.000000 +89003 -1.000000 +89004 -1.000000 +89005 -1.000000 +89006 -1.000000 +89007 -1.000000 +89008 -1.000000 +89012 -1.000000 +89014 -1.000000 +89018 -1.000000 +89019 -1.000000 +89020 -1.000000 +89021 -1.000000 +89022 -1.000000 +89023 -1.000000 +89024 -1.000000 +89025 -1.000000 +89026 -1.000000 +89027 -1.000000 +89028 -1.000000 +89029 -1.000000 +89030 -1.000000 +89031 -1.000000 +89032 -1.000000 +89033 -1.000000 +89034 -1.000000 +89035 -1.000000 +89036 -1.000000 +89037 -1.000000 +89038 -1.000000 +89039 -1.000000 +89040 -1.000000 +89042 -1.000000 +89043 -1.000000 +89044 -1.000000 +89045 -1.000000 +89046 -1.000000 +89047 -1.000000 +89048 -1.000000 +89049 -1.000000 +89050 -1.000000 +89052 -1.000000 +89054 -1.000000 +89055 -1.000000 +89056 -1.000000 +89057 -1.000000 +89058 -1.000000 +89059 -1.000000 +89060 -1.000000 +89061 -1.000000 +89062 -1.000000 +89063 -1.000000 +89066 -1.000000 +89067 -1.000000 +89072 -1.000000 +89073 -1.000000 +89074 -1.000000 +89078 -1.000000 +89079 -1.000000 +89080 -1.000000 +89081 -1.000000 +89082 -1.000000 +89085 -1.000000 +89086 -1.000000 +89087 -1.000000 +89089 -1.000000 +89090 -1.000000 +89091 -1.000000 +89092 -1.000000 +89093 -1.000000 +89094 -1.000000 +89095 -1.000000 +89096 -1.000000 +89097 -1.000000 +89098 -1.000000 +89099 -1.000000 +89100 -1.000000 +89101 -1.000000 +89102 -1.000000 +89103 -1.000000 +89104 -1.000000 +89105 -1.000000 +89106 -1.000000 +89107 -1.000000 +89109 -1.000000 +89110 -1.000000 +89113 -1.000000 +89114 -1.000000 +89115 -1.000000 +89116 -1.000000 +89117 -1.000000 +89118 -1.000000 +89119 -1.000000 +89120 -1.000000 +89121 -1.000000 +89122 -1.000000 +89123 -1.000000 +89124 -1.000000 +89125 -1.000000 +89126 -1.000000 +89128 -1.000000 +89129 -1.000000 +89131 -1.000000 +89132 -1.000000 +89133 -1.000000 +89134 -1.000000 +89135 -1.000000 +89136 -1.000000 +89137 -1.000000 +89138 -1.000000 +89139 -1.000000 +89140 -1.000000 +89141 -1.000000 +89143 -1.000000 +89144 -1.000000 +89145 -1.000000 +89146 -1.000000 +89147 -1.000000 +89148 -1.000000 +89150 -1.000000 +89151 -1.000000 +89152 -1.000000 +89153 -1.000000 +89154 -1.000000 +89155 -1.000000 +89156 -1.000000 +89157 -1.000000 +89158 -1.000000 +89159 -1.000000 +89161 -1.000000 +89162 -1.000000 +89163 -1.000000 +89164 -1.000000 +89165 -1.000000 +89166 -1.000000 +89167 -1.000000 +89168 -1.000000 +89169 -1.000000 +89170 -1.000000 +89171 -1.000000 +89173 -1.000000 +89175 -1.000000 +89176 -1.000000 +89177 -1.000000 +89178 -1.000000 +89179 -1.000000 +89181 -1.000000 +89184 -1.000000 +89188 -1.000000 +89189 -1.000000 +89190 -1.000000 +89193 -1.000000 +89196 -1.000000 +89197 -1.000000 +89198 -1.000000 +89199 -1.000000 +89200 -1.000000 +89201 -1.000000 +89202 -1.000000 +89203 -1.000000 +89204 -1.000000 +89205 -1.000000 +89206 -1.000000 +89207 -1.000000 +89208 -1.000000 +89209 -1.000000 +89215 -1.000000 +89216 -1.000000 +89217 -1.000000 +89218 -1.000000 +89220 -1.000000 +89221 -1.000000 +89222 -1.000000 +89224 -1.000000 +89225 -1.000000 +89227 -1.000000 +89228 -1.000000 +89229 -1.000000 +89230 -1.000000 +89231 -1.000000 +89232 -1.000000 +89233 -1.000000 +89234 -1.000000 +89235 -1.000000 +89236 -1.000000 +89237 -1.000000 +89238 -1.000000 +89240 -1.000000 +89241 -1.000000 +89242 -1.000000 +89243 -1.000000 +89244 -1.000000 +89246 -1.000000 +89249 -1.000000 +89250 -1.000000 +89251 -1.000000 +89252 -1.000000 +89253 -1.000000 +89254 -1.000000 +89255 -1.000000 +89256 -1.000000 +89257 -1.000000 +89259 -1.000000 +89260 -1.000000 +89263 -1.000000 +89264 -1.000000 +89266 -1.000000 +89267 -1.000000 +89268 -1.000000 +89269 -1.000000 +89270 -1.000000 +89271 -1.000000 +89272 -1.000000 +89273 -1.000000 +89276 -1.000000 +89277 -1.000000 +89278 -1.000000 +89279 -1.000000 +89280 -1.000000 +89281 -1.000000 +89282 -1.000000 +89283 -1.000000 +89284 -1.000000 +89285 -1.000000 +89286 -1.000000 +89287 -1.000000 +89289 -1.000000 +89290 -1.000000 +89292 -1.000000 +89293 -1.000000 +89294 -1.000000 +89295 -1.000000 +89296 -1.000000 +89297 -1.000000 +89298 -1.000000 +89299 -1.000000 +89300 -1.000000 +89301 -1.000000 +89302 -1.000000 +89303 -1.000000 +89304 -1.000000 +89305 -1.000000 +89306 -1.000000 +89307 -1.000000 +89308 -1.000000 +89309 -1.000000 +89310 -1.000000 +89311 -1.000000 +89312 -1.000000 +89313 -1.000000 +89314 -1.000000 +89315 -1.000000 +89316 -1.000000 +89318 -1.000000 +89322 -1.000000 +89323 -1.000000 +89324 -1.000000 +89326 -1.000000 +89327 -1.000000 +89328 -1.000000 +89329 -1.000000 +89330 -1.000000 +89331 -1.000000 +89332 -1.000000 +89334 -1.000000 +89335 -1.000000 +89336 -1.000000 +89337 -1.000000 +89339 -1.000000 +89341 -1.000000 +89342 -1.000000 +89343 -1.000000 +89344 -1.000000 +89345 -1.000000 +89346 -1.000000 +89347 -1.000000 +89348 -1.000000 +89349 -1.000000 +89350 -1.000000 +89351 -1.000000 +89352 -1.000000 +89353 -1.000000 +89354 -1.000000 +89355 -1.000000 +89356 -1.000000 +89357 -1.000000 +89358 -1.000000 +89359 -1.000000 +89360 -1.000000 +89361 -1.000000 +89362 -1.000000 +89363 -1.000000 +89364 -1.000000 +89365 -1.000000 +89367 -1.000000 +89368 -1.000000 +89369 -1.000000 +89370 -1.000000 +89372 -1.000000 +89374 -1.000000 +89375 -1.000000 +89376 -1.000000 +89377 -1.000000 +89378 -1.000000 +89379 -1.000000 +89380 -1.000000 +89381 -1.000000 +89382 -1.000000 +89383 -1.000000 +89384 -1.000000 +89385 -1.000000 +89386 -1.000000 +89387 -1.000000 +89388 -1.000000 +89389 -1.000000 +89391 -1.000000 +89392 -1.000000 +89393 -1.000000 +89394 -1.000000 +89395 -1.000000 +89396 -1.000000 +89397 -1.000000 +89398 -1.000000 +89399 -1.000000 +89400 -1.000000 +89401 -1.000000 +89402 -1.000000 +89403 -1.000000 +89404 -1.000000 +89405 -1.000000 +89406 -1.000000 +89407 -1.000000 +89408 -1.000000 +89409 -1.000000 +89410 -1.000000 +89411 -1.000000 +89412 -1.000000 +89413 -1.000000 +89414 -1.000000 +89415 -1.000000 +89416 -1.000000 +89417 -1.000000 +89419 -1.000000 +89423 -1.000000 +89424 -1.000000 +89426 -1.000000 +89427 -1.000000 +89428 -1.000000 +89433 -1.000000 +89435 -1.000000 +89442 -1.000000 +89443 -1.000000 +89444 -1.000000 +89445 -1.000000 +89446 -1.000000 +89447 -1.000000 +89449 -1.000000 +89452 -1.000000 +89454 -1.000000 +89455 -1.000000 +89459 -1.000000 +89460 -1.000000 +89461 -1.000000 +89462 -1.000000 +89464 -1.000000 +89465 -1.000000 +89467 -1.000000 +89468 -1.000000 +89469 -1.000000 +89471 -1.000000 +89473 -1.000000 +89474 -1.000000 +89475 -1.000000 +89476 -1.000000 +89477 -1.000000 +89478 -1.000000 +89479 -1.000000 +89480 -1.000000 +89481 -1.000000 +89482 -1.000000 +89483 -1.000000 +89484 -1.000000 +89485 -1.000000 +89486 -1.000000 +89487 -1.000000 +89488 -1.000000 +89489 -1.000000 +89490 -1.000000 +89491 -1.000000 +89492 -1.000000 +89493 -1.000000 +89494 -1.000000 +89495 -1.000000 +89496 -1.000000 +89497 -1.000000 +89498 -1.000000 +89499 -1.000000 +89500 -1.000000 +89501 -1.000000 +89502 -1.000000 +89503 -1.000000 +89504 -1.000000 +89505 -1.000000 +89506 -1.000000 +89507 -1.000000 +89508 -1.000000 +89509 -1.000000 +89510 -1.000000 +89511 -1.000000 +89512 -1.000000 +89513 -1.000000 +89514 -1.000000 +89516 -1.000000 +89517 -1.000000 +89518 -1.000000 +89520 -1.000000 +89521 -1.000000 +89522 -1.000000 +89523 -1.000000 +89524 -1.000000 +89525 -1.000000 +89526 -1.000000 +89527 -1.000000 +89528 -1.000000 +89529 -1.000000 +89530 -1.000000 +89531 -1.000000 +89532 -1.000000 +89533 -1.000000 +89534 -1.000000 +89535 -1.000000 +89536 -1.000000 +89537 -1.000000 +89538 -1.000000 +89539 -1.000000 +89540 -1.000000 +89541 -1.000000 +89542 -1.000000 +89543 -1.000000 +89544 -1.000000 +89545 -1.000000 +89546 -1.000000 +89547 -1.000000 +89548 -1.000000 +89549 -1.000000 +89550 -1.000000 +89551 -1.000000 +89552 -1.000000 +89553 -1.000000 +89554 -1.000000 +89555 -1.000000 +89556 -1.000000 +89557 -1.000000 +89558 -1.000000 +89559 -1.000000 +89560 -1.000000 +89561 -1.000000 +89562 -1.000000 +89563 -1.000000 +89564 -1.000000 +89565 -1.000000 +89566 -1.000000 +89567 -1.000000 +89568 -1.000000 +89569 -1.000000 +89570 -1.000000 +89571 -1.000000 +89572 -1.000000 +89573 -1.000000 +89574 -1.000000 +89575 -1.000000 +89576 -1.000000 +89577 -1.000000 +89578 -1.000000 +89579 -1.000000 +89580 -1.000000 +89581 -1.000000 +89582 -1.000000 +89583 -1.000000 +89584 -1.000000 +89585 -1.000000 +89586 -1.000000 +89587 -1.000000 +89588 -1.000000 +89589 -1.000000 +89590 -1.000000 +89591 -1.000000 +89592 -1.000000 +89593 -1.000000 +89594 -1.000000 +89595 -1.000000 +89597 -1.000000 +89598 -1.000000 +89599 -1.000000 +89601 -1.000000 +89602 -1.000000 +89605 -1.000000 +89606 -1.000000 +89608 -1.000000 +89609 -1.000000 +89611 -1.000000 +89612 -1.000000 +89613 -1.000000 +89614 -1.000000 +89616 -1.000000 +89617 -1.000000 +89618 -1.000000 +89619 -1.000000 +89620 -1.000000 +89621 -1.000000 +89622 -1.000000 +89623 -1.000000 +89624 -1.000000 +89625 -1.000000 +89626 -1.000000 +89627 -1.000000 +89628 -1.000000 +89629 -1.000000 +89630 -1.000000 +89632 -1.000000 +89633 -1.000000 +89634 -1.000000 +89636 -1.000000 +89637 -1.000000 +89638 -1.000000 +89639 -1.000000 +89641 -1.000000 +89642 -1.000000 +89643 -1.000000 +89644 -1.000000 +89645 -1.000000 +89646 -1.000000 +89647 -1.000000 +89648 -1.000000 +89649 -1.000000 +89650 -1.000000 +89651 -1.000000 +89652 -1.000000 +89653 -1.000000 +89654 -1.000000 +89655 -1.000000 +89656 -1.000000 +89658 -1.000000 +89661 -1.000000 +89662 -1.000000 +89663 -1.000000 +89664 -1.000000 +89665 -1.000000 +89666 -1.000000 +89669 -1.000000 +89670 -1.000000 +89671 -1.000000 +89672 -1.000000 +89673 -1.000000 +89674 -1.000000 +89675 -1.000000 +89676 -1.000000 +89677 -1.000000 +89678 -1.000000 +89679 -1.000000 +89680 -1.000000 +89681 -1.000000 +89682 -1.000000 +89683 -1.000000 +89684 -1.000000 +89685 -1.000000 +89686 -1.000000 +89687 -1.000000 +89688 -1.000000 +89689 -1.000000 +89690 -1.000000 +89691 -1.000000 +89692 -1.000000 +89693 -1.000000 +89694 -1.000000 +89695 -1.000000 +89696 -1.000000 +89697 -1.000000 +89698 -1.000000 +89699 -1.000000 +89700 -1.000000 +89701 -1.000000 +89702 -1.000000 +89703 -1.000000 +89704 -1.000000 +89705 -1.000000 +89707 -1.000000 +89708 -1.000000 +89709 -1.000000 +89710 -1.000000 +89711 -1.000000 +89712 -1.000000 +89713 -1.000000 +89714 -1.000000 +89715 -1.000000 +89716 -1.000000 +89717 -1.000000 +89718 -1.000000 +89719 -1.000000 +89720 -1.000000 +89721 -1.000000 +89722 -1.000000 +89723 -1.000000 +89724 -1.000000 +89725 -1.000000 +89726 -1.000000 +89727 -1.000000 +89728 -1.000000 +89729 -1.000000 +89730 -1.000000 +89731 -1.000000 +89732 -1.000000 +89734 -1.000000 +89735 -1.000000 +89736 -1.000000 +89737 -1.000000 +89738 -1.000000 +89739 -1.000000 +89740 -1.000000 +89741 -1.000000 +89742 -1.000000 +89743 -1.000000 +89746 -1.000000 +89747 -1.000000 +89748 -1.000000 +89749 -1.000000 +89750 -1.000000 +89751 -1.000000 +89752 -1.000000 +89754 -1.000000 +89756 -1.000000 +89757 -1.000000 +89758 -1.000000 +89759 -1.000000 +89767 -1.000000 +89775 -1.000000 +89776 -1.000000 +89777 -1.000000 +89778 -1.000000 +89779 -1.000000 +89781 -1.000000 +89783 -1.000000 +89784 -1.000000 +89786 -1.000000 +89787 -1.000000 +89788 -1.000000 +89789 -1.000000 +89790 -1.000000 +89791 -1.000000 +89792 -1.000000 +89793 -1.000000 +89794 -1.000000 +89807 -1.000000 +89808 -1.000000 +89809 -1.000000 +89810 -1.000000 +89811 -1.000000 +89812 -1.000000 +89813 -1.000000 +89814 -1.000000 +89815 -1.000000 +89818 -1.000000 +89819 -1.000000 +89820 -1.000000 +89821 -1.000000 +89822 -1.000000 +89823 -1.000000 +89824 -1.000000 +89825 -1.000000 +89826 -1.000000 +89827 -1.000000 +89828 -1.000000 +89829 -1.000000 +89830 -1.000000 +89831 -1.000000 +89832 -1.000000 +89833 -1.000000 +89834 -1.000000 +89835 -1.000000 +89836 -1.000000 +89837 -1.000000 +89838 -1.000000 +89839 -1.000000 +89840 -1.000000 +89841 -1.000000 +89842 -1.000000 +89844 -1.000000 +89845 -1.000000 +89846 -1.000000 +89847 -1.000000 +89848 -1.000000 +89849 -1.000000 +89850 -1.000000 +89851 -1.000000 +89852 -1.000000 +89853 -1.000000 +89854 -1.000000 +89855 -1.000000 +89856 -1.000000 +89857 -1.000000 +89858 -1.000000 +89859 -1.000000 +89860 -1.000000 +89861 -1.000000 +89862 -1.000000 +89863 -1.000000 +89865 -1.000000 +89866 -1.000000 +89867 -1.000000 +89868 -1.000000 +89869 -1.000000 +89871 -1.000000 +89872 -1.000000 +89873 -1.000000 +89874 -1.000000 +89875 -1.000000 +89876 -1.000000 +89877 -1.000000 +89878 -1.000000 +89879 -1.000000 +89880 -1.000000 +89881 -1.000000 +89882 -1.000000 +89883 -1.000000 +89884 -1.000000 +89886 -1.000000 +89887 -1.000000 +89888 -1.000000 +89889 -1.000000 +89890 -1.000000 +89891 -1.000000 +89893 -1.000000 +89894 -1.000000 +89895 -1.000000 +89896 -1.000000 +89897 -1.000000 +89898 -1.000000 +89899 -1.000000 +89900 -1.000000 +89901 -1.000000 +89902 -1.000000 +89903 -1.000000 +89905 -1.000000 +89906 -1.000000 +89907 -1.000000 +89908 -1.000000 +89909 -1.000000 +89910 -1.000000 +89911 -1.000000 +89912 -1.000000 +89913 -1.000000 +89914 -1.000000 +89916 -1.000000 +89917 -1.000000 +89918 -1.000000 +89919 -1.000000 +89920 -1.000000 +89921 -1.000000 +89922 -1.000000 +89923 -1.000000 +89924 -1.000000 +89925 -1.000000 +89926 -1.000000 +89927 -1.000000 +89928 -1.000000 +89929 -1.000000 +89930 -1.000000 +89931 -1.000000 +89932 -1.000000 +89933 -1.000000 +89934 -1.000000 +89935 -1.000000 +89936 -1.000000 +89937 -1.000000 +89938 -1.000000 +89939 -1.000000 +89940 -1.000000 +89941 -1.000000 +89942 -1.000000 +89943 -1.000000 +89944 -1.000000 +89945 -1.000000 +89946 -1.000000 +89947 -1.000000 +89948 -1.000000 +89949 -1.000000 +89951 -1.000000 +89952 -1.000000 +89953 -1.000000 +89954 -1.000000 +89955 -1.000000 +89956 -1.000000 +89957 -1.000000 +89958 -1.000000 +89959 -1.000000 +89960 -1.000000 +89961 -1.000000 +89962 -1.000000 +89963 -1.000000 +89964 -1.000000 +89965 -1.000000 +89966 -1.000000 +89967 -1.000000 +89968 -1.000000 +89969 -1.000000 +89970 -1.000000 +89971 -1.000000 +89972 -1.000000 +89973 -1.000000 +89974 -1.000000 +89975 -1.000000 +89976 -1.000000 +89977 -1.000000 +89978 -1.000000 +89979 -1.000000 +89980 -1.000000 +89981 -1.000000 +89982 -1.000000 +89983 -1.000000 +89984 -1.000000 +89985 -1.000000 +89986 -1.000000 +89987 -1.000000 +89988 -1.000000 +89989 -1.000000 +89990 -1.000000 +89991 -1.000000 +89992 -1.000000 +89993 -1.000000 +89994 -1.000000 +89995 -1.000000 +89996 -1.000000 +89997 -1.000000 +89999 -1.000000 +90000 -1.000000 +90001 -1.000000 +90002 -1.000000 +90003 -1.000000 +90004 -1.000000 +90005 -1.000000 +90006 -1.000000 +90007 -1.000000 +90008 -1.000000 +90009 -1.000000 +90010 -1.000000 +90011 -1.000000 +90012 -1.000000 +90013 -1.000000 +90014 -1.000000 +90015 -1.000000 +90016 -1.000000 +90017 -1.000000 +90018 -1.000000 +90019 -1.000000 +90020 -1.000000 +90021 -1.000000 +90022 -1.000000 +90023 -1.000000 +90024 -1.000000 +90025 -1.000000 +90026 -1.000000 +90027 -1.000000 +90028 -1.000000 +90029 -1.000000 +90030 -1.000000 +90032 -1.000000 +90033 -1.000000 +90034 -1.000000 +90035 -1.000000 +90036 -1.000000 +90038 -1.000000 +90039 -1.000000 +90040 -1.000000 +90041 -1.000000 +90042 -1.000000 +90043 -1.000000 +90044 -1.000000 +90045 -1.000000 +90046 -1.000000 +90050 -1.000000 +90051 -1.000000 +90052 -1.000000 +90053 -1.000000 +90054 -1.000000 +90055 -1.000000 +90056 -1.000000 +90057 -1.000000 +90058 -1.000000 +90059 -1.000000 +90060 -1.000000 +90061 -1.000000 +90062 -1.000000 +90063 -1.000000 +90064 -1.000000 +90065 -1.000000 +90066 -1.000000 +90067 -1.000000 +90068 -1.000000 +90069 -1.000000 +90070 -1.000000 +90071 -1.000000 +90072 -1.000000 +90073 -1.000000 +90074 -1.000000 +90075 -1.000000 +90076 -1.000000 +90077 -1.000000 +90078 -1.000000 +90079 -1.000000 +90080 -1.000000 +90081 -1.000000 +90082 -1.000000 +90083 -1.000000 +90084 -1.000000 +90085 -1.000000 +90086 -1.000000 +90087 -1.000000 +90088 -1.000000 +90089 -1.000000 +90090 -1.000000 +90091 -1.000000 +90092 -1.000000 +90093 -1.000000 +90094 -1.000000 +90095 -1.000000 +90096 -1.000000 +90097 -1.000000 +90098 -1.000000 +90099 -1.000000 +90102 -1.000000 +90103 -1.000000 +90104 -1.000000 +90105 -1.000000 +90106 -1.000000 +90107 -1.000000 +90108 -1.000000 +90109 -1.000000 +90110 -1.000000 +90111 -1.000000 +90113 -1.000000 +90115 -1.000000 +90116 -1.000000 +90117 -1.000000 +90118 -1.000000 +90119 -1.000000 +90120 -1.000000 +90127 -1.000000 +90128 -1.000000 +90130 -1.000000 +90131 -1.000000 +90132 -1.000000 +90133 -1.000000 +90134 -1.000000 +90135 -1.000000 +90136 -1.000000 +90137 -1.000000 +90139 -1.000000 +90140 -1.000000 +90141 -1.000000 +90142 -1.000000 +90143 -1.000000 +90144 -1.000000 +90145 -1.000000 +90146 -1.000000 +90147 -1.000000 +90148 -1.000000 +90149 -1.000000 +90151 -1.000000 +90152 -1.000000 +90154 -1.000000 +90155 -1.000000 +90156 -1.000000 +90157 -1.000000 +90158 -1.000000 +90159 -1.000000 +90160 -1.000000 +90161 -1.000000 +90163 -1.000000 +90165 -1.000000 +90166 -1.000000 +90167 -1.000000 +90168 -1.000000 +90169 -1.000000 +90170 -1.000000 +90171 -1.000000 +90172 -1.000000 +90173 -1.000000 +90174 -1.000000 +90175 -1.000000 +90176 -1.000000 +90177 -1.000000 +90178 -1.000000 +90179 -1.000000 +90180 -1.000000 +90181 -1.000000 +90182 -1.000000 +90183 -1.000000 +90184 -1.000000 +90185 -1.000000 +90186 -1.000000 +90187 -1.000000 +90189 -1.000000 +90190 -1.000000 +90192 -1.000000 +90193 -1.000000 +90194 -1.000000 +90195 -1.000000 +90196 -1.000000 +90197 -1.000000 +90198 -1.000000 +90199 -1.000000 +90200 -1.000000 +90201 -1.000000 +90202 -1.000000 +90203 -1.000000 +90204 -1.000000 +90205 -1.000000 +90206 -1.000000 +90207 -1.000000 +90208 -1.000000 +90209 -1.000000 +90211 -1.000000 +90212 -1.000000 +90213 -1.000000 +90214 -1.000000 +90215 -1.000000 +90216 -1.000000 +90217 -1.000000 +90220 -1.000000 +90221 -1.000000 +90222 -1.000000 +90223 -1.000000 +90224 -1.000000 +90227 -1.000000 +90228 -1.000000 +90229 -1.000000 +90230 -1.000000 +90231 -1.000000 +90232 -1.000000 +90233 -1.000000 +90234 -1.000000 +90235 -1.000000 +90236 -1.000000 +90237 -1.000000 +90238 -1.000000 +90239 -1.000000 +90240 -1.000000 +90241 -1.000000 +90242 -1.000000 +90243 -1.000000 +90244 -1.000000 +90245 -1.000000 +90247 -1.000000 +90248 -1.000000 +90249 -1.000000 +90251 -1.000000 +90255 -1.000000 +90256 -1.000000 +90257 -1.000000 +90258 -1.000000 +90259 -1.000000 +90260 -1.000000 +90261 -1.000000 +90262 -1.000000 +90263 -1.000000 +90264 -1.000000 +90265 -1.000000 +90266 -1.000000 +90267 -1.000000 +90270 -1.000000 +90271 -1.000000 +90272 -1.000000 +90273 -1.000000 +90274 -1.000000 +90275 -1.000000 +90277 -1.000000 +90278 -1.000000 +90279 -1.000000 +90280 -1.000000 +90282 -1.000000 +90283 -1.000000 +90284 -1.000000 +90285 -1.000000 +90286 -1.000000 +90287 -1.000000 +90288 -1.000000 +90289 -1.000000 +90291 -1.000000 +90292 -1.000000 +90293 -1.000000 +90294 -1.000000 +90295 -1.000000 +90296 -1.000000 +90297 -1.000000 +90298 -1.000000 +90308 -1.000000 +90313 -1.000000 +90314 -1.000000 +90315 -1.000000 +90316 -1.000000 +90317 -1.000000 +90318 -1.000000 +90321 -1.000000 +90323 -1.000000 +90324 -1.000000 +90325 -1.000000 +90326 -1.000000 +90327 -1.000000 +90328 -1.000000 +90329 -1.000000 +90331 -1.000000 +90332 -1.000000 +90338 -1.000000 +90339 -1.000000 +90340 -1.000000 +90341 -1.000000 +90342 -1.000000 +90343 -1.000000 +90344 -1.000000 +90345 -1.000000 +90348 -1.000000 +90349 -1.000000 +90350 -1.000000 +90351 -1.000000 +90352 -1.000000 +90353 -1.000000 +90354 -1.000000 +90355 -1.000000 +90356 -1.000000 +90360 -1.000000 +90361 -1.000000 +90363 -1.000000 +90364 -1.000000 +90365 -1.000000 +90366 -1.000000 +90372 -1.000000 +90373 -1.000000 +90374 -1.000000 +90375 -1.000000 +90376 -1.000000 +90377 -1.000000 +90379 -1.000000 +90380 -1.000000 +90381 -1.000000 +90382 -1.000000 +90383 -1.000000 +90385 -1.000000 +90398 -1.000000 +90399 -1.000000 +90400 -1.000000 +90401 -1.000000 +90402 -1.000000 +90403 -1.000000 +90405 -1.000000 +90406 -1.000000 +90407 -1.000000 +90408 -1.000000 +90409 -1.000000 +90410 -1.000000 +90411 -1.000000 +90413 -1.000000 +90414 -1.000000 +90415 -1.000000 +90416 -1.000000 +90417 -1.000000 +90418 -1.000000 +90419 -1.000000 +90421 -1.000000 +90422 -1.000000 +90423 -1.000000 +90424 -1.000000 +90425 -1.000000 +90426 -1.000000 +90427 -1.000000 +90428 -1.000000 +90429 -1.000000 +90430 -1.000000 +90431 -1.000000 +90432 -1.000000 +90435 -1.000000 +90436 -1.000000 +90437 -1.000000 +90438 -1.000000 +90439 -1.000000 +90440 -1.000000 +90441 -1.000000 +90442 -1.000000 +90443 -1.000000 +90446 -1.000000 +90447 -1.000000 +90448 -1.000000 +90449 -1.000000 +90450 -1.000000 +90451 -1.000000 +90452 -1.000000 +90453 -1.000000 +90454 -1.000000 +90455 -1.000000 +90456 -1.000000 +90457 -1.000000 +90458 -1.000000 +90459 -1.000000 +90460 -1.000000 +90462 -1.000000 +90463 -1.000000 +90465 -1.000000 +90466 -1.000000 +90468 -1.000000 +90471 -1.000000 +90472 -1.000000 +90473 -1.000000 +90474 -1.000000 +90475 -1.000000 +90476 -1.000000 +90477 -1.000000 +90478 -1.000000 +90479 -1.000000 +90480 -1.000000 +90481 -1.000000 +90482 -1.000000 +90483 -1.000000 +90484 -1.000000 +90485 -1.000000 +90486 -1.000000 +90487 -1.000000 +90488 -1.000000 +90489 -1.000000 +90490 -1.000000 +90491 -1.000000 +90492 -1.000000 +90493 -1.000000 +90494 -1.000000 +90496 -1.000000 +90497 -1.000000 +90498 -1.000000 +90499 -1.000000 +90500 -1.000000 +90501 -1.000000 +90502 -1.000000 +90503 -1.000000 +90504 -1.000000 +90505 -1.000000 +90506 -1.000000 +90507 -1.000000 +90508 -1.000000 +90509 -1.000000 +90510 -1.000000 +90511 -1.000000 +90514 -1.000000 +90515 -1.000000 +90516 -1.000000 +90517 -1.000000 +90518 -1.000000 +90519 -1.000000 +90520 -1.000000 +90521 -1.000000 +90522 -1.000000 +90523 -1.000000 +90524 -1.000000 +90525 -1.000000 +90526 -1.000000 +90527 -1.000000 +90528 -1.000000 +90530 -1.000000 +90531 -1.000000 +90534 -1.000000 +90535 -1.000000 +90536 -1.000000 +90537 -1.000000 +90538 -1.000000 +90539 -1.000000 +90540 -1.000000 +90541 -1.000000 +90542 -1.000000 +90543 -1.000000 +90545 -1.000000 +90546 -1.000000 +90547 -1.000000 +90549 -1.000000 +90550 -1.000000 +90551 -1.000000 +90552 -1.000000 +90553 -1.000000 +90554 -1.000000 +90555 -1.000000 +90556 -1.000000 +90557 -1.000000 +90558 -1.000000 +90559 -1.000000 +90560 -1.000000 +90561 -1.000000 +90562 -1.000000 +90563 -1.000000 +90564 -1.000000 +90565 -1.000000 +90566 -1.000000 +90567 -1.000000 +90568 -1.000000 +90569 -1.000000 +90570 -1.000000 +90571 -1.000000 +90572 -1.000000 +90573 -1.000000 +90574 -1.000000 +90575 -1.000000 +90576 -1.000000 +90577 -1.000000 +90578 -1.000000 +90580 -1.000000 +90581 -1.000000 +90582 -1.000000 +90583 -1.000000 +90584 -1.000000 +90586 -1.000000 +90587 -1.000000 +90588 -1.000000 +90589 -1.000000 +90590 -1.000000 +90591 -1.000000 +90592 -1.000000 +90593 -1.000000 +90597 -1.000000 +90604 -1.000000 +90605 -1.000000 +90608 -1.000000 +90609 -1.000000 +90610 -1.000000 +90622 -1.000000 +90626 -1.000000 +90627 -1.000000 +90628 -1.000000 +90629 -1.000000 +90630 -1.000000 +90631 -1.000000 +90632 -1.000000 +90633 -1.000000 +90634 -1.000000 +90635 -1.000000 +90636 -1.000000 +90637 -1.000000 +90638 -1.000000 +90640 -1.000000 +90642 -1.000000 +90645 -1.000000 +90646 -1.000000 +90647 -1.000000 +90649 -1.000000 +90650 -1.000000 +90651 -1.000000 +90652 -1.000000 +90653 -1.000000 +90654 -1.000000 +90655 -1.000000 +90656 -1.000000 +90657 -1.000000 +90658 -1.000000 +90660 -1.000000 +90661 -1.000000 +90662 -1.000000 +90663 -1.000000 +90664 -1.000000 +90665 -1.000000 +90666 -1.000000 +90667 -1.000000 +90668 -1.000000 +90669 -1.000000 +90670 -1.000000 +90671 -1.000000 +90672 -1.000000 +90674 -1.000000 +90677 -1.000000 +90678 -1.000000 +90682 -1.000000 +90683 -1.000000 +90697 -1.000000 +90698 -1.000000 +90702 -1.000000 +90703 -1.000000 +90704 -1.000000 +90705 -1.000000 +90708 -1.000000 +90711 -1.000000 +90712 -1.000000 +90713 -1.000000 +90714 -1.000000 +90715 -1.000000 +90716 -1.000000 +90717 -1.000000 +90718 -1.000000 +90719 -1.000000 +90722 -1.000000 +90725 -1.000000 +90726 -1.000000 +90728 -1.000000 +90730 -1.000000 +90731 -1.000000 +90732 -1.000000 +90733 -1.000000 +90734 -1.000000 +90735 -1.000000 +90736 -1.000000 +90737 -1.000000 +90738 -1.000000 +90739 -1.000000 +90740 -1.000000 +90741 -1.000000 +90742 -1.000000 +90743 -1.000000 +90744 -1.000000 +90745 -1.000000 +90746 -1.000000 +90747 -1.000000 +90748 -1.000000 +90749 -1.000000 +90750 -1.000000 +90751 -1.000000 +90752 -1.000000 +90753 -1.000000 +90754 -1.000000 +90755 -1.000000 +90757 -1.000000 +90758 -1.000000 +90759 -1.000000 +90760 -1.000000 +90761 -1.000000 +90762 -1.000000 +90763 -1.000000 +90764 -1.000000 +90765 -1.000000 +90766 -1.000000 +90767 -1.000000 +90768 -1.000000 +90769 -1.000000 +90770 -1.000000 +90771 -1.000000 +90772 -1.000000 +90773 -1.000000 +90774 -1.000000 +90775 -1.000000 +90776 -1.000000 +90777 -1.000000 +90778 -1.000000 +90779 -1.000000 +90780 -1.000000 +90781 -1.000000 +90782 -1.000000 +90783 -1.000000 +90784 -1.000000 +90785 -1.000000 +90786 -1.000000 +90788 -1.000000 +90789 -1.000000 +90790 -1.000000 +90791 -1.000000 +90793 -1.000000 +90794 -1.000000 +90795 -1.000000 +90796 -1.000000 +90797 -1.000000 +90798 -1.000000 +90799 -1.000000 +90800 -1.000000 +90801 -1.000000 +90802 -1.000000 +90803 -1.000000 +90804 -1.000000 +90805 -1.000000 +90806 -1.000000 +90807 -1.000000 +90808 -1.000000 +90809 -1.000000 +90810 -1.000000 +90811 -1.000000 +90812 -1.000000 +90813 -1.000000 +90814 -1.000000 +90815 -1.000000 +90816 -1.000000 +90817 -1.000000 +90818 -1.000000 +90819 -1.000000 +90820 -1.000000 +90821 -1.000000 +90822 -1.000000 +90823 -1.000000 +90824 -1.000000 +90825 -1.000000 +90826 -1.000000 +90827 -1.000000 +90828 -1.000000 +90829 -1.000000 +90830 -1.000000 +90831 -1.000000 +90832 -1.000000 +90833 -1.000000 +90834 -1.000000 +90835 -1.000000 +90836 -1.000000 +90837 -1.000000 +90838 -1.000000 +90839 -1.000000 +90840 -1.000000 +90841 -1.000000 +90842 -1.000000 +90843 -1.000000 +90844 -1.000000 +90845 -1.000000 +90846 -1.000000 +90847 -1.000000 +90848 -1.000000 +90849 -1.000000 +90850 -1.000000 +90851 -1.000000 +90852 -1.000000 +90853 -1.000000 +90854 -1.000000 +90855 -1.000000 +90856 -1.000000 +90857 -1.000000 +90858 -1.000000 +90859 -1.000000 +90860 -1.000000 +90861 -1.000000 +90862 -1.000000 +90863 -1.000000 +90864 -1.000000 +90865 -1.000000 +90866 -1.000000 +90867 -1.000000 +90868 -1.000000 +90869 -1.000000 +90870 -1.000000 +90871 -1.000000 +90872 -1.000000 +90873 -1.000000 +90874 -1.000000 +90875 -1.000000 +90876 -1.000000 +90877 -1.000000 +90878 -1.000000 +90879 -1.000000 +90880 -1.000000 +90881 -1.000000 +90882 -1.000000 +90883 -1.000000 +90884 -1.000000 +90885 -1.000000 +90886 -1.000000 +90888 -1.000000 +90889 -1.000000 +90890 -1.000000 +90891 -1.000000 +90892 -1.000000 +90893 -1.000000 +90894 -1.000000 +90895 -1.000000 +90896 -1.000000 +90897 -1.000000 +90898 -1.000000 +90899 -1.000000 +90901 -1.000000 +90902 -1.000000 +90903 -1.000000 +90905 -1.000000 +90906 -1.000000 +90907 -1.000000 +90908 -1.000000 +90909 -1.000000 +90910 -1.000000 +90911 -1.000000 +90912 -1.000000 +90913 -1.000000 +90914 -1.000000 +90915 -1.000000 +90916 -1.000000 +90917 -1.000000 +90918 -1.000000 +90919 -1.000000 +90920 -1.000000 +90921 -1.000000 +90922 -1.000000 +90923 -1.000000 +90925 -1.000000 +90926 -1.000000 +90927 -1.000000 +90928 -1.000000 +90929 -1.000000 +90930 -1.000000 +90931 -1.000000 +90932 -1.000000 +90933 -1.000000 +90934 -1.000000 +90935 -1.000000 +90936 -1.000000 +90937 -1.000000 +90938 -1.000000 +90939 -1.000000 +90940 -1.000000 +90941 -1.000000 +90942 -1.000000 +90943 -1.000000 +90944 -1.000000 +90945 -1.000000 +90946 -1.000000 +90947 -1.000000 +90948 -1.000000 +90949 -1.000000 +90950 -1.000000 +90952 -1.000000 +90954 -1.000000 +90955 -1.000000 +90956 -1.000000 +90957 -1.000000 +90958 -1.000000 +90959 -1.000000 +90960 -1.000000 +90961 -1.000000 +90962 -1.000000 +90963 -1.000000 +90964 -1.000000 +90967 -1.000000 +90968 -1.000000 +90969 -1.000000 +90970 -1.000000 +90971 -1.000000 +90972 -1.000000 +90973 -1.000000 +90974 -1.000000 +90975 -1.000000 +90976 -1.000000 +90977 -1.000000 +90978 -1.000000 +90979 -1.000000 +90980 -1.000000 +90981 -1.000000 +90983 -1.000000 +90984 -1.000000 +90986 -1.000000 +90987 -1.000000 +90988 -1.000000 +90989 -1.000000 +90990 -1.000000 +90991 -1.000000 +90992 -1.000000 +90993 -1.000000 +90994 -1.000000 +90995 -1.000000 +90996 -1.000000 +90997 -1.000000 +90998 -1.000000 +90999 -1.000000 +91000 -1.000000 +91001 -1.000000 +91002 -1.000000 +91003 -1.000000 +91004 -1.000000 +91005 -1.000000 +91007 -1.000000 +91008 -1.000000 +91009 -1.000000 +91010 -1.000000 +91011 -1.000000 +91012 -1.000000 +91014 -1.000000 +91015 -1.000000 +91016 -1.000000 +91017 -1.000000 +91018 -1.000000 +91019 -1.000000 +91020 -1.000000 +91021 -1.000000 +91022 -1.000000 +91023 -1.000000 +91027 -1.000000 +91034 -1.000000 +91041 -1.000000 +91043 -1.000000 +91044 -1.000000 +91045 -1.000000 +91046 -1.000000 +91047 -1.000000 +91048 -1.000000 +91049 -1.000000 +91050 -1.000000 +91051 -1.000000 +91052 -1.000000 +91053 -1.000000 +91054 -1.000000 +91055 -1.000000 +91056 -1.000000 +91057 -1.000000 +91058 -1.000000 +91059 -1.000000 +91060 -1.000000 +91061 -1.000000 +91062 -1.000000 +91063 -1.000000 +91064 -1.000000 +91065 -1.000000 +91066 -1.000000 +91070 -1.000000 +91075 -1.000000 +91083 -1.000000 +91084 -1.000000 +91085 -1.000000 +91086 -1.000000 +91087 -1.000000 +91089 -1.000000 +91097 -1.000000 +91099 -1.000000 +91100 -1.000000 +91101 -1.000000 +91102 -1.000000 +91104 -1.000000 +91105 -1.000000 +91106 -1.000000 +91107 -1.000000 +91108 -1.000000 +91109 -1.000000 +91110 -1.000000 +91111 -1.000000 +91112 -1.000000 +91113 -1.000000 +91114 -1.000000 +91115 -1.000000 +91116 -1.000000 +91117 -1.000000 +91118 -1.000000 +91119 -1.000000 +91120 -1.000000 +91121 -1.000000 +91122 -1.000000 +91123 -1.000000 +91124 -1.000000 +91125 -1.000000 +91126 -1.000000 +91127 -1.000000 +91128 -1.000000 +91129 -1.000000 +91130 -1.000000 +91131 -1.000000 +91133 -1.000000 +91134 -1.000000 +91135 -1.000000 +91136 -1.000000 +91137 -1.000000 +91138 -1.000000 +91139 -1.000000 +91140 -1.000000 +91141 -1.000000 +91142 -1.000000 +91143 -1.000000 +91144 -1.000000 +91145 -1.000000 +91146 -1.000000 +91147 -1.000000 +91148 -1.000000 +91149 -1.000000 +91150 -1.000000 +91151 -1.000000 +91152 -1.000000 +91153 -1.000000 +91154 -1.000000 +91155 -1.000000 +91156 -1.000000 +91157 -1.000000 +91158 -1.000000 +91159 -1.000000 +91160 -1.000000 +91161 -1.000000 +91162 -1.000000 +91163 -1.000000 +91164 -1.000000 +91165 -1.000000 +91166 -1.000000 +91167 -1.000000 +91168 -1.000000 +91169 -1.000000 +91170 -1.000000 +91171 -1.000000 +91172 -1.000000 +91173 -1.000000 +91174 -1.000000 +91175 -1.000000 +91176 -1.000000 +91177 -1.000000 +91178 -1.000000 +91179 -1.000000 +91180 -1.000000 +91181 -1.000000 +91182 -1.000000 +91183 -1.000000 +91184 -1.000000 +91185 -1.000000 +91186 -1.000000 +91187 -1.000000 +91188 -1.000000 +91189 -1.000000 +91190 -1.000000 +91191 -1.000000 +91192 -1.000000 +91193 -1.000000 +91194 -1.000000 +91195 -1.000000 +91196 -1.000000 +91197 -1.000000 +91198 -1.000000 +91199 -1.000000 +91200 -1.000000 +91201 -1.000000 +91202 -1.000000 +91203 -1.000000 +91204 -1.000000 +91205 -1.000000 +91206 -1.000000 +91207 -1.000000 +91208 -1.000000 +91209 -1.000000 +91210 -1.000000 +91211 -1.000000 +91212 -1.000000 +91213 -1.000000 +91214 -1.000000 +91215 -1.000000 +91216 -1.000000 +91217 -1.000000 +91218 -1.000000 +91219 -1.000000 +91220 -1.000000 +91221 -1.000000 +91222 -1.000000 +91223 -1.000000 +91224 -1.000000 +91225 -1.000000 +91226 -1.000000 +91227 -1.000000 +91228 -1.000000 +91229 -1.000000 +91230 -1.000000 +91231 -1.000000 +91232 -1.000000 +91233 -1.000000 +91234 -1.000000 +91235 -1.000000 +91236 -1.000000 +91237 -1.000000 +91238 -1.000000 +91239 -1.000000 +91240 -1.000000 +91241 -1.000000 +91242 -1.000000 +91244 -1.000000 +91245 -1.000000 +91246 -1.000000 +91247 -1.000000 +91248 -1.000000 +91249 -1.000000 +91250 -1.000000 +91251 -1.000000 +91252 -1.000000 +91253 -1.000000 +91254 -1.000000 +91255 -1.000000 +91256 -1.000000 +91257 -1.000000 +91258 -1.000000 +91259 -1.000000 +91260 -1.000000 +91261 -1.000000 +91262 -1.000000 +91263 -1.000000 +91264 -1.000000 +91265 -1.000000 +91267 -1.000000 +91268 -1.000000 +91270 -1.000000 +91271 -1.000000 +91272 -1.000000 +91273 -1.000000 +91274 -1.000000 +91275 -1.000000 +91276 -1.000000 +91277 -1.000000 +91279 -1.000000 +91280 -1.000000 +91281 -1.000000 +91282 -1.000000 +91283 -1.000000 +91286 -1.000000 +91287 -1.000000 +91288 -1.000000 +91289 -1.000000 +91290 -1.000000 +91295 -1.000000 +91296 -1.000000 +91297 -1.000000 +91298 -1.000000 +91299 -1.000000 +91301 -1.000000 +91303 -1.000000 +91304 -1.000000 +91305 -1.000000 +91306 -1.000000 +91307 -1.000000 +91308 -1.000000 +91309 -1.000000 +91310 -1.000000 +91311 -1.000000 +91312 -1.000000 +91313 -1.000000 +91314 -1.000000 +91315 -1.000000 +91316 -1.000000 +91317 -1.000000 +91318 -1.000000 +91319 -1.000000 +91320 -1.000000 +91321 -1.000000 +91322 -1.000000 +91323 -1.000000 +91324 -1.000000 +91325 -1.000000 +91326 -1.000000 +91327 -1.000000 +91328 -1.000000 +91329 -1.000000 +91330 -1.000000 +91331 -1.000000 +91332 -1.000000 +91333 -1.000000 +91335 -1.000000 +91336 -1.000000 +91337 -1.000000 +91341 -1.000000 +91343 -1.000000 +91344 -1.000000 +91345 -1.000000 +91346 -1.000000 +91347 -1.000000 +91348 -1.000000 +91349 -1.000000 +91350 -1.000000 +91351 -1.000000 +91352 -1.000000 +91353 -1.000000 +91354 -1.000000 +91356 -1.000000 +91357 -1.000000 +91359 -1.000000 +91360 -1.000000 +91361 -1.000000 +91362 -1.000000 +91363 -1.000000 +91364 -1.000000 +91365 -1.000000 +91366 -1.000000 +91367 -1.000000 +91368 -1.000000 +91369 -1.000000 +91370 -1.000000 +91371 -1.000000 +91372 -1.000000 +91373 -1.000000 +91374 -1.000000 +91375 -1.000000 +91376 -1.000000 +91377 -1.000000 +91378 -1.000000 +91379 -1.000000 +91380 -1.000000 +91381 -1.000000 +91382 -1.000000 +91383 -1.000000 +91384 -1.000000 +91385 -1.000000 +91386 -1.000000 +91387 -1.000000 +91389 -1.000000 +91390 -1.000000 +91391 -1.000000 +91392 -1.000000 +91393 -1.000000 +91394 -1.000000 +91395 -1.000000 +91396 -1.000000 +91397 -1.000000 +91398 -1.000000 +91399 -1.000000 +91400 -1.000000 +91401 -1.000000 +91402 -1.000000 +91403 -1.000000 +91404 -1.000000 +91405 -1.000000 +91406 -1.000000 +91408 -1.000000 +91409 -1.000000 +91410 -1.000000 +91411 -1.000000 +91412 -1.000000 +91413 -1.000000 +91414 -1.000000 +91415 -1.000000 +91416 -1.000000 +91417 -1.000000 +91418 -1.000000 +91419 -1.000000 +91420 -1.000000 +91421 -1.000000 +91422 -1.000000 +91423 -1.000000 +91424 -1.000000 +91425 -1.000000 +91426 -1.000000 +91428 -1.000000 +91430 -1.000000 +91431 -1.000000 +91432 -1.000000 +91433 -1.000000 +91434 -1.000000 +91435 -1.000000 +91436 -1.000000 +91437 -1.000000 +91438 -1.000000 +91439 -1.000000 +91440 -1.000000 +91441 -1.000000 +91442 -1.000000 +91444 -1.000000 +91447 -1.000000 +91448 -1.000000 +91449 -1.000000 +91450 -1.000000 +91451 -1.000000 +91452 -1.000000 +91453 -1.000000 +91454 -1.000000 +91455 -1.000000 +91456 -1.000000 +91457 -1.000000 +91458 -1.000000 +91459 -1.000000 +91460 -1.000000 +91461 -1.000000 +91462 -1.000000 +91464 -1.000000 +91465 -1.000000 +91466 -1.000000 +91467 -1.000000 +91468 -1.000000 +91469 -1.000000 +91470 -1.000000 +91471 -1.000000 +91472 -1.000000 +91473 -1.000000 +91474 -1.000000 +91475 -1.000000 +91476 -1.000000 +91477 -1.000000 +91478 -1.000000 +91479 -1.000000 +91480 -1.000000 +91481 -1.000000 +91482 -1.000000 +91483 -1.000000 +91484 -1.000000 +91485 -1.000000 +91486 -1.000000 +91487 -1.000000 +91488 -1.000000 +91489 -1.000000 +91490 -1.000000 +91491 -1.000000 +91492 -1.000000 +91493 -1.000000 +91494 -1.000000 +91495 -1.000000 +91496 -1.000000 +91498 -1.000000 +91499 -1.000000 +91500 -1.000000 +91501 -1.000000 +91502 -1.000000 +91503 -1.000000 +91504 -1.000000 +91505 -1.000000 +91507 -1.000000 +91508 -1.000000 +91510 -1.000000 +91511 -1.000000 +91512 -1.000000 +91513 -1.000000 +91514 -1.000000 +91515 -1.000000 +91516 -1.000000 +91517 -1.000000 +91518 -1.000000 +91519 -1.000000 +91520 -1.000000 +91521 -1.000000 +91523 -1.000000 +91524 -1.000000 +91525 -1.000000 +91526 -1.000000 +91527 -1.000000 +91528 -1.000000 +91529 -1.000000 +91530 -1.000000 +91531 -1.000000 +91532 -1.000000 +91533 -1.000000 +91535 -1.000000 +91536 -1.000000 +91537 -1.000000 +91538 -1.000000 +91539 -1.000000 +91540 -1.000000 +91542 -1.000000 +91543 -1.000000 +91544 -1.000000 +91545 -1.000000 +91546 -1.000000 +91547 -1.000000 +91548 -1.000000 +91549 -1.000000 +91550 -1.000000 +91551 -1.000000 +91552 -1.000000 +91553 -1.000000 +91554 -1.000000 +91555 -1.000000 +91556 -1.000000 +91557 -1.000000 +91558 -1.000000 +91559 -1.000000 +91560 -1.000000 +91561 -1.000000 +91562 -1.000000 +91563 -1.000000 +91564 -1.000000 +91565 -1.000000 +91566 -1.000000 +91567 -1.000000 +91568 -1.000000 +91569 -1.000000 +91571 -1.000000 +91573 -1.000000 +91574 -1.000000 +91575 -1.000000 +91576 -1.000000 +91577 -1.000000 +91578 -1.000000 +91579 -1.000000 +91580 -1.000000 +91581 -1.000000 +91582 -1.000000 +91583 -1.000000 +91584 -1.000000 +91585 -1.000000 +91587 -1.000000 +91588 -1.000000 +91590 -1.000000 +91591 -1.000000 +91592 -1.000000 +91593 -1.000000 +91594 -1.000000 +91595 -1.000000 +91596 -1.000000 +91597 -1.000000 +91599 -1.000000 +91600 -1.000000 +91601 -1.000000 +91602 -1.000000 +91603 -1.000000 +91604 -1.000000 +91605 -1.000000 +91606 -1.000000 +91607 -1.000000 +91608 -1.000000 +91609 -1.000000 +91610 -1.000000 +91611 -1.000000 +91612 -1.000000 +91613 -1.000000 +91614 -1.000000 +91616 -1.000000 +91617 -1.000000 +91618 -1.000000 +91619 -1.000000 +91620 -1.000000 +91621 -1.000000 +91622 -1.000000 +91623 -1.000000 +91624 -1.000000 +91625 -1.000000 +91626 -1.000000 +91630 -1.000000 +91631 -1.000000 +91632 -1.000000 +91633 -1.000000 +91634 -1.000000 +91635 -1.000000 +91637 -1.000000 +91638 -1.000000 +91639 -1.000000 +91640 -1.000000 +91643 -1.000000 +91644 -1.000000 +91645 -1.000000 +91646 -1.000000 +91647 -1.000000 +91648 -1.000000 +91649 -1.000000 +91650 -1.000000 +91652 -1.000000 +91653 -1.000000 +91654 -1.000000 +91655 -1.000000 +91656 -1.000000 +91658 -1.000000 +91659 -1.000000 +91660 -1.000000 +91661 -1.000000 +91662 -1.000000 +91663 -1.000000 +91664 -1.000000 +91665 -1.000000 +91666 -1.000000 +91667 -1.000000 +91668 -1.000000 +91669 -1.000000 +91670 -1.000000 +91671 -1.000000 +91672 -1.000000 +91673 -1.000000 +91674 -1.000000 +91675 -1.000000 +91676 -1.000000 +91679 -1.000000 +91680 -1.000000 +91682 -1.000000 +91683 -1.000000 +91684 -1.000000 +91685 -1.000000 +91686 -1.000000 +91687 -1.000000 +91688 -1.000000 +91689 -1.000000 +91690 -1.000000 +91691 -1.000000 +91692 -1.000000 +91693 -1.000000 +91694 -1.000000 +91695 -1.000000 +91697 -1.000000 +91698 -1.000000 +91699 -1.000000 +91700 -1.000000 +91701 -1.000000 +91702 -1.000000 +91703 -1.000000 +91705 -1.000000 +91706 -1.000000 +91708 -1.000000 +91710 -1.000000 +91711 -1.000000 +91712 -1.000000 +91713 -1.000000 +91714 -1.000000 +91715 -1.000000 +91716 -1.000000 +91717 -1.000000 +91718 -1.000000 +91719 -1.000000 +91720 -1.000000 +91721 -1.000000 +91722 -1.000000 +91723 -1.000000 +91724 -1.000000 +91725 -1.000000 +91726 -1.000000 +91729 -1.000000 +91730 -1.000000 +91731 -1.000000 +91732 -1.000000 +91733 -1.000000 +91734 -1.000000 +91735 -1.000000 +91736 -1.000000 +91737 -1.000000 +91738 -1.000000 +91739 -1.000000 +91740 -1.000000 +91741 -1.000000 +91742 -1.000000 +91743 -1.000000 +91744 -1.000000 +91745 -1.000000 +91746 -1.000000 +91747 -1.000000 +91748 -1.000000 +91749 -1.000000 +91750 -1.000000 +91751 -1.000000 +91752 -1.000000 +91753 -1.000000 +91754 -1.000000 +91755 -1.000000 +91756 -1.000000 +91757 -1.000000 +91758 -1.000000 +91759 -1.000000 +91760 -1.000000 +91761 -1.000000 +91762 -1.000000 +91763 -1.000000 +91764 -1.000000 +91765 -1.000000 +91766 -1.000000 +91767 -1.000000 +91768 -1.000000 +91769 -1.000000 +91770 -1.000000 +91771 -1.000000 +91772 -1.000000 +91773 -1.000000 +91774 -1.000000 +91775 -1.000000 +91776 -1.000000 +91777 -1.000000 +91778 -1.000000 +91779 -1.000000 +91780 -1.000000 +91781 -1.000000 +91782 -1.000000 +91783 -1.000000 +91784 -1.000000 +91785 -1.000000 +91786 -1.000000 +91787 -1.000000 +91788 -1.000000 +91789 -1.000000 +91790 -1.000000 +91791 -1.000000 +91792 -1.000000 +91794 -1.000000 +91795 -1.000000 +91797 -1.000000 +91798 -1.000000 +91799 -1.000000 +91800 -1.000000 +91801 -1.000000 +91802 -1.000000 +91803 -1.000000 +91804 -1.000000 +91805 -1.000000 +91806 -1.000000 +91807 -1.000000 +91808 -1.000000 +91809 -1.000000 +91810 -1.000000 +91811 -1.000000 +91812 -1.000000 +91813 -1.000000 +91814 -1.000000 +91815 -1.000000 +91816 -1.000000 +91817 -1.000000 +91818 -1.000000 +91819 -1.000000 +91821 -1.000000 +91822 -1.000000 +91823 -1.000000 +91824 -1.000000 +91825 -1.000000 +91827 -1.000000 +91828 -1.000000 +91829 -1.000000 +91830 -1.000000 +91831 -1.000000 +91833 -1.000000 +91834 -1.000000 +91835 -1.000000 +91836 -1.000000 +91837 -1.000000 +91838 -1.000000 +91839 -1.000000 +91840 -1.000000 +91841 -1.000000 +91842 -1.000000 +91844 -1.000000 +91845 -1.000000 +91846 -1.000000 +91847 -1.000000 +91848 -1.000000 +91849 -1.000000 +91850 -1.000000 +91851 -1.000000 +91852 -1.000000 +91853 -1.000000 +91854 -1.000000 +91855 -1.000000 +91856 -1.000000 +91857 -1.000000 +91858 -1.000000 +91859 -1.000000 +91860 -1.000000 +91861 -1.000000 +91862 -1.000000 +91863 -1.000000 +91864 -1.000000 +91865 -1.000000 +91866 -1.000000 +91867 -1.000000 +91868 -1.000000 +91869 -1.000000 +91870 -1.000000 +91871 -1.000000 +91872 -1.000000 +91873 -1.000000 +91874 -1.000000 +91875 -1.000000 +91876 -1.000000 +91877 -1.000000 +91878 -1.000000 +91879 -1.000000 +91880 -1.000000 +91881 -1.000000 +91882 -1.000000 +91883 -1.000000 +91884 -1.000000 +91885 -1.000000 +91886 -1.000000 +91887 -1.000000 +91890 -1.000000 +91891 -1.000000 +91892 -1.000000 +91893 -1.000000 +91894 -1.000000 +91895 -1.000000 +91896 -1.000000 +91897 -1.000000 +91898 -1.000000 +91899 -1.000000 +91900 -1.000000 +91901 -1.000000 +91902 -1.000000 +91903 -1.000000 +91904 -1.000000 +91905 -1.000000 +91906 -1.000000 +91908 -1.000000 +91909 -1.000000 +91910 -1.000000 +91911 -1.000000 +91912 -1.000000 +91913 -1.000000 +91914 -1.000000 +91915 -1.000000 +91916 -1.000000 +91917 -1.000000 +91918 -1.000000 +91919 -1.000000 +91920 -1.000000 +91921 -1.000000 +91922 -1.000000 +91923 -1.000000 +91925 -1.000000 +91926 -1.000000 +91927 -1.000000 +91928 -1.000000 +91929 -1.000000 +91930 -1.000000 +91931 -1.000000 +91932 -1.000000 +91933 -1.000000 +91934 -1.000000 +91935 -1.000000 +91936 -1.000000 +91937 -1.000000 +91938 -1.000000 +91939 -1.000000 +91940 -1.000000 +91941 -1.000000 +91942 -1.000000 +91943 -1.000000 +91944 -1.000000 +91945 -1.000000 +91946 -1.000000 +91947 -1.000000 +91948 -1.000000 +91949 -1.000000 +91950 -1.000000 +91951 -1.000000 +91952 -1.000000 +91953 -1.000000 +91954 -1.000000 +91955 -1.000000 +91956 -1.000000 +91957 -1.000000 +91958 -1.000000 +91959 -1.000000 +91960 -1.000000 +91961 -1.000000 +91962 -1.000000 +91964 -1.000000 +91965 -1.000000 +91966 -1.000000 +91967 -1.000000 +91968 -1.000000 +91969 -1.000000 +91973 -1.000000 +91974 -1.000000 +91975 -1.000000 +91976 -1.000000 +91978 -1.000000 +91979 -1.000000 +91980 -1.000000 +91982 -1.000000 +91984 -1.000000 +91985 -1.000000 +91986 -1.000000 +91987 -1.000000 +91988 -1.000000 +91996 -1.000000 +91999 -1.000000 +92000 -1.000000 +92002 -1.000000 +92004 -1.000000 +92005 -1.000000 +92007 -1.000000 +92011 -1.000000 +92023 -1.000000 +92024 -1.000000 +92025 -1.000000 +92026 -1.000000 +92027 -1.000000 +92028 -1.000000 +92029 -1.000000 +92030 -1.000000 +92032 -1.000000 +92039 -1.000000 +92041 -1.000000 +92042 -1.000000 +92046 -1.000000 +92048 -1.000000 +92051 -1.000000 +92052 -1.000000 +92053 -1.000000 +92055 -1.000000 +92058 -1.000000 +92060 -1.000000 +92061 -1.000000 +92062 -1.000000 +92063 -1.000000 +92064 -1.000000 +92065 -1.000000 +92066 -1.000000 +92067 -1.000000 +92068 -1.000000 +92071 -1.000000 +92073 -1.000000 +92074 -1.000000 +92075 -1.000000 +92076 -1.000000 +92078 -1.000000 +92079 -1.000000 +92083 -1.000000 +92084 -1.000000 +92085 -1.000000 +92086 -1.000000 +92089 -1.000000 +92090 -1.000000 +92091 -1.000000 +92092 -1.000000 +92093 -1.000000 +92094 -1.000000 +92095 -1.000000 +92096 -1.000000 +92097 -1.000000 +92102 -1.000000 +92103 -1.000000 +92104 -1.000000 +92106 -1.000000 +92108 -1.000000 +92109 -1.000000 +92110 -1.000000 +92111 -1.000000 +92112 -1.000000 +92114 -1.000000 +92115 -1.000000 +92117 -1.000000 +92118 -1.000000 +92119 -1.000000 +92121 -1.000000 +92122 -1.000000 +92126 -1.000000 +92127 -1.000000 +92128 -1.000000 +92130 -1.000000 +92131 -1.000000 +92133 -1.000000 +92137 -1.000000 +92138 -1.000000 +92139 -1.000000 +92141 -1.000000 +92142 -1.000000 +92143 -1.000000 +92145 -1.000000 +92146 -1.000000 +92147 -1.000000 +92148 -1.000000 +92149 -1.000000 +92150 -1.000000 +92151 -1.000000 +92154 -1.000000 +92155 -1.000000 +92158 -1.000000 +92159 -1.000000 +92160 -1.000000 +92161 -1.000000 +92162 -1.000000 +92163 -1.000000 +92164 -1.000000 +92166 -1.000000 +92167 -1.000000 +92168 -1.000000 +92169 -1.000000 +92170 -1.000000 +92171 -1.000000 +92172 -1.000000 +92173 -1.000000 +92175 -1.000000 +92177 -1.000000 +92178 -1.000000 +92179 -1.000000 +92180 -1.000000 +92183 -1.000000 +92184 -1.000000 +92187 -1.000000 +92191 -1.000000 +92193 -1.000000 +92194 -1.000000 +92195 -1.000000 +92196 -1.000000 +92198 -1.000000 +92200 -1.000000 +92201 -1.000000 +92202 -1.000000 +92203 -1.000000 +92204 -1.000000 +92206 -1.000000 +92207 -1.000000 +92209 -1.000000 +92216 -1.000000 +92218 -1.000000 +92219 -1.000000 +92222 -1.000000 +92223 -1.000000 +92224 -1.000000 +92228 -1.000000 +92232 -1.000000 +92234 -1.000000 +92237 -1.000000 +92242 -1.000000 +92243 -1.000000 +92244 -1.000000 +92245 -1.000000 +92247 -1.000000 +92248 -1.000000 +92249 -1.000000 +92250 -1.000000 +92251 -1.000000 +92252 -1.000000 +92254 -1.000000 +92255 -1.000000 +92256 -1.000000 +92257 -1.000000 +92259 -1.000000 +92260 -1.000000 +92261 -1.000000 +92262 -1.000000 +92263 -1.000000 +92265 -1.000000 +92266 -1.000000 +92268 -1.000000 +92272 -1.000000 +92273 -1.000000 +92274 -1.000000 +92275 -1.000000 +92276 -1.000000 +92277 -1.000000 +92279 -1.000000 +92280 -1.000000 +92281 -1.000000 +92282 -1.000000 +92283 -1.000000 +92284 -1.000000 +92285 -1.000000 +92286 -1.000000 +92290 -1.000000 +92291 -1.000000 +92292 -1.000000 +92293 -1.000000 +92294 -1.000000 +92295 -1.000000 +92298 -1.000000 +92301 -1.000000 +92302 -1.000000 +92303 -1.000000 +92304 -1.000000 +92306 -1.000000 +92307 -1.000000 +92310 -1.000000 +92314 -1.000000 +92315 -1.000000 +92316 -1.000000 +92318 -1.000000 +92319 -1.000000 +92320 -1.000000 +92321 -1.000000 +92322 -1.000000 +92323 -1.000000 +92324 -1.000000 +92326 -1.000000 +92327 -1.000000 +92328 -1.000000 +92329 -1.000000 +92330 -1.000000 +92331 -1.000000 +92332 -1.000000 +92333 -1.000000 +92334 -1.000000 +92335 -1.000000 +92336 -1.000000 +92337 -1.000000 +92338 -1.000000 +92339 -1.000000 +92340 -1.000000 +92341 -1.000000 +92342 -1.000000 +92343 -1.000000 +92344 -1.000000 +92345 -1.000000 +92346 -1.000000 +92347 -1.000000 +92348 -1.000000 +92350 -1.000000 +92351 -1.000000 +92352 -1.000000 +92353 -1.000000 +92354 -1.000000 +92355 -1.000000 +92369 -1.000000 +92370 -1.000000 +92371 -1.000000 +92373 -1.000000 +92374 -1.000000 +92379 -1.000000 +92380 -1.000000 +92382 -1.000000 +92383 -1.000000 +92385 -1.000000 +92386 -1.000000 +92387 -1.000000 +92388 -1.000000 +92390 -1.000000 +92391 -1.000000 +92392 -1.000000 +92396 -1.000000 +92397 -1.000000 +92400 -1.000000 +92401 -1.000000 +92403 -1.000000 +92405 -1.000000 +92406 -1.000000 +92407 -1.000000 +92409 -1.000000 +92411 -1.000000 +92412 -1.000000 +92416 -1.000000 +92417 -1.000000 +92418 -1.000000 +92419 -1.000000 +92420 -1.000000 +92421 -1.000000 +92423 -1.000000 +92425 -1.000000 +92426 -1.000000 +92428 -1.000000 +92429 -1.000000 +92430 -1.000000 +92431 -1.000000 +92434 -1.000000 +92436 -1.000000 +92438 -1.000000 +92439 -1.000000 +92440 -1.000000 +92441 -1.000000 +92442 -1.000000 +92443 -1.000000 +92444 -1.000000 +92445 -1.000000 +92446 -1.000000 +92450 -1.000000 +92451 -1.000000 +92455 -1.000000 +92463 -1.000000 +92464 -1.000000 +92465 -1.000000 +92467 -1.000000 +92468 -1.000000 +92469 -1.000000 +92470 -1.000000 +92471 -1.000000 +92472 -1.000000 +92473 -1.000000 +92474 -1.000000 +92475 -1.000000 +92476 -1.000000 +92478 -1.000000 +92479 -1.000000 +92480 -1.000000 +92481 -1.000000 +92482 -1.000000 +92483 -1.000000 +92484 -1.000000 +92485 -1.000000 +92486 -1.000000 +92487 -1.000000 +92488 -1.000000 +92489 -1.000000 +92490 -1.000000 +92491 -1.000000 +92492 -1.000000 +92493 -1.000000 +92494 -1.000000 +92495 -1.000000 +92496 -1.000000 +92498 -1.000000 +92499 -1.000000 +92501 -1.000000 +92502 -1.000000 +92503 -1.000000 diff --git a/data/shot_lists/jet/ILW_unint_full.txt b/data/shot_lists/jet/ILW_unint_full.txt new file mode 100644 index 00000000..62d403fd --- /dev/null +++ b/data/shot_lists/jet/ILW_unint_full.txt @@ -0,0 +1,626 @@ +80181 53.449000 +80195 68.864000 +80406 53.576000 +80487 57.883000 +80511 48.311000 +80512 46.088000 +80605 55.027000 +80609 64.760000 +80618 56.576000 +80652 53.142000 +80697 42.951000 +80706 54.171000 +80708 60.396000 +80749 51.260000 +80763 51.188000 +80764 50.835000 +80772 61.272000 +80773 51.189000 +80774 51.174000 +80847 59.291000 +80848 51.293000 +80853 52.554000 +80874 42.675000 +80947 62.458000 +80997 52.026000 +81008 63.293000 +81017 59.602000 +81024 55.104000 +81159 59.258000 +81191 66.438000 +81193 66.794000 +81197 54.891000 +81200 54.856000 +81203 54.936000 +81228 63.141000 +81308 55.002000 +81312 55.132000 +81313 56.724000 +81422 63.023000 +81438 54.892000 +81491 53.608000 +81502 64.762000 +81516 54.359000 +81517 50.880000 +81520 52.024000 +81521 51.304000 +81522 53.568000 +81526 52.243000 +81529 53.114000 +81530 52.591000 +81531 53.179000 +81538 63.496000 +81578 52.476000 +81579 52.796000 +81580 47.769000 +81581 53.094000 +81582 49.885000 +81583 51.887000 +81584 53.700000 +81585 50.693000 +81586 52.229000 +81610 45.852000 +81612 58.154000 +81628 44.725000 +81637 51.697000 +81638 45.290000 +81667 54.792000 +81714 57.592000 +81727 51.519000 +81731 63.080000 +81749 62.328000 +81765 53.840000 +81766 53.632000 +81781 53.987000 +81782 59.770000 +81788 51.856000 +81789 48.291000 +81799 57.511000 +81801 61.508000 +81803 59.220000 +81804 56.343000 +81819 60.875000 +81820 55.141000 +81821 59.954000 +81823 57.094000 +81824 57.760000 +81826 60.292000 +81827 56.010000 +81833 51.525000 +81834 54.860000 +81867 46.299000 +81914 59.429000 +81953 59.763000 +81978 53.788000 +81983 59.458000 +81985 52.270000 +81990 52.379000 +81992 66.240000 +82001 53.367000 +82004 48.801000 +82005 53.782000 +82012 56.558000 +82013 53.698000 +82015 50.654000 +82016 51.803000 +82023 55.920000 +82024 55.753000 +82025 55.629000 +82034 57.566000 +82140 59.529000 +82157 61.799000 +82196 62.124000 +82218 63.128000 +82257 53.535000 +82258 53.221000 +82259 54.130000 +82281 59.615000 +82299 55.541000 +82304 53.114000 +82314 56.868000 +82318 52.942000 +82320 53.545000 +82342 57.058000 +82390 54.844000 +82393 56.699000 +82409 54.388000 +82415 54.861000 +82465 54.839000 +82466 56.725000 +82470 55.486000 +82473 56.413000 +82495 60.085000 +82499 59.595000 +82500 53.945000 +82502 62.277000 +82504 65.249000 +82553 56.615000 +82554 58.037000 +82555 58.092000 +82556 57.986000 +82569 61.856000 +82572 61.306000 +82577 56.849000 +82578 58.836000 +82589 56.072000 +82590 58.582000 +82669 64.898000 +82673 62.856000 +82674 57.066000 +82687 49.673000 +82698 47.111000 +82775 64.753000 +82780 57.023000 +82797 43.896000 +82807 59.433000 +82811 59.528000 +82822 53.340000 +82851 58.081000 +82852 58.015000 +82853 58.228000 +82867 55.737000 +82927 55.730000 +82932 63.365000 +82933 67.044000 +82934 67.099000 +82937 69.501000 +82938 60.763000 +82939 69.798000 +82940 69.591000 +82941 61.479000 +82942 69.717000 +82955 56.989000 +82962 64.172000 +82980 54.919000 +82981 59.284000 +83065 66.290000 +83083 60.340000 +83137 58.849000 +83143 51.231000 +83144 51.481000 +83146 51.490000 +83153 55.796000 +83154 53.804000 +83191 64.030000 +83209 59.014000 +83210 60.174000 +83216 57.638000 +83232 56.948000 +83233 54.305000 +83295 56.443000 +83312 44.808000 +83313 50.361000 +83314 48.663000 +83317 48.346000 +83324 50.743000 +83326 50.272000 +83330 58.715000 +83331 59.586000 +83333 60.973000 +83337 59.998000 +83338 59.674000 +83346 56.938000 +83352 56.594000 +83356 60.968000 +83363 57.301000 +83366 58.293000 +83381 61.786000 +83390 52.392000 +83403 57.689000 +83404 54.601000 +83411 43.780000 +83417 47.619000 +83423 45.608000 +83425 49.346000 +83435 59.928000 +83453 55.353000 +83504 55.365000 +83514 43.549000 +83515 42.126000 +83518 48.918000 +83519 44.031000 +83521 48.180000 +83522 49.216000 +83526 49.543000 +83531 48.851000 +83533 48.773000 +83545 53.969000 +83557 49.912000 +83566 49.743000 +83567 42.390000 +83568 47.534000 +83570 47.871000 +83571 52.837000 +83572 59.435000 +83598 58.713000 +83601 67.306000 +83611 44.922000 +83615 44.992000 +83618 52.350000 +83619 56.538000 +83620 53.530000 +83633 54.718000 +83635 58.275000 +83681 51.627000 +83698 45.183000 +83730 56.689000 +83980 57.094000 +83982 58.260000 +83987 60.302000 +83999 67.703000 +84031 52.478000 +84036 60.611000 +84051 42.442000 +84054 48.645000 +84077 62.348000 +84079 43.558000 +84115 62.483000 +84142 55.516000 +84143 55.849000 +84158 66.375000 +84160 66.072000 +84161 57.578000 +84162 65.768000 +84163 65.745000 +84164 65.581000 +84165 64.432000 +84166 65.260000 +84167 65.758000 +84168 65.609000 +84169 65.562000 +84170 64.446000 +84171 64.653000 +84172 61.274000 +84173 61.241000 +84180 57.028000 +84271 68.689000 +84275 71.964000 +84308 54.066000 +84315 61.706000 +84322 52.757000 +84323 50.809000 +84324 67.761000 +84328 52.269000 +84352 56.338000 +84408 56.079000 +84428 52.550000 +84433 64.507000 +84435 48.404000 +84462 53.487000 +84464 54.373000 +84465 53.856000 +84467 53.787000 +84517 55.391000 +84519 62.414000 +84520 61.561000 +84524 64.352000 +84581 58.570000 +84595 55.371000 +84600 54.728000 +84620 60.637000 +84628 44.644000 +84639 56.795000 +84640 60.077000 +84648 52.534000 +84655 53.855000 +84656 52.276000 +84657 53.012000 +84658 48.198000 +84671 48.609000 +84674 48.703000 +84676 47.269000 +84677 45.992000 +84679 48.144000 +84681 48.622000 +84690 57.988000 +84691 56.899000 +84695 45.147000 +84696 57.971000 +84707 42.448000 +84715 55.956000 +84716 61.576000 +84748 51.816000 +84808 49.022000 +84820 53.125000 +84826 53.835000 +84827 54.505000 +84828 53.998000 +84829 53.174000 +84857 52.303000 +84858 49.521000 +84861 61.379000 +84862 44.904000 +84869 59.811000 +84870 52.772000 +84912 55.777000 +84933 63.841000 +84935 44.202000 +84937 65.729000 +84942 53.223000 +84943 53.002000 +84946 53.406000 +84947 52.257000 +84950 54.623000 +84955 49.091000 +84957 56.140000 +84960 60.014000 +84962 60.116000 +84965 58.565000 +84986 54.540000 +84987 59.135000 +84988 54.721000 +84989 51.132000 +84990 51.432000 +84991 61.108000 +84994 47.247000 +84995 54.857000 +84996 64.204000 +84998 51.615000 +85005 64.394000 +85033 46.945000 +85048 57.780000 +85070 52.611000 +85072 52.023000 +85073 54.245000 +85074 52.204000 +85076 59.550000 +85077 61.266000 +85162 54.661000 +85164 47.220000 +85165 56.460000 +85168 57.332000 +85170 60.336000 +85171 54.831000 +85176 58.868000 +85177 56.874000 +85179 55.828000 +85182 55.635000 +85185 54.932000 +85187 60.373000 +85188 51.912000 +85189 61.024000 +85190 52.724000 +85191 53.310000 +85192 60.552000 +85193 54.018000 +85200 47.141000 +85202 42.509000 +85204 46.227000 +85212 58.269000 +85222 48.971000 +85229 56.333000 +85236 55.035000 +85238 57.957000 +85239 55.257000 +85240 56.385000 +85241 60.581000 +85251 55.288000 +85252 53.744000 +85253 55.255000 +85257 56.118000 +85258 55.034000 +85259 57.055000 +85285 50.560000 +85286 50.533000 +85287 50.850000 +85290 61.096000 +85291 61.063000 +85292 60.681000 +85293 54.933000 +85294 59.465000 +85295 55.196000 +85300 62.739000 +85359 55.633000 +85360 59.611000 +85362 51.807000 +85363 52.514000 +85387 64.350000 +85390 59.210000 +85396 47.740000 +85401 51.074000 +85410 61.132000 +85411 61.146000 +85447 61.926000 +85455 52.266000 +85457 63.455000 +85460 53.227000 +85467 53.057000 +85472 51.837000 +85473 51.687000 +85482 60.001000 +85483 46.571000 +85484 46.700000 +85551 46.766000 +85554 43.391000 +85593 42.798000 +85603 42.851000 +85643 49.750000 +85674 53.981000 +85720 46.072000 +85722 64.085000 +85727 46.283000 +85753 56.435000 +85763 48.685000 +85767 46.787000 +85771 54.050000 +85774 46.670000 +85779 54.184000 +85780 51.615000 +85792 52.684000 +85808 45.813000 +85809 53.861000 +85843 51.943000 +85861 51.218000 +85865 49.968000 +85867 50.332000 +85869 47.270000 +85882 50.214000 +85883 49.687000 +85925 45.264000 +85944 56.634000 +85958 59.550000 +85959 59.602000 +85964 54.644000 +86102 53.739000 +86129 55.013000 +86268 49.344000 +86291 57.813000 +86337 49.797000 +86448 63.370000 +86449 62.157000 +86542 50.178000 +86545 45.187000 +86558 51.933000 +86580 53.509000 +86585 50.981000 +86586 50.455000 +86588 49.225000 +86591 51.230000 +86601 50.629000 +86603 53.093000 +86614 54.230000 +86826 47.503000 +86866 64.704000 +86911 55.250000 +86986 60.128000 +86991 54.040000 +87011 43.133000 +87056 44.702000 +87068 51.719000 +87134 60.154000 +87151 63.217000 +87248 51.773000 +87254 52.503000 +87356 43.628000 +87434 45.286000 +87435 43.294000 +87441 44.718000 +87477 53.006000 +87478 48.760000 +87490 53.713000 +87510 54.918000 +87515 56.566000 +87640 53.051000 +87643 56.193000 +87644 56.042000 +87647 53.279000 +87649 54.203000 +87702 64.004000 +87713 59.769000 +87714 61.537000 +87719 59.209000 +87720 59.168000 +87728 46.427000 +87743 43.074000 +87746 52.294000 +87751 64.087000 +87757 62.878000 +87782 60.344000 +87787 50.256000 +87790 56.591000 +87791 46.965000 +87796 53.479000 +87797 57.577000 +87798 48.821000 +87811 48.633000 +87812 48.700000 +87813 48.773000 +87820 53.615000 +87821 57.520000 +87822 57.475000 +87824 57.865000 +87826 57.000000 +87827 52.926000 +87828 54.126000 +87832 47.632000 +87848 50.181000 +87856 56.234000 +87857 47.886000 +87859 56.751000 +87861 55.045000 +87863 55.824000 +87864 53.267000 +87878 51.628000 +87942 60.100000 +88098 50.168000 +88197 47.009000 +88220 47.283000 +88238 48.187000 +88239 55.773000 +88258 54.906000 +88367 58.489000 +88420 53.942000 +88494 59.049000 +88498 49.252000 +88499 49.996000 +88511 56.585000 +88520 57.916000 +88535 49.761000 +88547 55.957000 +88550 49.700000 +88577 52.380000 +88613 51.149000 +88627 56.809000 +88632 55.500000 +88736 62.365000 +88745 60.511000 +89127 54.600000 +89182 51.103000 +89186 51.129000 +89192 55.701000 +89325 56.046000 +89418 58.982000 +89439 59.408000 +89635 64.828000 +89659 50.743000 +89774 54.764000 +89780 54.256000 +89843 43.449000 +89892 50.899000 +90164 55.479000 +90218 51.472000 +90219 52.108000 +90226 56.852000 +90276 57.241000 +90333 45.624000 +90336 51.288000 +90337 51.364000 +90404 50.415000 +90433 50.711000 +90434 50.211000 +90444 50.718000 +90445 54.664000 +90544 55.149000 +90548 55.156000 +90641 53.673000 +91042 67.259000 +91072 62.929000 +91098 43.410000 +91103 47.177000 +91266 50.725000 +91302 55.967000 +91407 55.518000 +91627 53.645000 +91681 57.620000 +92038 54.712000 +92056 49.993000 +92057 49.949000 +92069 53.972000 +92077 50.535000 +92082 51.342000 +92101 48.219000 +92125 52.160000 +92135 58.630000 +92136 59.924000 +92140 60.312000 +92229 53.957000 +92264 53.918000 +92270 55.440000 +92305 53.205000 +92309 54.446000 +92312 54.222000 +92376 57.186000 +92377 56.089000 +92410 51.739000 +92422 54.664000 +92453 43.290000 diff --git a/data/shot_lists/jet/ILW_unint_late.txt b/data/shot_lists/jet/ILW_unint_late.txt new file mode 100644 index 00000000..3c6cf97b --- /dev/null +++ b/data/shot_lists/jet/ILW_unint_late.txt @@ -0,0 +1,380 @@ +83980 57.094000 +83982 58.260000 +83987 60.302000 +83999 67.703000 +84031 52.478000 +84036 60.611000 +84051 42.442000 +84054 48.645000 +84077 62.348000 +84079 43.558000 +84115 62.483000 +84142 55.516000 +84143 55.849000 +84158 66.375000 +84160 66.072000 +84161 57.578000 +84162 65.768000 +84163 65.745000 +84164 65.581000 +84165 64.432000 +84166 65.260000 +84167 65.758000 +84168 65.609000 +84169 65.562000 +84170 64.446000 +84171 64.653000 +84172 61.274000 +84173 61.241000 +84180 57.028000 +84271 68.689000 +84275 71.964000 +84308 54.066000 +84315 61.706000 +84322 52.757000 +84323 50.809000 +84324 67.761000 +84328 52.269000 +84352 56.338000 +84408 56.079000 +84428 52.550000 +84433 64.507000 +84435 48.404000 +84462 53.487000 +84464 54.373000 +84465 53.856000 +84467 53.787000 +84517 55.391000 +84519 62.414000 +84520 61.561000 +84524 64.352000 +84581 58.570000 +84595 55.371000 +84600 54.728000 +84620 60.637000 +84628 44.644000 +84639 56.795000 +84640 60.077000 +84648 52.534000 +84655 53.855000 +84656 52.276000 +84657 53.012000 +84658 48.198000 +84671 48.609000 +84674 48.703000 +84676 47.269000 +84677 45.992000 +84679 48.144000 +84681 48.622000 +84690 57.988000 +84691 56.899000 +84695 45.147000 +84696 57.971000 +84707 42.448000 +84715 55.956000 +84716 61.576000 +84748 51.816000 +84808 49.022000 +84820 53.125000 +84826 53.835000 +84827 54.505000 +84828 53.998000 +84829 53.174000 +84857 52.303000 +84858 49.521000 +84861 61.379000 +84862 44.904000 +84869 59.811000 +84870 52.772000 +84912 55.777000 +84933 63.841000 +84935 44.202000 +84937 65.729000 +84942 53.223000 +84943 53.002000 +84946 53.406000 +84947 52.257000 +84950 54.623000 +84955 49.091000 +84957 56.140000 +84960 60.014000 +84962 60.116000 +84965 58.565000 +84986 54.540000 +84987 59.135000 +84988 54.721000 +84989 51.132000 +84990 51.432000 +84991 61.108000 +84994 47.247000 +84995 54.857000 +84996 64.204000 +84998 51.615000 +85005 64.394000 +85033 46.945000 +85048 57.780000 +85070 52.611000 +85072 52.023000 +85073 54.245000 +85074 52.204000 +85076 59.550000 +85077 61.266000 +85162 54.661000 +85164 47.220000 +85165 56.460000 +85168 57.332000 +85170 60.336000 +85171 54.831000 +85176 58.868000 +85177 56.874000 +85179 55.828000 +85182 55.635000 +85185 54.932000 +85187 60.373000 +85188 51.912000 +85189 61.024000 +85190 52.724000 +85191 53.310000 +85192 60.552000 +85193 54.018000 +85200 47.141000 +85202 42.509000 +85204 46.227000 +85212 58.269000 +85222 48.971000 +85229 56.333000 +85236 55.035000 +85238 57.957000 +85239 55.257000 +85240 56.385000 +85241 60.581000 +85251 55.288000 +85252 53.744000 +85253 55.255000 +85257 56.118000 +85258 55.034000 +85259 57.055000 +85285 50.560000 +85286 50.533000 +85287 50.850000 +85290 61.096000 +85291 61.063000 +85292 60.681000 +85293 54.933000 +85294 59.465000 +85295 55.196000 +85300 62.739000 +85359 55.633000 +85360 59.611000 +85362 51.807000 +85363 52.514000 +85387 64.350000 +85390 59.210000 +85396 47.740000 +85401 51.074000 +85410 61.132000 +85411 61.146000 +85447 61.926000 +85455 52.266000 +85457 63.455000 +85460 53.227000 +85467 53.057000 +85472 51.837000 +85473 51.687000 +85482 60.001000 +85483 46.571000 +85484 46.700000 +85551 46.766000 +85554 43.391000 +85593 42.798000 +85603 42.851000 +85643 49.750000 +85674 53.981000 +85720 46.072000 +85722 64.085000 +85727 46.283000 +85753 56.435000 +85763 48.685000 +85767 46.787000 +85771 54.050000 +85774 46.670000 +85779 54.184000 +85780 51.615000 +85792 52.684000 +85808 45.813000 +85809 53.861000 +85843 51.943000 +85861 51.218000 +85865 49.968000 +85867 50.332000 +85869 47.270000 +85882 50.214000 +85883 49.687000 +85925 45.264000 +85944 56.634000 +85958 59.550000 +85959 59.602000 +85964 54.644000 +86102 53.739000 +86129 55.013000 +86268 49.344000 +86291 57.813000 +86337 49.797000 +86448 63.370000 +86449 62.157000 +86542 50.178000 +86545 45.187000 +86558 51.933000 +86580 53.509000 +86585 50.981000 +86586 50.455000 +86588 49.225000 +86591 51.230000 +86601 50.629000 +86603 53.093000 +86614 54.230000 +86826 47.503000 +86866 64.704000 +86911 55.250000 +86986 60.128000 +86991 54.040000 +87011 43.133000 +87056 44.702000 +87068 51.719000 +87134 60.154000 +87151 63.217000 +87248 51.773000 +87254 52.503000 +87356 43.628000 +87434 45.286000 +87435 43.294000 +87441 44.718000 +87477 53.006000 +87478 48.760000 +87490 53.713000 +87510 54.918000 +87515 56.566000 +87640 53.051000 +87643 56.193000 +87644 56.042000 +87647 53.279000 +87649 54.203000 +87702 64.004000 +87713 59.769000 +87714 61.537000 +87719 59.209000 +87720 59.168000 +87728 46.427000 +87743 43.074000 +87746 52.294000 +87751 64.087000 +87757 62.878000 +87782 60.344000 +87787 50.256000 +87790 56.591000 +87791 46.965000 +87796 53.479000 +87797 57.577000 +87798 48.821000 +87811 48.633000 +87812 48.700000 +87813 48.773000 +87820 53.615000 +87821 57.520000 +87822 57.475000 +87824 57.865000 +87826 57.000000 +87827 52.926000 +87828 54.126000 +87832 47.632000 +87848 50.181000 +87856 56.234000 +87857 47.886000 +87859 56.751000 +87861 55.045000 +87863 55.824000 +87864 53.267000 +87878 51.628000 +87942 60.100000 +88098 50.168000 +88197 47.009000 +88220 47.283000 +88238 48.187000 +88239 55.773000 +88258 54.906000 +88367 58.489000 +88420 53.942000 +88494 59.049000 +88498 49.252000 +88499 49.996000 +88511 56.585000 +88520 57.916000 +88535 49.761000 +88547 55.957000 +88550 49.700000 +88577 52.380000 +88613 51.149000 +88627 56.809000 +88632 55.500000 +88736 62.365000 +88745 60.511000 +89127 54.600000 +89182 51.103000 +89186 51.129000 +89192 55.701000 +89325 56.046000 +89418 58.982000 +89439 59.408000 +89635 64.828000 +89659 50.743000 +89774 54.764000 +89780 54.256000 +89843 43.449000 +89892 50.899000 +90164 55.479000 +90218 51.472000 +90219 52.108000 +90226 56.852000 +90276 57.241000 +90333 45.624000 +90336 51.288000 +90337 51.364000 +90404 50.415000 +90433 50.711000 +90434 50.211000 +90444 50.718000 +90445 54.664000 +90544 55.149000 +90548 55.156000 +90641 53.673000 +91042 67.259000 +91072 62.929000 +91098 43.410000 +91103 47.177000 +91266 50.725000 +91302 55.967000 +91407 55.518000 +91627 53.645000 +91681 57.620000 +92038 54.712000 +92056 49.993000 +92057 49.949000 +92069 53.972000 +92077 50.535000 +92082 51.342000 +92101 48.219000 +92125 52.160000 +92135 58.630000 +92136 59.924000 +92140 60.312000 +92229 53.957000 +92264 53.918000 +92270 55.440000 +92305 53.205000 +92309 54.446000 +92312 54.222000 +92376 57.186000 +92377 56.089000 +92410 51.739000 +92422 54.664000 +92453 43.290000 From af7a920af3e03618b68d0cb259d5a6ae2205b712 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Mon, 12 Nov 2018 03:25:25 -0500 Subject: [PATCH 069/272] new option to use later ILW campaigns --- plasma/conf_parser.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index a13ca7e4..18be63fd 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -70,6 +70,9 @@ def parameters(input_file): #shot lists jet_carbon_wall = ShotListFiles(jet,params['paths']['shot_list_dir'],['CWall_clear.txt','CFC_unint.txt'],'jet carbon wall data') jet_iterlike_wall = ShotListFiles(jet,params['paths']['shot_list_dir'],['ILW_unint.txt','BeWall_clear.txt'],'jet iter like wall data') + jet_iterlike_wall_late = ShotListFiles(jet,params['paths']['shot_list_dir'],['ILW_unint_late.txt','ILW_clear_late.txt'],'Late jet iter like wall data') + jet_iterlike_wall_full = ShotListFiles(jet,params['paths']['shot_list_dir'],['ILW_unint_full.txt','ILW_clear_full.txt'],'Full jet iter like wall data') + jenkins_jet_carbon_wall = ShotListFiles(jet,params['paths']['shot_list_dir'],['jenkins_CWall_clear.txt','jenkins_CFC_unint.txt'],'Subset of jet carbon wall data for Jenkins tests') jenkins_jet_iterlike_wall = ShotListFiles(jet,params['paths']['shot_list_dir'],['jenkins_ILW_unint.txt','jenkins_BeWall_clear.txt'],'Subset of jet iter like wall data for Jenkins tests') From 6ab3c32f32426a4dc2275b09ecc9c772577fc2c2 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Mon, 12 Nov 2018 03:29:40 -0500 Subject: [PATCH 070/272] new option to use later ILW campaigns --- plasma/conf_parser.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index 18be63fd..cebe50c4 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -100,6 +100,14 @@ def parameters(input_file): params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = jet_signals_1D + elif params['paths']['data'] == 'jet_data_late': + params['paths']['shot_files'] = [jet_iterlike_wall_late] + params['paths']['shot_files_test'] = [jet_iterlike_wall_late] + params['paths']['use_signals_dict'] = jet_signals + elif params['paths']['data'] == 'jet_data_carbon_to_late_0D': + params['paths']['shot_files'] = [jet_carbon_wall] + params['paths']['shot_files_test'] = [jet_iterlike_wall_late] + params['paths']['use_signals_dict'] = jet_signals elif params['paths']['data'] == 'jet_data_temp_profile': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] @@ -183,6 +191,10 @@ def parameters(input_file): params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = fully_defined_signals + elif params['paths']['data'] == 'd3d_to_late_jet_data': + params['paths']['shot_files'] = [d3d_full] + params['paths']['shot_files_test'] = [jet_iterlike_wall_late] + params['paths']['use_signals_dict'] = fully_defined_signals elif params['paths']['data'] == 'jet_to_d3d_data_0D': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] From d476d41bde063adaff10ab013658a71115bdd5a4 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Mon, 12 Nov 2018 03:45:42 -0500 Subject: [PATCH 071/272] fixed import error --- plasma/utils/downloading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plasma/utils/downloading.py b/plasma/utils/downloading.py index 91f4e968..0f36c65b 100644 --- a/plasma/utils/downloading.py +++ b/plasma/utils/downloading.py @@ -23,7 +23,7 @@ import multiprocessing as mp from functools import partial from multiprocessing import Queue -import os +import os,time import errno import dill,hashlib From 12646bd3becd76f2a71628408c5b58271f9fa4a7 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Mon, 12 Nov 2018 04:11:02 -0500 Subject: [PATCH 072/272] added density & temperature profiles for jet again --- data/signals.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/data/signals.py b/data/signals.py index e6083dc9..eb5c85c2 100644 --- a/data/signals.py +++ b/data/signals.py @@ -137,11 +137,11 @@ def fetch_nstx_data(signal_path,shot_num,c): profile_num_channels = 64 #ZIPFIT comes from actual measurements -#etemp_profile = ProfileSignal("Electron temperature profile",["ppf/hrts/te","ZIPFIT01/PROFILES.ETEMPFIT"],[jet,d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) -#edens_profile = ProfileSignal("Electron density profile",["ppf/hrts/ne","ZIPFIT01/PROFILES.EDENSFIT"],[jet,d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) +etemp_profile = ProfileSignal("Electron temperature profile",["ppf/hrts/te","ZIPFIT01/PROFILES.ETEMPFIT"],[jet,d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[25,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) +edens_profile = ProfileSignal("Electron density profile",["ppf/hrts/ne","ZIPFIT01/PROFILES.EDENSFIT"],[jet,d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[25,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) -etemp_profile = ProfileSignal("Electron temperature profile",["ZIPFIT01/PROFILES.ETEMPFIT"],[d3d],mapping_paths=[None],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) -edens_profile = ProfileSignal("Electron density profile",["ZIPFIT01/PROFILES.EDENSFIT"],[d3d],mapping_paths=[None],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) +# etemp_profile = ProfileSignal("Electron temperature profile",["ZIPFIT01/PROFILES.ETEMPFIT"],[d3d],mapping_paths=[None],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) +# edens_profile = ProfileSignal("Electron density profile",["ZIPFIT01/PROFILES.EDENSFIT"],[d3d],mapping_paths=[None],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) itemp_profile = ProfileSignal("Ion temperature profile",["ZIPFIT01/PROFILES.ITEMPFIT"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) zdens_profile = ProfileSignal("Impurity density profile",["ZIPFIT01/PROFILES.ZDENSFIT"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) trot_profile = ProfileSignal("Rotation profile",["ZIPFIT01/PROFILES.TROTFIT"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) @@ -179,6 +179,7 @@ def fetch_nstx_data(signal_path,shot_num,c): pin = Signal("Input Power (beam for d3d)",['jpf/gs/bl-ptotout'],[jet]) +#pradtot = Signal("Radiated Power",['jpf/db/b5r-ptot>out', 'd3d/'+r'\prad_tot'],[jet,d3d]) #pradcore = ChannelSignal("Radiated Power Core",[ 'd3d/'+r'\bol_l15_p'],[d3d]) #pradedge = ChannelSignal("Radiated Power Edge",['d3d/'+r'\bol_l03_p'],[d3d]) pradcore = ChannelSignal("Radiated Power Core",['ppf/bolo/kb5h/channel14', 'd3d/'+r'\bol_l15_p'],[jet,d3d]) From 23587d8e9baeddb1978927ee1c3900414f0ba211 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Mon, 12 Nov 2018 04:38:31 -0500 Subject: [PATCH 073/272] added density & temperature profiles for jet again --- data/signals.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/data/signals.py b/data/signals.py index eb5c85c2..61659d34 100644 --- a/data/signals.py +++ b/data/signals.py @@ -137,8 +137,8 @@ def fetch_nstx_data(signal_path,shot_num,c): profile_num_channels = 64 #ZIPFIT comes from actual measurements -etemp_profile = ProfileSignal("Electron temperature profile",["ppf/hrts/te","ZIPFIT01/PROFILES.ETEMPFIT"],[jet,d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[25,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) -edens_profile = ProfileSignal("Electron density profile",["ppf/hrts/ne","ZIPFIT01/PROFILES.EDENSFIT"],[jet,d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[25,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) +etemp_profile = ProfileSignal("Electron temperature profile",["ppf/hrts/te","ZIPFIT01/PROFILES.ETEMPFIT"],[jet,d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) +edens_profile = ProfileSignal("Electron density profile",["ppf/hrts/ne","ZIPFIT01/PROFILES.EDENSFIT"],[jet,d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) # etemp_profile = ProfileSignal("Electron temperature profile",["ZIPFIT01/PROFILES.ETEMPFIT"],[d3d],mapping_paths=[None],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) # edens_profile = ProfileSignal("Electron density profile",["ZIPFIT01/PROFILES.EDENSFIT"],[d3d],mapping_paths=[None],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) @@ -159,9 +159,9 @@ def fetch_nstx_data(signal_path,shot_num,c): # epress_profile_spatial = ProfileSignal("Electron pressure profile",["ppf/hrts/pe/"],[jet],causal_shifts=[25],mapping_range=(2,4),num_channels=profile_num_channels) -etemp_profile_spatial = ProfileSignal("Electron temperature profile",["ppf/hrts/te"],[jet],causal_shifts=[25],mapping_range=(2,4),num_channels=profile_num_channels,data_avail_tolerances=[0.05]) -edens_profile_spatial = ProfileSignal("Electron density profile",["ppf/hrts/ne"],[jet],causal_shifts=[25],mapping_range=(2,4),num_channels=profile_num_channels,data_avail_tolerances=[0.05]) -rho_profile_spatial = ProfileSignal("Rho at spatial positions",["ppf/hrts/rho"],[jet],causal_shifts=[25],mapping_range=(2,4),num_channels=profile_num_channels,data_avail_tolerances=[0.05]) +etemp_profile_spatial = ProfileSignal("Electron temperature profile",["ppf/hrts/te"],[jet],causal_shifts=[0],mapping_range=(2,4),num_channels=profile_num_channels,data_avail_tolerances=[0.05]) +edens_profile_spatial = ProfileSignal("Electron density profile",["ppf/hrts/ne"],[jet],causal_shifts=[0],mapping_range=(2,4),num_channels=profile_num_channels,data_avail_tolerances=[0.05]) +rho_profile_spatial = ProfileSignal("Rho at spatial positions",["ppf/hrts/rho"],[jet],causal_shifts=[0],mapping_range=(2,4),num_channels=profile_num_channels,data_avail_tolerances=[0.05]) etemp = Signal("electron temperature",["ppf/hrtx/te0"],[jet],causal_shifts=[25],data_avail_tolerances=[0.05]) # epress = Signal("electron pressure",["ppf/hrtx/pe0/"],[jet],causal_shifts=[25]) From 8641d40ec85973703ac282041ca8351194819025 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Fri, 16 Nov 2018 00:02:32 -0500 Subject: [PATCH 074/272] changed jet profiles to available --- data/signals.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data/signals.py b/data/signals.py index e6083dc9..88533887 100644 --- a/data/signals.py +++ b/data/signals.py @@ -137,11 +137,11 @@ def fetch_nstx_data(signal_path,shot_num,c): profile_num_channels = 64 #ZIPFIT comes from actual measurements -#etemp_profile = ProfileSignal("Electron temperature profile",["ppf/hrts/te","ZIPFIT01/PROFILES.ETEMPFIT"],[jet,d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) -#edens_profile = ProfileSignal("Electron density profile",["ppf/hrts/ne","ZIPFIT01/PROFILES.EDENSFIT"],[jet,d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) +etemp_profile = ProfileSignal("Electron temperature profile",["ppf/hrts/te","ZIPFIT01/PROFILES.ETEMPFIT"],[jet,d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) +edens_profile = ProfileSignal("Electron density profile",["ppf/hrts/ne","ZIPFIT01/PROFILES.EDENSFIT"],[jet,d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) -etemp_profile = ProfileSignal("Electron temperature profile",["ZIPFIT01/PROFILES.ETEMPFIT"],[d3d],mapping_paths=[None],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) -edens_profile = ProfileSignal("Electron density profile",["ZIPFIT01/PROFILES.EDENSFIT"],[d3d],mapping_paths=[None],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) +#etemp_profile = ProfileSignal("Electron temperature profile",["ZIPFIT01/PROFILES.ETEMPFIT"],[d3d],mapping_paths=[None],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) +#edens_profile = ProfileSignal("Electron density profile",["ZIPFIT01/PROFILES.EDENSFIT"],[d3d],mapping_paths=[None],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) itemp_profile = ProfileSignal("Ion temperature profile",["ZIPFIT01/PROFILES.ITEMPFIT"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) zdens_profile = ProfileSignal("Impurity density profile",["ZIPFIT01/PROFILES.ZDENSFIT"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) trot_profile = ProfileSignal("Rotation profile",["ZIPFIT01/PROFILES.TROTFIT"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) From 8c48c7bd483aebea79a0882dec0fde3f4068840e Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Fri, 16 Nov 2018 01:59:24 -0500 Subject: [PATCH 075/272] keywords --- plasma/models/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plasma/models/builder.py b/plasma/models/builder.py index 28459936..bc2de201 100644 --- a/plasma/models/builder.py +++ b/plasma/models/builder.py @@ -182,7 +182,7 @@ def slicer_output_shape(input_shape,indices): else: pre_rnn = pre_rnn_input - if model_conf['rnn_layers'] == 0 or model_conf['extra_dense_input']: + if model_conf['rnn_layers'] == 0 or ('extra_dense_input' in model_conf.keys() and model_conf['extra_dense_input']): pre_rnn = Dense(dense_size,activation='relu',kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn) pre_rnn = Dense(dense_size//2,activation='relu',kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn) pre_rnn = Dense(dense_size//4,activation='relu',kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn) From c0d00819438d839eabc9bde84f172e41df50ab05 Mon Sep 17 00:00:00 2001 From: Julian Kates-Harbeck Date: Fri, 23 Nov 2018 20:25:46 -0500 Subject: [PATCH 076/272] made signal hash dependent on number of machines (explicitly on paths) --- plasma/conf_parser.py | 6 +++--- plasma/primitives/data.py | 12 +++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index cebe50c4..bcd8e293 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -102,12 +102,12 @@ def parameters(input_file): params['paths']['use_signals_dict'] = jet_signals_1D elif params['paths']['data'] == 'jet_data_late': params['paths']['shot_files'] = [jet_iterlike_wall_late] - params['paths']['shot_files_test'] = [jet_iterlike_wall_late] + params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = jet_signals elif params['paths']['data'] == 'jet_data_carbon_to_late_0D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall_late] - params['paths']['use_signals_dict'] = jet_signals + params['paths']['use_signals_dict'] = jet_signals_0D elif params['paths']['data'] == 'jet_data_temp_profile': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] @@ -244,7 +244,7 @@ def parameters(input_file): return params def get_unique_signal_hash(signals): - return int(hashlib.md5(''.join(tuple(map(lambda x: x.description, sorted(signals)))).encode('utf-8')).hexdigest(),16) + return int(hashlib.md5(''.join(tuple(map(lambda x: "{}".format(x.__hash__()), sorted(signals)))).encode('utf-8')).hexdigest(),16) #make sure 1D signals come last! This is necessary for model builder. def sort_by_channels(list_of_signals): diff --git a/plasma/primitives/data.py b/plasma/primitives/data.py index fb19f466..e2b1ed49 100644 --- a/plasma/primitives/data.py +++ b/plasma/primitives/data.py @@ -171,21 +171,23 @@ def get_idx(self,machine): idx = self.machines.index(machine) return idx + def description_plus_paths(self): + return self.description + ' ' + ' '.join(self.paths) + def __eq__(self,other): if other is None: return False - return self.description.__eq__(other.description) - + return self.description_plus_paths().__eq__(other.description_plus_paths()) def __ne__(self,other): - return self.description.__ne__(other.description) + return self.description_plus_paths().__ne__(other.description_plus_paths()) def __lt__(self,other): - return self.description.__lt__(other.description) + return self.description_plus_paths().__lt__(other.description_plus_paths()) def __hash__(self): import hashlib - return int(hashlib.md5(self.description.encode('utf-8')).hexdigest(),16) + return int(hashlib.md5(self.description_plus_paths().encode('utf-8')).hexdigest(),16) def __str__(self): return self.description From 78da591167818038f8bdeb199b9b82e1a61437c8 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Fri, 18 Jan 2019 14:30:32 -0500 Subject: [PATCH 077/272] Fix link to tutorial in README.md (use relative link) --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9a96c8f5..d6cfb48b 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ It consists of 4 core modules: - `primitives`: contains abstractions specific to the domain, implemented as Python classes. For instance: Shot - a measurement of plasma current as a function of time. The Shot object contains attributes corresponding to unique identifier of a shot, disruption time in milliseconds, time profile of the shot converted to time-to- disruption values, validity of a shot (whether plasma current reaches a certain value during the shot), etc. Other primitives include `Machines` and `Signals` which carry the relevant information necessary for incorporating physics data into the overall pipeline. Signals know the Machine they live on, their mds+ paths, code for being downloaded, preprocessing approaches, their dimensionality, etc. Machines know which Signals are defined on them, which mds+ server houses the data, etc. -- `utilities`: a set of auxiliary functions for preprocessing, performance evaluation and learning curves analysis. +- `utilities`: a set of auxiliary functions for preprocessing, performance evaluation and learning curves analysis. In addition to the `utilities` FRNN supports TensorBoard scaler variable summaries, histogramms of layers, activations and gradients and graph visualizations. @@ -47,5 +47,4 @@ The Sphinx pages for FRNN are building up here: http://tigress-web.princeton.edu ## Tutorials -For tutorial check: -https://github.com/PPPLDeepLearning/plasma-python/blob/mpicc-travis/docs/PrincetonUTutorial.md +For a tutorial, check out: [PrincetonUTutorial.md](docs/PrincetonUTutorial.md) From 888c10439fae695f0d9e09d3b7d4d4bc68ed9b34 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Fri, 18 Jan 2019 14:57:22 -0500 Subject: [PATCH 078/272] Update ANL_Theta.md --- docs/ANL_Theta.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ANL_Theta.md b/docs/ANL_Theta.md index cc1a68bb..abc2f2eb 100644 --- a/docs/ANL_Theta.md +++ b/docs/ANL_Theta.md @@ -1,4 +1,4 @@ -#First time setup on Theta, Argonne +# First time setup on Theta, Argonne ```bash mkdir PPPL From 82b60ec65cca75ceb71876714697302338bb0676 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Fri, 18 Jan 2019 15:06:26 -0500 Subject: [PATCH 079/272] Update Primitives.md --- docs/Primitives.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/Primitives.md b/docs/Primitives.md index 253aa349..d1f38ad2 100644 --- a/docs/Primitives.md +++ b/docs/Primitives.md @@ -2,11 +2,11 @@ Each shot is a measurement of plasma current as a function of time. The Shot objects contains following attributes: - 1. number - integer, unique identifier of a shot - 1. t_disrupt - double, disruption time in milliseconds (second column in the shotlist input file) - 1. ttd - array of doubles, time profile of the shot converted to time-to-disruption values - 1. valid - boolean, whether plasma current reaches a certain value during the shot - 1. is_disruptive - boolean, + 1. `number` - integer, unique identifier of a shot + 1. `t_disrupt` - double, disruption time in milliseconds (second column in the shotlist input file) + 1. `ttd` - array of doubles, time profile of the shot converted to time-to-disruption values + 1. `valid` - boolean, whether plasma current reaches a certain value during the shot + 1. `is_disruptive` - boolean, whether the shot was determined to be disruptive by an expert For 0D data, each shot is modeled as 2D array - time vs plasma current. From f3a69582a9abe051e9b3fed4a1f0c7b226329d3b Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Sun, 24 Feb 2019 17:27:33 -0500 Subject: [PATCH 080/272] Update ICC 17 version available on TigerGPU --- docs/PrincetonUTutorial.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/PrincetonUTutorial.md b/docs/PrincetonUTutorial.md index 61f45d52..b9a97565 100644 --- a/docs/PrincetonUTutorial.md +++ b/docs/PrincetonUTutorial.md @@ -1,13 +1,13 @@ ## Tutorials -### Login to Tigergpu +### Login to TigerGPU First, login to TigerGPU cluster headnode via ssh: ``` ssh -XC @tigergpu.princeton.edu ``` -### Sample usage on Tigergpu +### Sample usage on TigerGPU Next, check out the source code from github: ``` @@ -26,7 +26,7 @@ export OMPI_MCA_btl="tcp,self,sm" module load cudatoolkit/8.0 module load cudnn/cuda-8.0/6.0 module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 -module load intel/17.0/64/17.0.4.196 +module load intel/17.0/64/17.0.5.239 ``` and install the `plasma-python` package: From a78867aee1e4f20388f07a0d590c0d9e8fb47534 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 11 Sep 2019 15:42:51 -0500 Subject: [PATCH 081/272] Add encrypted credentials for new Slack plugin --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 03026741..50f292fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,3 +34,7 @@ env: - TEST_DIR=.; TEST_SCRIPT="python setup.py test" script: cd $TEST_DIR && $TEST_SCRIPT && cd .. + +notifications: + slack: + secure: NBjGYpIF2VO/GvhbC7XVPIfi0WLGFEuVhi51UbZjfHg4IOv1UuCF0fImi8GN2UZZBFRnZcbtPVffZnyMUyJI0Krw0M1ropkjA4YDaJx1JUGnYuQLxNmX+jbQmN61Usjg5MQOgcRnAn1bdMN1ttWInqkKejpV5buCjZbt8SZbDePfXz4U3No68P/pRsDTVXSy0xLTtRacuEITJjwxfjp6phbmR3qs127MZMRbVYDC2HA6KsoJW6YSKF1vFyHqnFMl7GSavxYw/XQpqFLJkGKXfnNgPZV6qAbVk5+bzyytAbbwLGvgbuFpnJsGPvAyebV8wVYaIPg7OeK+Sm1A3q0jt774NnqFp2AZ+pSKrxbxIkygDM0zoWLm4i3pt6ToJ6dKcqSbCKbELnSY5NphcyYuiJ8uhLVpAR0Y+vp+fOvhb8td/nH2AWkxFpp6xwOHPXvtodBsyPMkiaeKoVElYBfbfOhDSYH2KafADECTX75S63A9KleeNZh0DSImfFdQPaN3GFLEL8Z9UFABzTkM9eYKP9pyEP82Wh/JGDaqbARazEgNzy8rwghomsEguV247XHdOx32PSd0att541gsRFZ5uyMuVFKzI0jiLijekibY2I1c5b6dDeuK4O8uBiq4FTS+bM55Rj4Job01kdxSCKw3RwOh0amzITRTQvEWKpTAekg= From a992a93db9c201e127f8bf200bc5d6b58b025721 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 11 Sep 2019 15:46:38 -0500 Subject: [PATCH 082/272] Update broken Travis CI badge --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d6cfb48b..5ec11e38 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,7 @@ -# FRNN [![Build Status](https://travis-ci.org/PPPLDeepLearning/plasma-python.svg?branch=master)](https://travis-ci.org/PPPLDeepLearning/plasma-python.svg?branch=master) [![Build Status](https://jenkins.princeton.edu/buildStatus/icon?job=FRNM/PPPL)](https://jenkins.princeton.edu/job/FRNM/job/PPPL/) +# FRNN + +[![Build Status](https://travis-ci.org/PPPLDeepLearning/plasma-python.svg?branch=master)](https://travis-ci.org/PPPLDeepLearning/plasma-python) +[![Build Status](https://jenkins.princeton.edu/buildStatus/icon?job=FRNM/PPPL)](https://jenkins.princeton.edu/job/FRNM/job/PPPL/) ## Package description From c4236ea31a5882f2f180674457e7d8e6673a01d2 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 17 Sep 2019 13:37:09 -0500 Subject: [PATCH 083/272] Add basic flake8 linting style rules to setup.cfg Ported from Athena++ linter rules --- setup.cfg | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/setup.cfg b/setup.cfg index b88034e4..912ef080 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,29 @@ [metadata] description-file = README.md + +[flake8] +max-line-length = 79 +# exclude = cpplint.py +ignore = + # Subset of DEFAULT_IGNORE error codes from pycodestyle + # (not universally accepted / not enforced by PEP 8 document) + # E1: Indentation + # E121: continuation line under-indented for hanging indent (allow fewer than 4 spaces w/ hanging indent) + E121, + # E123: closing bracket does not match indentation of opening bracket’s line + E123, + # E126: continuation line over-indented for hanging inden (allow fewer than 4 spaces w/ hanging indent) + E126, + # E2: Whitespace + # E226: missing whitespace around arithmetic operator (allow nx+1, ...) + E226, + # E241: multiple spaces after ‘,’ + E241, + # E7: Statements + # E704: multiple statements on one line (def) + E704, + # E731: Do not assign a lambda expression, use a def + E731, + # W5: Line break warning + # W503: line break before binary operator (use mutually exclusive W504) + W503 \ No newline at end of file From 3dcf8636cfc0970b6465763520288056dc057c3c Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 19 Sep 2019 16:09:03 -0500 Subject: [PATCH 084/272] Start applying subset of PEP8 style rules to existing files Via autopep8 + manual touchups --- jenkins-ci/jenkins.sh | 18 ---------- jenkins-ci/run_jenkins.py | 51 +++++++++++++++++--------- jenkins-ci/validate_jenkins.py | 12 +++---- makedocs.py | 17 +++++---- setup.py | 66 ++++++++++++++++++---------------- tests/testbasic.py | 2 +- 6 files changed, 87 insertions(+), 79 deletions(-) diff --git a/jenkins-ci/jenkins.sh b/jenkins-ci/jenkins.sh index 1fa10ffe..6dbf1170 100644 --- a/jenkins-ci/jenkins.sh +++ b/jenkins-ci/jenkins.sh @@ -31,21 +31,3 @@ sed -i -e 's/data: jet_data/data: jenkins_jet/g' conf.yaml srun python mpi_learn.py echo "Jenkins test Python2.7" -#rm /tigress/alexeys/model_checkpoints/* - -#source deactivate -#module purge -#module load anaconda/4.4.0 -#source activate /tigress/alexeys/jenkins/.conda/envs/jenkins2 -#module load cudatoolkit/8.0 -#module load cudnn/cuda-8.0/6.0 -#module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 -#module load intel/17.0/64/17.0.4.196 - -#cd .. -#python setup.py install - -#echo $SLURM_NODELIST -#cd examples -#sed -i -e 's/data: jenkins_jet/data: jenkins_d3d/g' conf.yaml -#srun python mpi_learn.py diff --git a/jenkins-ci/run_jenkins.py b/jenkins-ci/run_jenkins.py index 57be2675..282b3900 100644 --- a/jenkins-ci/run_jenkins.py +++ b/jenkins-ci/run_jenkins.py @@ -1,19 +1,28 @@ -from plasma.utils.batch_jobs import generate_working_dirname,copy_files_to_environment,start_jenkins_job +from plasma.utils.batch_jobs import ( + generate_working_dirname, copy_files_to_environment, start_jenkins_job + ) import yaml -import sys,os,getpass +import os +import getpass import plasma.conf -num_nodes = 4 #Set in the Jenkins project area!! -test_matrix = [("Python3","jet_data"),("Python2","jet_data")] +num_nodes = 4 # Set in the Jenkins project area!! +test_matrix = [("Python3", "jet_data"), ("Python2", "jet_data")] -run_directory = "{}/{}/jenkins/".format(plasma.conf.conf['fs_path'],getpass.getuser()) +run_directory = "{}/{}/jenkins/".format( + plasma.conf.conf['fs_path'], getpass.getuser()) template_path = os.environ['PWD'] conf_name = "conf.yaml" executable_name = "mpi_learn.py" -def generate_conf_file(test_configuration,template_path = "../",save_path = "./",conf_name="conf.yaml"): + +def generate_conf_file( + test_configuration, + template_path="../", + save_path="./", + conf_name="conf.yaml"): assert(template_path != save_path) - with open(os.path.join(template_path,conf_name), 'r') as yaml_file: + with open(os.path.join(template_path, conf_name), 'r') as yaml_file: conf = yaml.load(yaml_file) conf['training']['num_epochs'] = 2 conf['paths']['data'] = test_configuration[1] @@ -24,25 +33,30 @@ def generate_conf_file(test_configuration,template_path = "../",save_path = "./" conf['env']['name'] = "PPPL" conf['env']['type'] = "anaconda" - with open(os.path.join(save_path,conf_name), 'w') as outfile: + with open(os.path.join(save_path, conf_name), 'w') as outfile: yaml.dump(conf, outfile, default_flow_style=False) return conf + working_directory = generate_working_dirname(run_directory) os.makedirs(working_directory) -os.system(" ".join(["cp -p",os.path.join(template_path,conf_name),working_directory])) -os.system(" ".join(["cp -p",os.path.join(template_path,executable_name),working_directory])) +os.system( + " ".join(["cp -p", os.path.join(template_path, conf_name), + working_directory])) +os.system(" ".join( + ["cp -p", os.path.join(template_path, executable_name), + working_directory])) -#os.chdir(working_directory) -#print("Going into {}".format(working_directory)) +# os.chdir(working_directory) +# print("Going into {}".format(working_directory)) for ci in test_matrix: - subdir = working_directory + "/{}/".format(ci[0]) + subdir = working_directory + "/{}/".format(ci[0]) os.makedirs(subdir) copy_files_to_environment(subdir) print("Making modified conf") - conf = generate_conf_file(ci,working_directory,subdir,conf_name) + conf = generate_conf_file(ci, working_directory, subdir, conf_name) print("Starting job") if ci[1] == "Python3": env_name = "PPPL_dev3" @@ -50,7 +64,12 @@ def generate_conf_file(test_configuration,template_path = "../",save_path = "./" else: env_name = "PPPL" env_type = "anaconda" - start_jenkins_job(subdir,num_nodes,executable_name,ci,env_name,env_type) - + start_jenkins_job( + subdir, + num_nodes, + executable_name, + ci, + env_name, + env_type) print("submitted jobs.") diff --git a/jenkins-ci/validate_jenkins.py b/jenkins-ci/validate_jenkins.py index 6014fb2b..abed06f4 100644 --- a/jenkins-ci/validate_jenkins.py +++ b/jenkins-ci/validate_jenkins.py @@ -1,17 +1,15 @@ #!/usr/bin/env python +import sys +from mpi4py import MPI +import tensorflow as tf +import keras as kk import mpi4py as mmm -print(mmm.__version__) -import keras as kk +print(mmm.__version__) print(kk.__version__) - -import tensorflow as tf print(tf.__version__) -from mpi4py import MPI -import sys - size = MPI.COMM_WORLD.Get_size() rank = MPI.COMM_WORLD.Get_rank() name = MPI.Get_processor_name() diff --git a/makedocs.py b/makedocs.py index 51446a2b..fca2371e 100644 --- a/makedocs.py +++ b/makedocs.py @@ -14,7 +14,7 @@ "plasma.preprocessor.preprocess", "plasma.utils.evaluation", "plasma.utils.performance", - "plasma.utils.processing" + "plasma.utils.processing" ] modules = {name: importlib.import_module(name) for name in modules} @@ -23,11 +23,13 @@ for moduleName, module in modules.items(): for objName in dir(module): obj = getattr(module, objName) - if not objName.startswith("_") and callable(obj) and obj.__module__ == moduleName: - print objName, obj + if (not objName.startswith("_") + and callable(obj) and obj.__module__ == moduleName): + print(objName, obj) documented.append(moduleName + "." + objName) if inspect.isclass(obj): - open("docs/" + moduleName + "." + objName + ".rst", "w").write(''':orphan: + open("docs/" + moduleName + "." + objName + ".rst", "w").write( + ''':orphan: {0} {1} @@ -37,12 +39,13 @@ :special-members: __init__, __add__ :inherited-members: :show-inheritance: -'''.format(moduleName + "." + objName, "=" * (len(moduleName) + len(objName) + 1))) +'''.format(moduleName + "." + objName, "="*(len(moduleName)+len(objName)+1))) else: - open("docs/" + moduleName + "." + objName + ".rst", "w").write(''':orphan: + open("docs/" + moduleName + "." + objName + ".rst", + "w").write(''':orphan: {0} {1} .. autofunction:: {0} -'''.format(moduleName + "." + objName, "=" * (len(moduleName) + len(objName) + 1))) +'''.format(moduleName + "." + objName, "="*(len(moduleName)+len(objName)+1))) diff --git a/setup.py b/setup.py index ab3be99f..5c8fec7f 100644 --- a/setup.py +++ b/setup.py @@ -1,40 +1,46 @@ import os -import sys import subprocess from setuptools import setup, find_packages import plasma.version try: - os.environ['MPICC'] = subprocess.check_output("which mpicc", shell=True).decode("utf-8") -except: - print ("Please set up the OpenMPI environment") + os.environ['MPICC'] = subprocess.check_output( + "which mpicc", shell=True).decode("utf-8") +except BaseException: + print("Please set up the OpenMPI environment") exit(1) -setup(name = "plasma", - version = plasma.version.__version__, - packages = find_packages(), - #scripts = [""], - description = "PPPL deep learning package.", - long_description = """Add description here""", - author = "Julian Kates-Harbeck, Alexey Svyatkovskiy", - author_email = "jkatesharbeck@g.harvard.edu", - maintainer = "Alexey Svyatkovskiy", - maintainer_email = "alexeys@princeton.edu", - #url = "http://", - download_url = "https://github.com/PPPLDeepLearning/plasma-python", - #license = "Apache Software License v2", - test_suite = "tests", - install_requires = ['keras>2.0.8','pathos','matplotlib==2.0.2','hyperopt','mpi4py','xgboost'], - tests_require = [], - classifiers = ["Development Status :: 3 - Alpha", - "Environment :: Console", - "Intended Audience :: Science/Research", - "Programming Language :: Python", - "Topic :: Scientific/Engineering :: Information Analysis", - "Topic :: Scientific/Engineering :: Physics", - "Topic :: Scientific/Engineering :: Mathematics", - "Topic :: System :: Distributed Computing", - ], - platforms = "Any", +setup(name="plasma", + version=plasma.version.__version__, + packages=find_packages(), + # scripts = [""], + description="PPPL deep learning package.", + long_description="""Add description here""", + author="Julian Kates-Harbeck, Alexey Svyatkovskiy", + author_email="jkatesharbeck@g.harvard.edu", + maintainer="Alexey Svyatkovskiy", + maintainer_email="alexeys@princeton.edu", + # url = "http://", + download_url="https://github.com/PPPLDeepLearning/plasma-python", + # license = "Apache Software License v2", + test_suite="tests", + install_requires=[ + 'keras>2.0.8', + 'pathos', + 'matplotlib==2.0.2', + 'hyperopt', + 'mpi4py', + 'xgboost'], + tests_require=[], + classifiers=["Development Status :: 3 - Alpha", + "Environment :: Console", + "Intended Audience :: Science/Research", + "Programming Language :: Python", + "Topic :: Scientific/Engineering :: Information Analysis", + "Topic :: Scientific/Engineering :: Physics", + "Topic :: Scientific/Engineering :: Mathematics", + "Topic :: System :: Distributed Computing", + ], + platforms="Any", ) diff --git a/tests/testbasic.py b/tests/testbasic.py index 6fb2f4b7..261b31ab 100644 --- a/tests/testbasic.py +++ b/tests/testbasic.py @@ -1,5 +1,5 @@ -import sys import unittest + class TestBasic(unittest.TestCase): pass From 1e985a9223853867441eb9f84e2d3de0e3f07f0e Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Mon, 23 Sep 2019 17:39:07 -0500 Subject: [PATCH 085/272] Apply PEP8 styling to examples/*.py --- examples/analyze_tuning.py | 103 ++++++++------- examples/check_tuning.py | 32 ++--- examples/compare_performance.py | 59 +++++---- examples/custom_plot.py | 42 ++++-- examples/distributed_job.sh | 3 +- examples/extract_best_overtime.py | 126 ++++++++++-------- examples/guarantee_preprocessed.py | 9 +- examples/hyper_learn.py | 37 +++--- examples/individual_shot_performance.py | 21 ++- examples/learn.py | 108 ++++++++------- examples/mpi_augment_learn.py | 104 ++++++++------- examples/mpi_learn.py | 97 ++++++++------ examples/performance_analysis.py | 101 ++++++++------ examples/prepare_pbs_configs_titan.py | 111 ++++++++-------- examples/prepare_slurm_configs.py | 48 ++++--- examples/signal_influence.py | 143 ++++++++++++-------- examples/simple_augmentation.py | 136 +++++++++++-------- examples/submit_batch_job.py | 54 +++++--- examples/test.py | 69 +++++----- examples/tune_hyperparams.py | 167 ++++++++++++++++-------- 20 files changed, 923 insertions(+), 647 deletions(-) diff --git a/examples/analyze_tuning.py b/examples/analyze_tuning.py index 1cb91f0f..f93b7f5a 100644 --- a/examples/analyze_tuning.py +++ b/examples/analyze_tuning.py @@ -1,18 +1,22 @@ -from plasma.primitives.hyperparameters import CategoricalHyperparam,ContinuousHyperparam,LogContinuousHyperparam,HyperparamExperiment +from plasma.primitives.hyperparameters import ( + # CategoricalHyperparam, ContinuousHyperparam, + # LogContinuousHyperparam, + HyperparamExperiment +) import matplotlib.pylab as plt -from pprint import pprint -import yaml -import datetime -import uuid -import sys,os,getpass -import shutil -import subprocess as sp -import pandas +# from pprint import pprint +# import yaml, datetime, uuid +import sys +import os +import getpass +# import shutil, pandas +# import subprocess as sp import numpy as np import plasma.conf -dir_path = "/{}/{}/hyperparams/".format(plasma.conf.conf['fs_path'],getpass.getuser()) +dir_path = "/{}/{}/hyperparams/".format( + plasma.conf.conf['fs_path'], getpass.getuser()) if len(sys.argv) <= 1: dir_path = dir_path + os.listdir(dir_path)[0] + '/' @@ -21,80 +25,85 @@ dir_path = sys.argv[1] -def get_experiments(path,verbose=0): +def get_experiments(path, verbose=0): experiments = [] num_tot = 0 num_finished = 0 num_success = 0 for name in sorted(os.listdir(path)): - if os.path.isdir(os.path.join(path,name)): - print(os.path.join(path,name)) - exp= HyperparamExperiment(os.path.join(path,name)) + if os.path.isdir(os.path.join(path, name)): + print(os.path.join(path, name)) + exp = HyperparamExperiment(os.path.join(path, name)) num_finished += 1 if exp.finished else 0 num_success += 1 if exp.success else 0 num_tot += 1 experiments.append(exp) - if verbose: - print("Read {} experiments, {} finished ({} success)".format(num_tot,num_finished,num_success)) + if verbose: + print("Read {} experiments, {} finished ({} success)".format( + num_tot, num_finished, num_success)) return experiments + experiments = sorted(get_experiments(dir_path)) -best_experiments = np.argsort(np.array([e.get_maximum(False)[0] for e in experiments])) -best = [] +best_experiments = np.argsort( + np.array([e.get_maximum(False)[0] for e in experiments])) +best = [] for e in np.array(experiments)[best_experiments][-5:]: best.append(e.get_number()) bigdict = {} for base in best: - f = "/{}/{}/changed_params.out".format(dir_path,base) + f = "/{}/{}/changed_params.out".format(dir_path, base) data = open(f).readlines() for line in data: tuples = line.split(":") - #if len(tuples) == 2: + # if len(tuples) == 2: key, values = tuples[-2:] - key = key.strip() + key = key.strip() try: bigdict[key] += [values] except KeyError: bigdict[key] = [values] -def make_comparison_plot(key,tunable,trial): - values,edges = tunable +def make_comparison_plot(key, tunable, trial): + values, edges = tunable - trial = list(map(lambda x: eval(x),trial)) - trial_values,_ = np.histogram(trial,bins=edges) + trial = list(map(lambda x: eval(x), trial)) + trial_values, _ = np.histogram(trial, bins=edges) total = trial_values.sum() - values_percentages =list(map(lambda x: x*100.0/total, trial_values)) + values_percentages = list(map(lambda x: x*100.0/total, trial_values)) - plt.bar(edges[:-1], values_percentages, width=np.diff(edges), ec="k", align="edge") + plt.bar(edges[:-1], values_percentages, + width=np.diff(edges), ec="k", align="edge") plt.xlabel(key, fontsize=20) - #plt.yscale('log') + # plt.yscale('log') plt.ylabel('Fraction of trials [%]/bin', fontsize=20) plt.bar(edges[:-1], values, width=np.diff(edges), ec="k", align="edge") plt.savefig(key+".png") - #plt.show() + # plt.show() plt.clf() -#default tunables: +# default tunables: defaults = {} -defaults['lr'] = np.histogram([1e-7,1e-4]) -defaults['lr_decay'] = np.histogram([0.97,0.985,1.0]) -defaults['positive_example_penalty'] = np.histogram([1.0,4.0,16.0]) -#defaults['target'] = np.histogram([50,50,50],bins=['hinge','ttdinv','ttd']) -defaults['batch_size'] = np.histogram([64,256,1024]) -defaults['dropout_prob'] = np.histogram([0.1,0.3,0.5]) -defaults['rnn_layers'] = np.histogram([1,4]) -defaults['rnn_size'] = np.histogram([100,200,300]) -defaults['num_conv_filters'] = np.histogram([5,10]) -defaults['num_conv_layers'] = np.histogram([2,4]) -defaults['T_warning'] = np.histogram([0.256,1.024,10.024]) -defaults['cut_shot_ends'] = np.histogram([False,True]) - -#Histogram it -for key,trial in bigdict.items(): - if key == 'target': continue - make_comparison_plot(key,defaults[key],trial) +defaults['lr'] = np.histogram([1e-7, 1e-4]) +defaults['lr_decay'] = np.histogram([0.97, 0.985, 1.0]) +defaults['positive_example_penalty'] = np.histogram([1.0, 4.0, 16.0]) +# defaults['target'] = np.histogram([50,50,50],bins=['hinge','ttdinv','ttd']) +defaults['batch_size'] = np.histogram([64, 256, 1024]) +defaults['dropout_prob'] = np.histogram([0.1, 0.3, 0.5]) +defaults['rnn_layers'] = np.histogram([1, 4]) +defaults['rnn_size'] = np.histogram([100, 200, 300]) +defaults['num_conv_filters'] = np.histogram([5, 10]) +defaults['num_conv_layers'] = np.histogram([2, 4]) +defaults['T_warning'] = np.histogram([0.256, 1.024, 10.024]) +defaults['cut_shot_ends'] = np.histogram([False, True]) + +# Histogram it +for key, trial in bigdict.items(): + if key == 'target': + continue + make_comparison_plot(key, defaults[key], trial) diff --git a/examples/check_tuning.py b/examples/check_tuning.py index b65581f6..ce2875c4 100644 --- a/examples/check_tuning.py +++ b/examples/check_tuning.py @@ -1,16 +1,15 @@ -from plasma.primitives.hyperparameters import CategoricalHyperparam,ContinuousHyperparam,LogContinuousHyperparam,HyperparamExperiment -from pprint import pprint -import yaml -import datetime -import uuid -import sys,os,getpass -import shutil -import subprocess as sp -import pandas +from plasma.primitives.hyperparameters import ( + # CategoricalHyperparam, ContinuousHyperparam, LogContinuousHyperparam, + HyperparamExperiment +) +import sys +import os +import getpass import numpy as np import plasma.conf -dir_path = "/{}/{}/hyperparams/".format(plasma.conf.conf['fs_path'],getpass.getuser()) +dir_path = "/{}/{}/hyperparams/".format( + plasma.conf.conf['fs_path'], getpass.getuser()) if len(sys.argv) <= 1: dir_path = dir_path + os.listdir(dir_path)[0] + '/' print("using default dir {}".format(dir_path)) @@ -24,19 +23,22 @@ def get_experiments(path): num_finished = 0 num_success = 0 for name in sorted(os.listdir(path)): - if os.path.isdir(os.path.join(path,name)): - print(os.path.join(path,name)) - exp= HyperparamExperiment(os.path.join(path,name)) + if os.path.isdir(os.path.join(path, name)): + print(os.path.join(path, name)) + exp = HyperparamExperiment(os.path.join(path, name)) num_finished += 1 if exp.finished else 0 num_success += 1 if exp.success else 0 num_tot += 1 experiments.append(exp) - print("Read {} experiments, {} finished ({} success)".format(num_tot,num_finished,num_success)) + print("Read {} experiments, {} finished ({} success)".format( + num_tot, num_finished, num_success)) return experiments + experiments = sorted(get_experiments(dir_path)) print(len(experiments)) -best_experiments = np.argsort(np.array([e.get_maximum(False)[0] for e in experiments])) +best_experiments = np.argsort( + np.array([e.get_maximum(False)[0] for e in experiments])) for e in experiments: e.summary() print("Best experiment so far: \n") diff --git a/examples/compare_performance.py b/examples/compare_performance.py index 498927f8..b91d0ff0 100644 --- a/examples/compare_performance.py +++ b/examples/compare_performance.py @@ -1,10 +1,8 @@ -import os,sys -import numpy as np - -from plasma.utils.performance import * +import sys +from plasma.utils.performance import PerformanceAnalyzer from plasma.conf import conf -#mode = 'test' +# mode = 'test' file_num = 0 save_figure = True pred_ttd = False @@ -12,51 +10,64 @@ # cut_shot_ends = conf['data']['cut_shot_ends'] # dt = conf['data']['dt'] # T_max_warn = int(round(conf['data']['T_warning']/dt)) -# T_min_warn = conf['data']['T_min_warn']#int(round(conf['data']['T_min_warn']/dt)) +# T_min_warn = conf['data']['T_min_warn'] +# T_min_warn = int(round(conf['data']['T_min_warn']/dt)) # if cut_shot_ends: # T_max_warn = T_max_warn-T_min_warn # T_min_warn = 0 -T_min_warn = 30 #None #take value from conf #30 +T_min_warn = 30 # None #take value from conf #30 -verbose=False +verbose = False assert(sys.argv > 1) results_dirs = sys.argv[1:] shots_dir = conf['paths']['processed_prepath'] -analyzers = [PerformanceAnalyzer(conf=conf,results_dir=results_dir,shots_dir=shots_dir,i = file_num,T_min_warn = T_min_warn, verbose = verbose, pred_ttd=pred_ttd) for results_dir in results_dirs] +analyzers = [PerformanceAnalyzer(conf=conf, results_dir=results_dir, + shots_dir=shots_dir, i=file_num, + T_min_warn=T_min_warn, + verbose=verbose, pred_ttd=pred_ttd) + for results_dir in results_dirs] for analyzer in analyzers: analyzer.load_ith_file() analyzer.verbose = False -P_threshs = [analyzer.compute_tradeoffs_and_print_from_training() for analyzer in analyzers] +P_threshs = [analyzer.compute_tradeoffs_and_print_from_training() + for analyzer in analyzers] print('Test ROC:') for analyzer in analyzers: print(analyzer.get_roc_area_by_mode('test')) -#P_thresh_opt = 0.566#0.566#0.92# analyzer.compute_tradeoffs_and_print_from_training() -linestyle="-" +# P_thresh_opt = 0.566#0.566#0.92# +# analyzer.compute_tradeoffs_and_print_from_training() +linestyle = "-" -#analyzer.compute_tradeoffs_and_plot('test',save_figure=save_figure,plot_string='_test',linestyle=linestyle) -#analyzer.compute_tradeoffs_and_plot('train',save_figure=save_figure,plot_string='_train',linestyle=linestyle) -#analyzer.summarize_shot_prediction_stats_by_mode(P_thresh_opt,'test') +# analyzer.compute_tradeoffs_and_plot('test', save_figure=save_figure, +# plot_string='_test',linestyle=linestyle) +# analyzer.compute_tradeoffs_and_plot('train', save_figure=save_figure, +# plot_string='_train',linestyle=linestyle) +# analyzer.summarize_shot_prediction_stats_by_mode(P_thresh_opt,'test') shots = analyzers[0].shot_list_test for shot in shots: - if all([(shot in analyzer.shot_list_test or shot in analyzer.shot_list_train) for analyzer in analyzers]): - types = [analyzers[i].get_prediction_type_for_individual_shot(P_threshs[i],shot,mode='test') for i in range(len(analyzers))] - #if len(set(types)) > 1: - if types == ['TP','late']: + if all([(shot in analyzer.shot_list_test + or shot in analyzer.shot_list_train) + for analyzer in analyzers]): + types = [ + analyzers[i].get_prediction_type_for_individual_shot( + P_threshs[i], shot, mode='test') + for i in range(len(analyzers))] + if types == ['TP', 'late']: if shot in analyzers[1].shot_list_test: print("TEST") else: print("TRAIN") print(shot.number) print(types) - for i,analyzer in enumerate(analyzers): - analyzer.save_shot(shot,P_thresh_opt=P_threshs[i],extra_filename=['1D','0D'][i]) + for i, analyzer in enumerate(analyzers): + analyzer.save_shot(shot, P_thresh_opt=P_threshs[i], + extra_filename=['1D', '0D'][i]) else: pass - #print("shot {} not in train or test shot list (must be in validation)".format(shot)) - - + # print("shot {} not in train or test shot list + # (must be in validation)".format(shot)) diff --git a/examples/custom_plot.py b/examples/custom_plot.py index 31d21c76..8f48f725 100644 --- a/examples/custom_plot.py +++ b/examples/custom_plot.py @@ -1,16 +1,37 @@ import numpy as np -from bokeh.plotting import figure, show, output_file, save +from bokeh.plotting import figure, output_file, save # , show from tensorboard.backend.event_processing import event_accumulator -ea1 = event_accumulator.EventAccumulator("/tigress/alexeys/worked_Graphs/Graph16_momSGD_new/events.out.tfevents.1502649990.tiger-i19g10") +file_path = "/tigress/alexeys/worked_Graphs/Graph16_momSGD_new/" +ea1 = event_accumulator.EventAccumulator( + file_path + "events.out.tfevents.1502649990.tiger-i19g10") ea1.Reload() -ea2 = event_accumulator.EventAccumulator("/tigress/alexeys/worked_Graphs/Graph32_momSGD_new/events.out.tfevents.1502652797.tiger-i19g10") +ea2 = event_accumulator.EventAccumulator( + file_path + "events.out.tfevents.1502652797.tiger-i19g10") ea2.Reload() histograms = ea1.Tags()['histograms'] -#ages': [], 'audio': [], 'histograms': ['input_2_out', 'time_distributed_1_out', 'lstm_1/kernel_0', 'lstm_1/kernel_0_grad', 'lstm_1/recurrent_kernel_0', 'lstm_1/recurrent_kernel_0_grad', 'lstm_1/bias_0', 'lstm_1/bias_0_grad', 'lstm_1_out', 'dropout_1_out', 'lstm_2/kernel_0', 'lstm_2/kernel_0_grad', 'lstm_2/recurrent_kernel_0', 'lstm_2/recurrent_kernel_0_grad', 'lstm_2/bias_0', 'lstm_2/bias_0_grad', 'lstm_2_out', 'dropout_2_out', 'time_distributed_2/kernel_0', 'time_distributed_2/kernel_0_grad', 'time_distributed_2/bias_0', 'time_distributed_2/bias_0_grad', 'time_distributed_2_out'], 'scalars': ['val_roc', 'val_loss', 'train_loss'], 'distributions': ['input_2_out', 'time_distributed_1_out', 'lstm_1/kernel_0', 'lstm_1/kernel_0_grad', 'lstm_1/recurrent_kernel_0', 'lstm_1/recurrent_kernel_0_grad', 'lstm_1/bias_0', 'lstm_1/bias_0_grad', 'lstm_1_out', 'dropout_1_out', 'lstm_2/kernel_0', 'lstm_2/kernel_0_grad', 'lstm_2/recurrent_kernel_0', 'lstm_2/recurrent_kernel_0_grad', 'lstm_2/bias_0', 'lstm_2/bias_0_grad', 'lstm_2_out', 'dropout_2_out', 'time_distributed_2/kernel_0', 'time_distributed_2/kernel_0_grad', 'time_distributed_2/bias_0', 'time_distributed_2/bias_0_grad', 'time_distributed_2_out'], 'tensors': [], 'graph': True, 'meta_graph': True, 'run_metadata': []} +# ages': [], 'audio': [], 'histograms': ['input_2_out', +# 'time_distributed_1_out', 'lstm_1/kernel_0', 'lstm_1/kernel_0_grad', +# 'lstm_1/recurrent_kernel_0', 'lstm_1/recurrent_kernel_0_grad', +# 'lstm_1/bias_0', 'lstm_1/bias_0_grad', 'lstm_1_out', 'dropout_1_out', +# 'lstm_2/kernel_0', 'lstm_2/kernel_0_grad', 'lstm_2/recurrent_kernel_0', +# 'lstm_2/recurrent_kernel_0_grad', 'lstm_2/bias_0', 'lstm_2/bias_0_grad', +# 'lstm_2_out', 'dropout_2_out', 'time_distributed_2/kernel_0', +# 'time_distributed_2/kernel_0_grad', 'time_distributed_2/bias_0', +# 'time_distributed_2/bias_0_grad', 'time_distributed_2_out'], 'scalars': +# ['val_roc', 'val_loss', 'train_loss'], 'distributions': ['input_2_out', +# 'time_distributed_1_out', 'lstm_1/kernel_0', 'lstm_1/kernel_0_grad', +# 'lstm_1/recurrent_kernel_0', 'lstm_1/recurrent_kernel_0_grad', +# 'lstm_1/bias_0', 'lstm_1/bias_0_grad', 'lstm_1_out', 'dropout_1_out', +# 'lstm_2/kernel_0', 'lstm_2/kernel_0_grad', 'lstm_2/recurrent_kernel_0', +# 'lstm_2/recurrent_kernel_0_grad', 'lstm_2/bias_0', 'lstm_2/bias_0_grad', +# 'lstm_2_out', 'dropout_2_out', 'time_distributed_2/kernel_0', +# 'time_distributed_2/kernel_0_grad', 'time_distributed_2/bias_0', +# 'time_distributed_2/bias_0_grad', 'time_distributed_2_out'], 'tensors': +# [], 'graph': True, 'meta_graph': True, 'run_metadata': []} for h in histograms: x1 = np.array(ea1.Histograms(h)[0].histogram_value.bucket_limit[:-1]) @@ -18,18 +39,19 @@ x2 = np.array(ea2.Histograms(h)[0].histogram_value.bucket_limit[:-1]) y2 = ea2.Histograms(h)[0].histogram_value.bucket[:-1] - h = h.replace("/","_") + h = h.replace("/", "_") - p = figure(title=h, y_axis_label="Arbitrary units", x_axis_label="Arbitrary units") - # ,y_axis_type="log") + p = figure(title=h, y_axis_label="Arbitrary units", + x_axis_label="Arbitrary units") + # , y_axis_type="log") p.line(x1, y1, legend="float16, SGD with momentum", - line_color="green", line_width=2) + line_color="green", line_width=2) p.line(x2, y2, legend="float32, SGD with momentum", - line_color="indigo", line_width=2) + line_color="indigo", line_width=2) p.legend.location = "top_right" - output_file("plot"+h+".html", title=h) + output_file("plot" + h + ".html", title=h) save(p) # open a browser diff --git a/examples/distributed_job.sh b/examples/distributed_job.sh index 97e3cfd3..421a4225 100644 --- a/examples/distributed_job.sh +++ b/examples/distributed_job.sh @@ -1,6 +1,6 @@ #!/bin/bash #SBATCH -t 0-2:00 -#SBATCH -N 10 #how many nodes. The number of GPUs is this times 4. +#SBATCH -N 10 # how many nodes. The totla number of GPUs is equal to this x4 #SBATCH --ntasks-per-node=16 #SBATCH --ntasks-per-socket=8 #SBATCH --gres=gpu:4 @@ -13,4 +13,3 @@ echo "Removing old model checkpoints." rm /tigress/jk7/data/model_checkpoints/* echo "Running distributed learning" mpirun -npernode 4 python mpi_learn.py - diff --git a/examples/extract_best_overtime.py b/examples/extract_best_overtime.py index f20fed60..867a08ef 100644 --- a/examples/extract_best_overtime.py +++ b/examples/extract_best_overtime.py @@ -1,9 +1,6 @@ +import matplotlib.pylab as plt import pandas as pd import glob -from subprocess import Popen -import yaml -import os -import math import numpy as np from random import shuffle from joblib import Parallel, delayed @@ -11,11 +8,9 @@ import matplotlib matplotlib.use('Agg') -import matplotlib.pylab as plt -import pdb -def arrangeTrialsAtRandom(filenames,scale=1.0): +def arrangeTrialsAtRandom(filenames, scale=1.0): shuffle(filenames) previous = pd.read_csv(filenames[0]) previous['times'] = previous['times'].apply(lambda x: x/60.0/scale) @@ -23,26 +18,29 @@ def arrangeTrialsAtRandom(filenames,scale=1.0): for filename in filenames[1:]: shift = max(previous['times'].values) current = pd.read_csv(filename) - current['times'] = current['times'].apply(lambda x: x/60.0/scale+shift) + current['times'] = current['times'].apply( + lambda x: x/60.0/scale + shift) dataframes.append(current) previous = current return pd.concat(dataframes) -def getOneBestValidationAUC(T_of_test,dataset): - #select subset of dataframe by time for all + +def getOneBestValidationAUC(T_of_test, dataset): + # select subset of dataframe by time for all dataset = dataset[dataset.times <= T_of_test] - - #apply emulate_converge script + + # apply emulate_converge script aucs = dataset['val_roc'].values if len(aucs) > 0: return max(aucs) else: return 0.0 + def doPlot(parallel_aucs, serial_aucs, times, errors): times = list(times) - times_histo = np.histogram(parallel_aucs,bins=times) - #values,edges = times_histo + np.histogram(parallel_aucs, bins=times) + # values,edges = times_histo parallel_values = parallel_aucs[1:] edges = times print(len(parallel_values)) @@ -54,106 +52,120 @@ def doPlot(parallel_aucs, serial_aucs, times, errors): print(edges.shape) print(serial_values.shape) - plt.figure() - plt.plot(edges, parallel_values,label = "Distributed search") #, width=np.diff(edges), ec="k", align="edge") - plt.plot(edges, serial_values, label="Sequential search") #, width=np.diff(edges), ec="k", align="edge") - #plt.fill_between(edges, serial_values-errors,serial_values+errors) - plt.legend(loc = (0.6,0.7)) + # , width=np.diff(edges), ec="k", align="edge") + plt.plot(edges, parallel_values, label="Distributed search") + # , width=np.diff(edges), ec="k", align="edge") + plt.plot(edges, serial_values, label="Sequential search") + # plt.fill_between(edges, serial_values-errors,serial_values+errors) + plt.legend(loc=(0.6, 0.7)) plt.xlabel("Time [minutes]", fontsize=20) - #plt.yscale('log') + # plt.yscale('log') plt.ylabel('Best validation AUC', fontsize=20) plt.savefig("times.png") plt.figure() - plt.plot(edges, parallel_values,label = "Distributed search") #, width=np.diff(edges), ec="k", align="edge") - plt.plot(edges, serial_values, label="Sequential search") #, width=np.diff(edges), ec="k", align="edge") - #plt.fill_between(edges, serial_values-errors,serial_values+errors) - plt.legend(loc = (0.6,0.7)) + # , width=np.diff(edges), ec="k", align="edge") + plt.plot(edges, parallel_values, label="Distributed search") + # , width=np.diff(edges), ec="k", align="edge") + plt.plot(edges, serial_values, label="Sequential search") + # plt.fill_between(edges, serial_values-errors,serial_values+errors) + plt.legend(loc=(0.6, 0.7)) plt.xlabel("Time [minutes]", fontsize=20) plt.xscale('log') - plt.xlim([0,100]) + plt.xlim([0, 100]) plt.ylabel('Best validation AUC', fontsize=20) plt.savefig("times_logx_start.png") plt.figure() - plt.plot(edges, parallel_values,label = "Distributed search") #, width=np.diff(edges), ec="k", align="edge") - plt.plot(edges, serial_values, label="Sequential search") #, width=np.diff(edges), ec="k", align="edge") - #plt.fill_between(edges, serial_values-errors,serial_values+errors) - plt.legend(loc = (0.6,0.7)) + # , width=np.diff(edges), ec="k", align="edge") + plt.plot(edges, parallel_values, label="Distributed search") + # , width=np.diff(edges), ec="k", align="edge") + plt.plot(edges, serial_values, label="Sequential search") + # plt.fill_between(edges, serial_values-errors,serial_values+errors) + plt.legend(loc=(0.6, 0.7)) plt.xlabel("Time [minutes]", fontsize=20) plt.xscale('log') - plt.xlim([100,10000]) + plt.xlim([100, 10000]) plt.ylabel('Best validation AUC', fontsize=20) plt.savefig("times_logx.png") def getReplica(filenames, times): - serial_auc_replica = arrangeTrialsAtRandom(filenames,100.0) + serial_auc_replica = arrangeTrialsAtRandom(filenames, 100.0) best_serial_aucs_over_time = [] for T in times: current_best = 0 - ##pass AUCs and real epoch counts to emulate_converge - auc = getOneBestValidationAUC(T,serial_auc_replica) - if auc > current_best: current_best = auc + # pass AUCs and real epoch counts to emulate_converge + auc = getOneBestValidationAUC(T, serial_auc_replica) + if auc > current_best: + current_best = auc best_serial_aucs_over_time.append(current_best) - #replicas.append(best_serial_aucs_over_time) + # replicas.append(best_serial_aucs_over_time) return best_serial_aucs_over_time -def getTimeReplica(filenames,T): + +def getTimeReplica(filenames, T): current_best = 0 for filename in filenames: - #get AUCs for this trial, one per effective epoch + # get AUCs for this trial, one per effective epoch try: dataset = pd.read_csv(filename) dataset['times'] = dataset['times'].apply(lambda x: x/60.0) - except: + except BaseException: print("No data in {}".format(filename)) continue - ##pass AUCs and real epoch counts to emulate_converge - auc = getOneBestValidationAUC(T,dataset) - if auc > current_best: current_best = auc + # pass AUCs and real epoch counts to emulate_converge + auc = getOneBestValidationAUC(T, dataset) + if auc > current_best: + current_best = auc return current_best -def getTimeReplicaSerial(serial_auc_replica,T): + +def getTimeReplicaSerial(serial_auc_replica, T): current_best = 0 - ##pass AUCs and real epoch counts to emulate_converge - auc = getOneBestValidationAUC(T,serial_auc_replica) - if auc > current_best: current_best = auc + # pass AUCs and real epoch counts to emulate_converge + auc = getOneBestValidationAUC(T, serial_auc_replica) + if auc > current_best: + current_best = auc - #replicas.append(best_serial_aucs_over_time) + # replicas.append(best_serial_aucs_over_time) return current_best if __name__ == '__main__': - filenames = glob.glob("/tigress/FRNN/JET_Titan_hyperparameter_run/*/temporal_csv_log.csv") + filenames = glob.glob( + "/tigress/FRNN/JET_Titan_hyperparameter_run/*/temporal_csv_log.csv") patience = 5 - times = np.linspace(0,310*30,186*30) + times = np.linspace(0, 310*30, 186*30) best_parallel_aucs_over_time = [] num_cores = multiprocessing.cpu_count() - print ("Running on ", num_cores, " CPU cores") - best_parallel_aucs_over_time = Parallel(n_jobs=num_cores)(delayed(getTimeReplica)(filenames, T) for T in times) + print("Running on ", num_cores, " CPU cores") + best_parallel_aucs_over_time = Parallel(n_jobs=num_cores)( + delayed(getTimeReplica)(filenames, T) for T in times) Nreplicas = 20 replicas = [] - for i in range(Nreplicas): - serial_auc_replica = arrangeTrialsAtRandom(filenames,100.0) + serial_auc_replica = arrangeTrialsAtRandom(filenames, 100.0) - #replicas = Parallel(n_jobs=num_cores)(delayed(getReplica)(filenames, times) for i in range(Nreplicas)) - best_serial_aucs_over_time = Parallel(n_jobs=num_cores)(delayed(getTimeReplicaSerial)(serial_auc_replica, T) for T in times) + # replicas = Parallel(n_jobs=num_cores)(delayed(getReplica)(filenames, + # times) for i in range(Nreplicas)) + best_serial_aucs_over_time = Parallel(n_jobs=num_cores)( + delayed(getTimeReplicaSerial)(serial_auc_replica, + T) for T in times) replicas.append(best_serial_aucs_over_time) - - from statistics import mean,stdev + from statistics import mean, stdev best_serial_aucs_over_time = list(map(mean, zip(*replicas))) errors = list(map(stdev, zip(*replicas))) - doPlot(best_parallel_aucs_over_time, best_serial_aucs_over_time, times, errors) + doPlot(best_parallel_aucs_over_time, best_serial_aucs_over_time, times, + errors) diff --git a/examples/guarantee_preprocessed.py b/examples/guarantee_preprocessed.py index 67826ad0..310ed300 100644 --- a/examples/guarantee_preprocessed.py +++ b/examples/guarantee_preprocessed.py @@ -1,18 +1,13 @@ -from __future__ import print_function -import os -import sys -import time -import datetime +from plasma.preprocessor.preprocess import guarantee_preprocessed import random import numpy as np from plasma.conf import conf from pprint import pprint pprint(conf) -from plasma.preprocessor.preprocess import guarantee_preprocessed ##################################################### -####################PREPROCESSING#################### +# PREPROCESSING # ##################################################### np.random.seed(0) random.seed(0) diff --git a/examples/hyper_learn.py b/examples/hyper_learn.py index 4306835b..0dc41a8a 100644 --- a/examples/hyper_learn.py +++ b/examples/hyper_learn.py @@ -1,4 +1,5 @@ -from __future__ import print_function +from plasma.models import runner +from plasma.models.loader import Loader import numpy as np from hyperopt import Trials, tpe @@ -6,37 +7,41 @@ from plasma.conf import conf from pprint import pprint pprint(conf) -#from plasma.primitives.shots import Shot, ShotList -from plasma.preprocessor.normalize import Normalizer -from plasma.models.loader import Loader -#from plasma.models.runner import train, make_predictions,make_predictions_gpu +# from plasma.primitives.shots import Shot, ShotList +# from plasma.models.runner import train, make_predictions,make_predictions_gpu if conf['data']['normalizer'] == 'minmax': from plasma.preprocessor.normalize import MinMaxNormalizer as Normalizer elif conf['data']['normalizer'] == 'meanvar': from plasma.preprocessor.normalize import MeanVarNormalizer as Normalizer elif conf['data']['normalizer'] == 'var': - from plasma.preprocessor.normalize import VarNormalizer as Normalizer #performs !much better than minmaxnormalizer + # performs !much better than minmaxnormalizer + from plasma.preprocessor.normalize import VarNormalizer as Normalizer elif conf['data']['normalizer'] == 'averagevar': - from plasma.preprocessor.normalize import AveragingVarNormalizer as Normalizer #performs !much better than minmaxnormalizer + # performs !much better than minmaxnormalizer + from plasma.preprocessor.normalize import ( + AveragingVarNormalizer as Normalizer + ) else: print('unkown normalizer. exiting') exit(1) np.random.seed(1) -print("normalization",end='') +print("normalization", end='') nn = Normalizer(conf) nn.train() -loader = Loader(conf,nn) -shot_list_train,shot_list_validate,shot_list_test = loader.load_shotlists(conf) +loader = Loader(conf, nn) +shot_list_train, shot_list_validate, shot_list_test = loader.load_shotlists( + conf) print("...done") -print('Training on {} shots, testing on {} shots'.format(len(shot_list_train),len(shot_list_test))) -from plasma.models import runner +print('Training on {} shots, testing on {} shots'.format( + len(shot_list_train), len(shot_list_test))) -specific_runner = runner.HyperRunner(conf,loader,shot_list_train) +specific_runner = runner.HyperRunner(conf, loader, shot_list_train) -best_run, best_model = specific_runner.frnn_minimize(algo=tpe.suggest,max_evals=2,trials=Trials()) -print (best_run) -print (best_model) +best_run, best_model = specific_runner.frnn_minimize( + algo=tpe.suggest, max_evals=2, trials=Trials()) +print(best_run) +print(best_model) diff --git a/examples/individual_shot_performance.py b/examples/individual_shot_performance.py index dec7a6b7..a2ad6989 100644 --- a/examples/individual_shot_performance.py +++ b/examples/individual_shot_performance.py @@ -1,17 +1,15 @@ -import os,sys -import numpy as np - -from plasma.utils.performance import * +import sys +from plasma.utils.performance import PerformanceAnalyzer from plasma.conf import conf -#mode = 'test' +# mode = 'test' file_num = 0 save_figure = True pred_ttd = False -T_min_warn = 30 #None #take value from conf #30 +T_min_warn = 30 # None #take value from conf #30 -verbose=False +verbose = False if len(sys.argv) == 3: results_dir = sys.argv[1] num = int(sys.argv[2]) @@ -23,10 +21,11 @@ print("Plotting shot {}".format(num)) shots_dir = conf['paths']['processed_prepath'] -analyzer = PerformanceAnalyzer(conf=conf,results_dir=results_dir,shots_dir=shots_dir,i = file_num, -T_min_warn = T_min_warn, verbose = verbose, pred_ttd=pred_ttd) +analyzer = PerformanceAnalyzer(conf=conf, results_dir=results_dir, + shots_dir=shots_dir, i=file_num, + T_min_warn=T_min_warn, verbose=verbose, + pred_ttd=pred_ttd) analyzer.load_ith_file() P_thresh_opt = analyzer.compute_tradeoffs_and_print_from_training() -analyzer.plot_individual_shot(P_thresh_opt,num) - +analyzer.plot_individual_shot(P_thresh_opt, num) diff --git a/examples/learn.py b/examples/learn.py index 14eae8fb..947e58fc 100644 --- a/examples/learn.py +++ b/examples/learn.py @@ -1,3 +1,9 @@ +from plasma.models.loader import Loader +from plasma.preprocessor.preprocess import guarantee_preprocessed +from pprint import pprint +from plasma.conf import conf +import multiprocessing as old_mp +import numpy as np ''' ######################################################### This file trains a deep learning model to predict @@ -14,38 +20,35 @@ ######################################################### ''' -from __future__ import print_function -import datetime,time,random -import sys,os -import dill -from functools import partial +import datetime +import random +import sys +import os import matplotlib matplotlib.use('Agg') -import numpy as np -import multiprocessing as old_mp -from plasma.conf import conf -from pprint import pprint pprint(conf) -from plasma.primitives.shots import Shot, ShotList -from plasma.preprocessor.normalize import Normalizer -from plasma.preprocessor.preprocess import Preprocessor, guarantee_preprocessed -from plasma.models.loader import Loader if conf['model']['shallow']: - from plasma.models.shallow_runner import train, make_predictions_and_evaluate_gpu + from plasma.models.shallow_runner import ( + train, make_predictions_and_evaluate_gpu + ) else: from plasma.models.runner import train, make_predictions_and_evaluate_gpu if conf['data']['normalizer'] == 'minmax': from plasma.preprocessor.normalize import MinMaxNormalizer as Normalizer elif conf['data']['normalizer'] == 'meanvar': - from plasma.preprocessor.normalize import MeanVarNormalizer as Normalizer + from plasma.preprocessor.normalize import MeanVarNormalizer as Normalizer elif conf['data']['normalizer'] == 'var': - from plasma.preprocessor.normalize import VarNormalizer as Normalizer #performs !much better than minmaxnormalizer + # performs !much better than minmaxnormalizer + from plasma.preprocessor.normalize import VarNormalizer as Normalizer elif conf['data']['normalizer'] == 'averagevar': - from plasma.preprocessor.normalize import AveragingVarNormalizer as Normalizer #performs !much better than minmaxnormalizer + # performs !much better than minmaxnormalizer + from plasma.preprocessor.normalize import ( + AveragingVarNormalizer as Normalizer + ) else: print('unkown normalizer. exiting') exit(1) @@ -55,7 +58,7 @@ shot_files_test = conf['paths']['shot_files_test'] train_frac = conf['training']['train_frac'] stateful = conf['model']['stateful'] -# if stateful: +# if stateful: # batch_size = conf['model']['length'] # else: # batch_size = conf['training']['batch_size_large'] @@ -70,37 +73,43 @@ print("predicting using path {}".format(custom_path)) ##################################################### -####################PREPROCESSING#################### +# PREPROCESSING # ##################################################### -shot_list_train,shot_list_validate,shot_list_test = guarantee_preprocessed(conf) +# TODO(KGF): check tuple unpack +(shot_list_train, shot_list_validate, + shot_list_test) = guarantee_preprocessed(conf) ##################################################### -####################Normalization#################### +# NORMALIZATION # ##################################################### -print("normalization",end='') +print("normalization", end='') nn = Normalizer(conf) nn.train() -loader = Loader(conf,nn) +loader = Loader(conf, nn) print("...done") -print('Training on {} shots, testing on {} shots'.format(len(shot_list_train),len(shot_list_test))) +print('Training on {} shots, testing on {} shots'.format( + len(shot_list_train), len(shot_list_test))) ##################################################### -######################TRAINING####################### +# TRAINING # ##################################################### -#train(conf,shot_list_train,loader) +# train(conf,shot_list_train,loader) if not only_predict: - p = old_mp.Process(target = train,args=(conf,shot_list_train,shot_list_validate,loader)) + p = old_mp.Process(target=train, + args=(conf, shot_list_train, + shot_list_validate, loader) + ) p.start() p.join() ##################################################### -####################PREDICTING####################### +# PREDICTING # ##################################################### loader.set_inference_mode(True) -#load last model for testing +# load last model for testing print('saving results') y_prime = [] y_prime_test = [] @@ -110,15 +119,22 @@ y_gold_test = [] y_gold_train = [] -disruptive= [] -disruptive_train= [] -disruptive_test= [] - -# y_prime_train,y_gold_train,disruptive_train = make_predictions(conf,shot_list_train,loader) -# y_prime_test,y_gold_test,disruptive_test = make_predictions(conf,shot_list_test,loader) - -y_prime_train,y_gold_train,disruptive_train,roc_train,loss_train = make_predictions_and_evaluate_gpu(conf,shot_list_train,loader,custom_path) -y_prime_test,y_gold_test,disruptive_test,roc_test,loss_test = make_predictions_and_evaluate_gpu(conf,shot_list_test,loader,custom_path) +disruptive = [] +disruptive_train = [] +disruptive_test = [] + +# y_prime_train, y_gold_train, disruptive_train = +# make_predictions(conf, shot_list_train, loader) +# y_prime_test, y_gold_test, disruptive_test = +# make_predictions(conf, shot_list_test, loader) + +# TODO(KGF): check tuple unpack +(y_prime_train, y_gold_train, disruptive_train, roc_train, + loss_train) = make_predictions_and_evaluate_gpu( + conf, shot_list_train, loader, custom_path) +(y_prime_test, y_gold_test, disruptive_test, roc_test, + loss_test) = make_predictions_and_evaluate_gpu( + conf, shot_list_test, loader, custom_path) print('=========Summary========') print('Train Loss: {:.3e}'.format(loss_train)) print('Train ROC: {:.4f}'.format(roc_train)) @@ -126,13 +142,12 @@ print('Test ROC: {:.4f}'.format(roc_test)) - disruptive_train = np.array(disruptive_train) disruptive_test = np.array(disruptive_test) y_gold = y_gold_train + y_gold_test y_prime = y_prime_train + y_prime_test -disruptive = np.concatenate((disruptive_train,disruptive_test)) +disruptive = np.concatenate((disruptive_train, disruptive_test)) shot_list_validate.make_light() shot_list_test.make_light() @@ -142,11 +157,12 @@ result_base_path = conf['paths']['results_prepath'] if not os.path.exists(result_base_path): os.makedirs(result_base_path) -np.savez(result_base_path+save_str, - y_gold=y_gold,y_gold_train=y_gold_train,y_gold_test=y_gold_test, - y_prime=y_prime,y_prime_train=y_prime_train,y_prime_test=y_prime_test, - disruptive=disruptive,disruptive_train=disruptive_train,disruptive_test=disruptive_test, - shot_list_validate=shot_list_validate,shot_list_train=shot_list_train,shot_list_test=shot_list_test, - conf = conf) +np.savez(result_base_path+save_str, y_gold=y_gold, y_gold_train=y_gold_train, + y_gold_test=y_gold_test, y_prime=y_prime, y_prime_train=y_prime_train, + y_prime_test=y_prime_test, disruptive=disruptive, + disruptive_train=disruptive_train, disruptive_test=disruptive_test, + shot_list_validate=shot_list_validate, + shot_list_train=shot_list_train, shot_list_test=shot_list_test, + conf=conf) print('finished.') diff --git a/examples/mpi_augment_learn.py b/examples/mpi_augment_learn.py index 35df1aaa..727d559b 100644 --- a/examples/mpi_augment_learn.py +++ b/examples/mpi_augment_learn.py @@ -1,3 +1,12 @@ +from plasma.models.mpi_runner import ( + mpi_train, mpi_make_predictions_and_evaluate + ) +from mpi4py import MPI +from plasma.preprocessor.preprocess import guarantee_preprocessed +from plasma.preprocessor.augment import Augmentator +from plasma.models.loader import Loader +from plasma.conf import conf +from pprint import pprint ''' ######################################################### This file trains a deep learning model to predict @@ -16,10 +25,8 @@ ######################################################### ''' -from __future__ import print_function import os -import sys -import time +import sys import datetime import random import numpy as np @@ -27,38 +34,35 @@ import matplotlib matplotlib.use('Agg') -from pprint import pprint sys.setrecursionlimit(10000) -from plasma.conf import conf -from plasma.models.loader import Loader -from plasma.preprocessor.normalize import Normalizer -from plasma.preprocessor.augment import Augmentator -from plasma.preprocessor.preprocess import guarantee_preprocessed if conf['model']['shallow']: - print("Shallow learning using MPI is not supported yet. set conf['model']['shallow'] to false.") + print( + "Shallow learning using MPI is not supported yet. ", + "Set conf['model']['shallow'] to False.") exit(1) if conf['data']['normalizer'] == 'minmax': from plasma.preprocessor.normalize import MinMaxNormalizer as Normalizer elif conf['data']['normalizer'] == 'meanvar': from plasma.preprocessor.normalize import MeanVarNormalizer as Normalizer elif conf['data']['normalizer'] == 'var': - from plasma.preprocessor.normalize import VarNormalizer as Normalizer #performs !much better than minmaxnormalizer + # performs !much better than minmaxnormalizer + from plasma.preprocessor.normalize import VarNormalizer as Normalizer elif conf['data']['normalizer'] == 'averagevar': - from plasma.preprocessor.normalize import AveragingVarNormalizer as Normalizer #performs !much better than minmaxnormalizer + # performs !much better than minmaxnormalizer + from plasma.preprocessor.normalize import ( + AveragingVarNormalizer as Normalizer) else: print('unkown normalizer. exiting') exit(1) -from mpi4py import MPI comm = MPI.COMM_WORLD task_index = comm.Get_rank() num_workers = comm.Get_size() NUM_GPUS = 4 MY_GPU = task_index % NUM_GPUS -from plasma.models.mpi_runner import * np.random.seed(task_index) random.seed(task_index) @@ -72,47 +76,54 @@ print("predicting using path {}".format(custom_path)) ##################################################### -####################Normalization#################### +# NORMALIZATION # ##################################################### -if task_index == 0: #make sure preprocessing has been run, and is saved as a file - shot_list_train,shot_list_validate,shot_list_test = guarantee_preprocessed(conf) +# TODO(KGF): identical in at least 3x files in examples/ +# make sure preprocessing has been run, and is saved as a file +if task_index == 0: + # TODO(KGF): check tuple unpack + (shot_list_train, shot_list_validate, + shot_list_test) = guarantee_preprocessed(conf) comm.Barrier() -shot_list_train,shot_list_validate,shot_list_test = guarantee_preprocessed(conf) - +(shot_list_train, shot_list_validate, + shot_list_test) = guarantee_preprocessed(conf) -print("normalization",end='') +print("normalization", end='') raw_normalizer = Normalizer(conf) raw_normalizer.train() -is_inference= False -normalizer = Augmentator(raw_normalizer,is_inference,conf) -loader = Loader(conf,normalizer) +is_inference = False +normalizer = Augmentator(raw_normalizer, is_inference, conf) +loader = Loader(conf, normalizer) print("...done") if not only_predict: - mpi_train(conf,shot_list_train,shot_list_validate,loader) + mpi_train(conf, shot_list_train, shot_list_validate, loader) -#load last model for testing +# load last model for testing print('saving results') y_prime = [] y_gold = [] -disruptive= [] - -# y_prime_train,y_gold_train,disruptive_train = make_predictions(conf,shot_list_train,loader) -# y_prime_test,y_gold_test,disruptive_test = make_predictions(conf,shot_list_test,loader) +disruptive = [] normalizer.set_inference(True) -y_prime_train,y_gold_train,disruptive_train,roc_train,loss_train = mpi_make_predictions_and_evaluate(conf,shot_list_train,loader,custom_path) -y_prime_test,y_gold_test,disruptive_test,roc_test,loss_test = mpi_make_predictions_and_evaluate(conf,shot_list_test,loader,custom_path) +# TODO(KGF): check tuple unpack +(y_prime_train, y_gold_train, disruptive_train, roc_train, + loss_train) = mpi_make_predictions_and_evaluate(conf, shot_list_train, + loader, custom_path) +(y_prime_test, y_gold_test, disruptive_test, roc_test, + loss_test) = mpi_make_predictions_and_evaluate(conf, shot_list_test, + loader, custom_path) if task_index == 0: - print('=========Summary========') - print('Train Loss: {:.3e}'.format(loss_train)) - print('Train ROC: {:.4f}'.format(roc_train)) - print('Test Loss: {:.3e}'.format(loss_test)) - print('Test ROC: {:.4f}'.format(roc_test)) - if roc_test < 0.8: sys.exit(1) + print('=========Summary========') + print('Train Loss: {:.3e}'.format(loss_train)) + print('Train ROC: {:.4f}'.format(roc_train)) + print('Test Loss: {:.3e}'.format(loss_test)) + print('Test ROC: {:.4f}'.format(roc_test)) + if roc_test < 0.8: + sys.exit(1) if task_index == 0: @@ -121,22 +132,25 @@ y_gold = y_gold_train + y_gold_test y_prime = y_prime_train + y_prime_test - disruptive = np.concatenate((disruptive_train,disruptive_test)) + disruptive = np.concatenate((disruptive_train, disruptive_test)) shot_list_test.make_light() shot_list_train.make_light() - save_str = 'results_' + datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + save_str = 'results_' + datetime.datetime.now().strftime( + "%Y-%m-%d-%H-%M-%S") result_base_path = conf['paths']['results_prepath'] if not os.path.exists(result_base_path): os.makedirs(result_base_path) - np.savez(result_base_path+save_str, - y_gold=y_gold,y_gold_train=y_gold_train,y_gold_test=y_gold_test, - y_prime=y_prime,y_prime_train=y_prime_train,y_prime_test=y_prime_test, - disruptive=disruptive,disruptive_train=disruptive_train,disruptive_test=disruptive_test, - shot_list_train=shot_list_train,shot_list_test=shot_list_test, - conf = conf) + np.savez(result_base_path+save_str, y_gold=y_gold, + y_gold_train=y_gold_train, y_gold_test=y_gold_test, + y_prime=y_prime, y_prime_train=y_prime_train, + y_prime_test=y_prime_test, disruptive=disruptive, + disruptive_train=disruptive_train, + disruptive_test=disruptive_test, + shot_list_train=shot_list_train, shot_list_test=shot_list_test, + conf=conf) sys.stdout.flush() if task_index == 0: diff --git a/examples/mpi_learn.py b/examples/mpi_learn.py index 7bc4ce8b..16321888 100644 --- a/examples/mpi_learn.py +++ b/examples/mpi_learn.py @@ -1,3 +1,11 @@ +from plasma.models.mpi_runner import ( + mpi_train, mpi_make_predictions_and_evaluate + ) +from mpi4py import MPI +from plasma.preprocessor.preprocess import guarantee_preprocessed +from plasma.models.loader import Loader +from plasma.conf import conf +from pprint import pprint ''' ######################################################### This file trains a deep learning model to predict @@ -16,10 +24,8 @@ ######################################################### ''' -from __future__ import print_function import os -import sys -import time +import sys import datetime import random import numpy as np @@ -27,37 +33,36 @@ import matplotlib matplotlib.use('Agg') -from pprint import pprint sys.setrecursionlimit(10000) -from plasma.conf import conf -from plasma.models.loader import Loader -from plasma.preprocessor.normalize import Normalizer -from plasma.preprocessor.preprocess import guarantee_preprocessed if conf['model']['shallow']: - print("Shallow learning using MPI is not supported yet. set conf['model']['shallow'] to false.") + print( + "Shallow learning using MPI is not supported yet. ", + "Set conf['model']['shallow'] to False.") exit(1) if conf['data']['normalizer'] == 'minmax': from plasma.preprocessor.normalize import MinMaxNormalizer as Normalizer elif conf['data']['normalizer'] == 'meanvar': from plasma.preprocessor.normalize import MeanVarNormalizer as Normalizer elif conf['data']['normalizer'] == 'var': - from plasma.preprocessor.normalize import VarNormalizer as Normalizer #performs !much better than minmaxnormalizer + # performs !much better than minmaxnormalizer + from plasma.preprocessor.normalize import VarNormalizer as Normalizer elif conf['data']['normalizer'] == 'averagevar': - from plasma.preprocessor.normalize import AveragingVarNormalizer as Normalizer #performs !much better than minmaxnormalizer + # performs !much better than minmaxnormalizer + from plasma.preprocessor.normalize import ( + AveragingVarNormalizer as Normalizer + ) else: print('unkown normalizer. exiting') exit(1) -from mpi4py import MPI comm = MPI.COMM_WORLD task_index = comm.Get_rank() num_workers = comm.Get_size() NUM_GPUS = conf['num_gpus'] MY_GPU = task_index % NUM_GPUS -from plasma.models.mpi_runner import * np.random.seed(task_index) random.seed(task_index) @@ -70,47 +75,53 @@ custom_path = sys.argv[1] print("predicting using path {}".format(custom_path)) + ##################################################### -####################Normalization#################### +# NORMALIZATION # ##################################################### -if task_index == 0: #make sure preprocessing has been run, and is saved as a file - shot_list_train,shot_list_validate,shot_list_test = guarantee_preprocessed(conf) +# make sure preprocessing has been run, and is saved as a file +if task_index == 0: + # TODO(KGF): check tuple unpack + (shot_list_train, shot_list_validate, + shot_list_test) = guarantee_preprocessed(conf) comm.Barrier() -shot_list_train,shot_list_validate,shot_list_test = guarantee_preprocessed(conf) - +(shot_list_train, shot_list_validate, + shot_list_test) = guarantee_preprocessed(conf) -print("normalization",end='') +print("normalization", end='') normalizer = Normalizer(conf) normalizer.train() -loader = Loader(conf,normalizer) +loader = Loader(conf, normalizer) print("...done") -#ensure training has a separate random seed for every worker +# ensure training has a separate random seed for every worker np.random.seed(task_index) random.seed(task_index) if not only_predict: - mpi_train(conf,shot_list_train,shot_list_validate,loader) + mpi_train(conf, shot_list_train, shot_list_validate, loader) -#load last model for testing +# load last model for testing loader.set_inference_mode(True) print('saving results') y_prime = [] y_gold = [] -disruptive= [] - -# y_prime_train,y_gold_train,disruptive_train = make_predictions(conf,shot_list_train,loader) -# y_prime_test,y_gold_test,disruptive_test = make_predictions(conf,shot_list_test,loader) +disruptive = [] -y_prime_train,y_gold_train,disruptive_train,roc_train,loss_train = mpi_make_predictions_and_evaluate(conf,shot_list_train,loader,custom_path) -y_prime_test,y_gold_test,disruptive_test,roc_test,loss_test = mpi_make_predictions_and_evaluate(conf,shot_list_test,loader,custom_path) +# TODO(KGF): check tuple unpack +(y_prime_train, y_gold_train, disruptive_train, roc_train, + loss_train) = mpi_make_predictions_and_evaluate(conf, shot_list_train, + loader, custom_path) +(y_prime_test, y_gold_test, disruptive_test, roc_test, + loss_test) = mpi_make_predictions_and_evaluate(conf, shot_list_test, + loader, custom_path) if task_index == 0: - print('=========Summary========') - print('Train Loss: {:.3e}'.format(loss_train)) - print('Train ROC: {:.4f}'.format(roc_train)) - print('Test Loss: {:.3e}'.format(loss_test)) - print('Test ROC: {:.4f}'.format(roc_test)) + print('=========Summary========') + print('Train Loss: {:.3e}'.format(loss_train)) + print('Train ROC: {:.4f}'.format(roc_train)) + print('Test Loss: {:.3e}'.format(loss_test)) + print('Test ROC: {:.4f}'.format(roc_test)) if task_index == 0: @@ -119,22 +130,24 @@ y_gold = y_gold_train + y_gold_test y_prime = y_prime_train + y_prime_test - disruptive = np.concatenate((disruptive_train,disruptive_test)) + disruptive = np.concatenate((disruptive_train, disruptive_test)) shot_list_test.make_light() shot_list_train.make_light() - save_str = 'results_' + datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + save_str = 'results_' + datetime.datetime.now().strftime( + "%Y-%m-%d-%H-%M-%S") result_base_path = conf['paths']['results_prepath'] if not os.path.exists(result_base_path): os.makedirs(result_base_path) - np.savez(result_base_path+save_str, - y_gold=y_gold,y_gold_train=y_gold_train,y_gold_test=y_gold_test, - y_prime=y_prime,y_prime_train=y_prime_train,y_prime_test=y_prime_test, - disruptive=disruptive,disruptive_train=disruptive_train,disruptive_test=disruptive_test, - shot_list_train=shot_list_train,shot_list_test=shot_list_test, - conf = conf) + np.savez(result_base_path+save_str, y_gold=y_gold, + y_gold_train=y_gold_train, y_gold_test=y_gold_test, + y_prime=y_prime, y_prime_train=y_prime_train, + y_prime_test=y_prime_test, disruptive=disruptive, + disruptive_train=disruptive_train, + disruptive_test=disruptive_test, shot_list_train=shot_list_train, + shot_list_test=shot_list_test, conf=conf) sys.stdout.flush() if task_index == 0: diff --git a/examples/performance_analysis.py b/examples/performance_analysis.py index 31568581..499abd45 100644 --- a/examples/performance_analysis.py +++ b/examples/performance_analysis.py @@ -1,66 +1,93 @@ -import os,sys +import sys import numpy as np -from plasma.utils.performance import * +from plasma.utils.performance import PerformanceAnalyzer from plasma.conf import conf -#mode = 'test' +# mode = 'test' file_num = 0 save_figure = True pred_ttd = False -# cut_shot_ends = conf['data']['cut_shot_ends'] -# dt = conf['data']['dt'] -# T_max_warn = int(round(conf['data']['T_warning']/dt)) -# T_min_warn = conf['data']['T_min_warn']#int(round(conf['data']['T_min_warn']/dt)) -# if cut_shot_ends: -# T_max_warn = T_max_warn-T_min_warn -# T_min_warn = 0 -T_min_warn = 30 #None #take value from conf #30 +T_min_warn = 30 # None #take value from conf #30 -verbose=False +verbose = False if len(sys.argv) > 1: results_dir = sys.argv[1] else: results_dir = conf['paths']['results_prepath'] shots_dir = conf['paths']['processed_prepath'] -analyzer = PerformanceAnalyzer(conf=conf,results_dir=results_dir,shots_dir=shots_dir,i = file_num, -T_min_warn = T_min_warn, verbose = verbose, pred_ttd=pred_ttd) +analyzer = PerformanceAnalyzer( + conf=conf, + results_dir=results_dir, + shots_dir=shots_dir, + i=file_num, + T_min_warn=T_min_warn, + verbose=verbose, + pred_ttd=pred_ttd) analyzer.load_ith_file() P_thresh_opt = analyzer.compute_tradeoffs_and_print_from_training() -#P_thresh_opt = 0.566#0.566#0.92# analyzer.compute_tradeoffs_and_print_from_training() -linestyle="-" +# P_thresh_opt = 0.566 # 0.566 # 0.92 +# analyzer.compute_tradeoffs_and_print_from_training() +linestyle = "-" -P_thresh_range,missed_range,fp_range = analyzer.compute_tradeoffs_and_plot('test',save_figure=save_figure,plot_string='_test',linestyle=linestyle) -np.savez('test_roc.npz',"P_thresh_range",P_thresh_range,"missed_range",missed_range,"fp_range",fp_range) -analyzer.compute_tradeoffs_and_plot('train',save_figure=save_figure,plot_string='_train',linestyle=linestyle) +P_thresh_range, missed_range, fp_range = analyzer.compute_tradeoffs_and_plot( + 'test', save_figure=save_figure, plot_string='_test', linestyle=linestyle) +np.savez( + 'test_roc.npz', + "P_thresh_range", + P_thresh_range, + "missed_range", + missed_range, + "fp_range", + fp_range) +analyzer.compute_tradeoffs_and_plot( + 'train', + save_figure=save_figure, + plot_string='_train', + linestyle=linestyle) -analyzer.summarize_shot_prediction_stats_by_mode(P_thresh_opt,'test') +analyzer.summarize_shot_prediction_stats_by_mode(P_thresh_opt, 'test') normalize = True -analyzer.example_plots(P_thresh_opt,'test','any',normalize=normalize) -analyzer.example_plots(P_thresh_opt,'test',['FP'],extra_filename='test',normalize=normalize) -analyzer.example_plots(P_thresh_opt,'test',['FN'],extra_filename='test',normalize=normalize) -analyzer.example_plots(P_thresh_opt,'test',['TP'],extra_filename='test',normalize=normalize) -analyzer.example_plots(P_thresh_opt,'test',['late'],extra_filename='test',normalize=normalize) +analyzer.example_plots(P_thresh_opt, 'test', 'any', normalize=normalize) +analyzer.example_plots(P_thresh_opt, 'test', ['FP'], extra_filename='test', + normalize=normalize) +analyzer.example_plots(P_thresh_opt, 'test', ['FN'], extra_filename='test', + normalize=normalize) +analyzer.example_plots(P_thresh_opt, 'test', ['TP'], extra_filename='test', + normalize=normalize) +analyzer.example_plots(P_thresh_opt, 'test', ['late'], extra_filename='test', + normalize=normalize) -analyzer.example_plots(P_thresh_opt,'train',['TN'],extra_filename='train',normalize=normalize) -analyzer.example_plots(P_thresh_opt,'train',['FP'],extra_filename='train',normalize=normalize) -analyzer.example_plots(P_thresh_opt,'train',['FN'],extra_filename='train',normalize=normalize) -analyzer.example_plots(P_thresh_opt,'train',['TP'],extra_filename='train',normalize=normalize) -analyzer.example_plots(P_thresh_opt,'train',['late'],extra_filename='train',normalize=normalize) +analyzer.example_plots(P_thresh_opt, 'train', ['TN'], extra_filename='train', + normalize=normalize) +analyzer.example_plots(P_thresh_opt, 'train', ['FP'], extra_filename='train', + normalize=normalize) +analyzer.example_plots(P_thresh_opt, 'train', ['FN'], extra_filename='train', + normalize=normalize) +analyzer.example_plots(P_thresh_opt, 'train', ['TP'], extra_filename='train', + normalize=normalize) +analyzer.example_plots(P_thresh_opt, 'train', ['late'], extra_filename='train', + normalize=normalize) -alarms,disr_alarms,nondisr_alarms = analyzer.gather_first_alarms(P_thresh_opt,'test') -analyzer.hist_alarms(disr_alarms,'disruptive alarms, P thresh = {}'.format(P_thresh_opt),save_figure=save_figure,linestyle=linestyle) -np.savez('disruptive_alarms_test.npz',"disr_alarms",disr_alarms,"P_thresh_opt",P_thresh_opt) +alarms, disr_alarms, nondisr_alarms = analyzer.gather_first_alarms( + P_thresh_opt, 'test') +analyzer.hist_alarms(disr_alarms, 'disruptive alarms, P thresh = {}'.format( + P_thresh_opt), save_figure=save_figure, linestyle=linestyle) +np.savez('disruptive_alarms_test.npz', "disr_alarms", disr_alarms, + "P_thresh_opt", P_thresh_opt) print('{} disruptive alarms'.format(len(disr_alarms))) -print('{} seconds mean alarm time'.format(np.mean(disr_alarms[disr_alarms > 0]))) -print('{} seconds median alarm time'.format(np.median(disr_alarms[disr_alarms > 0]))) -analyzer.hist_alarms(nondisr_alarms,'nondisruptive alarms, P thresh = {}'.format(P_thresh_opt)) +print('{} seconds mean alarm time'.format( + np.mean(disr_alarms[disr_alarms > 0]))) +print('{} seconds median alarm time'.format( + np.median(disr_alarms[disr_alarms > 0]))) +analyzer.hist_alarms(nondisr_alarms, + 'nondisruptive alarms, P thresh = {}'.format(P_thresh_opt) + ) print('{} nondisruptive alarms'.format(len(nondisr_alarms))) - diff --git a/examples/prepare_pbs_configs_titan.py b/examples/prepare_pbs_configs_titan.py index f036c9d8..bd92b2be 100644 --- a/examples/prepare_pbs_configs_titan.py +++ b/examples/prepare_pbs_configs_titan.py @@ -3,76 +3,85 @@ from subprocess import Popen from time import sleep -def checkAndSchedule(configBaseName,nextGPUcount,GPUstep,maxGPUcount): - if nextGPUcount > maxGPUcount: return - job_is_running = subprocess.check_output(['qstat','-u','alexeys']) - if len(job_is_running) > 0: - #sleep 500 seconds + +def checkAndSchedule(configBaseName, nextGPUcount, GPUstep, maxGPUcount): + if nextGPUcount > maxGPUcount: + return + job_is_running = subprocess.check_output(['qstat', '-u', 'alexeys']) + if len(job_is_running) > 0: + # sleep 500 seconds sleep(500) - checkAndSchedule(configBaseName,nextGPUcount,GPUstep,maxGPUcount) + checkAndSchedule(configBaseName, nextGPUcount, GPUstep, maxGPUcount) else: - #create a config - nextConfigName = createOneConfig(configBaseName,nextGPUcount) - print "Submitting next PBS job {} to run on {} GPUs".format(configBaseName,nextGPUcount) - print "qsub "+nextConfigName - Popen("qsub "+nextConfigName,shell=True).wait() - #update parameters + # create a config + nextConfigName = createOneConfig(configBaseName, nextGPUcount) + print("Submitting next PBS job {} to run on {} GPUs".format( + configBaseName, nextGPUcount)) + print("qsub ", nextConfigName) + Popen("qsub " + nextConfigName, shell=True).wait() + # update parameters nextGPUcount += GPUstep sleep(10) - checkAndSchedule(configBaseName,nextGPUcount,GPUstep,maxGPUcount) + checkAndSchedule(configBaseName, nextGPUcount, GPUstep, maxGPUcount) def createOneConfig(configBaseName, GPUcount): configFullName = configBaseName+str(GPUcount)+".cmd" - with open(configFullName,"w") as f: - f.write('#!/bin/bash\n') - f.write('#PBS -A FUS117\n') - f.write('#PBS -l walltime=1:30:00\n') #FIXME this depends a lot on the number of GPUs 1900s/1epoch at 50, 2350s/1epoch at 4 - f.write('#PBS -l nodes='+str(GPUcount)+'\n') - f.write('##PBS -l procs=1\n') - f.write('##PBS -l gres=atlas1%atlas2\n') + with open(configFullName, "w") as f: + f.write('#!/bin/bash\n') + f.write('#PBS -A FUS117\n') + # FIXME this depends a lot on the number of GPUs 1900s/1epoch at 50, + # 2350s/1epoch at 4 + f.write('#PBS -l walltime=1:30:00\n') + f.write('#PBS -l nodes='+str(GPUcount)+'\n') + f.write('##PBS -l procs=1\n') + f.write('##PBS -l gres=atlas1%atlas2\n') f.write('\n\n') - f.write('export HOME=/lustre/atlas/proj-shared/fus117\n') - f.write('cd $HOME/PPPL/plasma-python/examples\n') + f.write('export HOME=/lustre/atlas/proj-shared/fus117\n') + f.write('cd $HOME/PPPL/plasma-python/examples\n') f.write('\n\n') - f.write('source $MODULESHOME/init/bash\n') - f.write('module switch PrgEnv-pgi PrgEnv-gnu\n') + f.write('source $MODULESHOME/init/bash\n') + f.write('module switch PrgEnv-pgi PrgEnv-gnu\n') f.write('\n\n') - f.write('module load cudatoolkit\n') - f.write('export LIBRARY_PATH=/opt/nvidia/cudatoolkit7.5/7.5.18-1.0502.10743.2.1/lib64:$LIBRARY_PATH\n') + f.write('module load cudatoolkit\n') + f.write(('export LIBRARY_PATH=/opt/nvidia/cudatoolkit7.5/' + '7.5.18-1.0502.10743.2.1/lib64:$LIBRARY_PATH\n')) f.write('\n\n') - f.write('#This block is CuDNN module\n') - f.write('export LD_LIBRARY_PATH=$HOME/cuda/lib64:$LD_LIBRARY_PATH\n') - f.write('export LIBRARY_PATH=$HOME/cuda/lib64:$LIBRARY_PATH\n') - f.write('export LDFLAGS=$LDFLAGS:$HOME/cuda/lib64\n') - f.write('export INCLUDE=$INCLUDE:$HOME/cuda/include\n') - f.write('export CPATH=$CPATH:$HOME/cuda/include\n') - f.write('export FFLAGS=$FFLAGS:$HOME/cuda/include\n') - f.write('export LOCAL_LDFLAGS=$LOCAL_LDFLAGS:$HOME/cuda/lib64\n') - f.write('export LOCAL_INCLUDE=$LOCAL_INCLUDE:$HOME/cuda/include\n') - f.write('export LOCAL_CFLAGS=$LOCAL_CFLAGS:$HOME/cuda/include\n') - f.write('export LOCAL_FFLAGS=$LOCAL_FFLAGS:$HOME/cuda/include\n') - f.write('export LOCAL_CXXFLAGS=$LOCAL_CXXFLAGS:$HOME/cuda/include\n') + f.write('#This block is CuDNN module\n') + f.write('export LD_LIBRARY_PATH=$HOME/cuda/lib64:$LD_LIBRARY_PATH\n') + f.write('export LIBRARY_PATH=$HOME/cuda/lib64:$LIBRARY_PATH\n') + f.write('export LDFLAGS=$LDFLAGS:$HOME/cuda/lib64\n') + f.write('export INCLUDE=$INCLUDE:$HOME/cuda/include\n') + f.write('export CPATH=$CPATH:$HOME/cuda/include\n') + f.write('export FFLAGS=$FFLAGS:$HOME/cuda/include\n') + f.write('export LOCAL_LDFLAGS=$LOCAL_LDFLAGS:$HOME/cuda/lib64\n') + f.write('export LOCAL_INCLUDE=$LOCAL_INCLUDE:$HOME/cuda/include\n') + f.write('export LOCAL_CFLAGS=$LOCAL_CFLAGS:$HOME/cuda/include\n') + f.write('export LOCAL_FFLAGS=$LOCAL_FFLAGS:$HOME/cuda/include\n') + f.write('export LOCAL_CXXFLAGS=$LOCAL_CXXFLAGS:$HOME/cuda/include\n') f.write('\n\n') - f.write('#This sets new home and Anaconda module\n') - f.write('export PATH=$HOME/anaconda2/bin:$PATH\n') - f.write('export LD_LIBRARY_PATH=$HOME/anaconda2/lib:$LD_LIBRARY_PATH\n') - f.write('source activate PPPL\n') + f.write('#This sets new home and Anaconda module\n') + f.write('export PATH=$HOME/anaconda2/bin:$PATH\n') + f.write( + 'export LD_LIBRARY_PATH=$HOME/anaconda2/lib:$LD_LIBRARY_PATH\n') + f.write('source activate PPPL\n') f.write('\n\n') - f.write('PYTHON=`which python`\n') - f.write('echo $PYTHON\n') + f.write('PYTHON=`which python`\n') + f.write('echo $PYTHON\n') f.write('\n\n') - f.write('export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH\n') - f.write('export MPICH_RDMA_ENABLED_CUDA=1\n') + f.write( + 'export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH\n') + f.write('export MPICH_RDMA_ENABLED_CUDA=1\n') f.write('\n\n') - f.write('rm $HOME/tigress/alexeys/model_checkpoints/*\n') - f.write('aprun -n'+str(GPUcount)+' -N1 $PYTHON mpi_learn.py\n') + f.write('rm $HOME/tigress/alexeys/model_checkpoints/*\n') + f.write('aprun -n'+str(GPUcount)+' -N1 $PYTHON mpi_learn.py\n') return configFullName -if __name__=='__main__': + +if __name__ == '__main__': nextGPUcount = 50 - GPUstep = 50 + GPUstep = 50 maxGPUcount = 101 configBaseName = "FRNN_Titan" - checkAndSchedule(configBaseName,nextGPUcount,GPUstep,maxGPUcount) + checkAndSchedule(configBaseName, nextGPUcount, GPUstep, maxGPUcount) diff --git a/examples/prepare_slurm_configs.py b/examples/prepare_slurm_configs.py index 32223b53..518eccf8 100644 --- a/examples/prepare_slurm_configs.py +++ b/examples/prepare_slurm_configs.py @@ -3,29 +3,34 @@ from subprocess import Popen from time import sleep -def checkAndSchedule(configBaseName,gpuNodeCountGrid,nextGPUNodeCount): - if nextGPUNodeCount > len(gpuNodeCountGrid)-1: return - job_is_running = subprocess.check_output(['squeue','-u','alexeys']) #['qstat','-u','alexeys']) + +def checkAndSchedule(configBaseName, gpuNodeCountGrid, nextGPUNodeCount): + if nextGPUNodeCount > len(gpuNodeCountGrid)-1: + return + job_is_running = subprocess.check_output( + ['squeue', '-u', 'alexeys']) # ['qstat','-u','alexeys']) if 'alexeys' in job_is_running: - #sleep 500 seconds + # sleep 500 seconds sleep(500) - checkAndSchedule(configBaseName,gpuNodeCountGrid,nextGPUNodeCount) + checkAndSchedule(configBaseName, gpuNodeCountGrid, nextGPUNodeCount) else: - #create a config - nextConfigName = createOneConfig(configBaseName,gpuNodeCountGrid[nextGPUNodeCount]) - print "Submitting next PBS job {} to run on {} GPUs".format(configBaseName,gpuNodeCountGrid[nextGPUNodeCount]) - print "sbatch "+nextConfigName - Popen("sbatch "+nextConfigName,shell=True).wait() - #update parameters + # create a config + nextConfigName = createOneConfig( + configBaseName, gpuNodeCountGrid[nextGPUNodeCount]) + print("Submitting next PBS job {} to run on {} GPUs".format( + configBaseName, gpuNodeCountGrid[nextGPUNodeCount])) + print("sbatch ", nextConfigName) + Popen("sbatch " + nextConfigName, shell=True).wait() + # update parameters nextGPUNodeCount += 1 sleep(10) - checkAndSchedule(configBaseName,gpuNodeCountGrid,nextGPUNodeCount) + checkAndSchedule(configBaseName, gpuNodeCountGrid, nextGPUNodeCount) def createOneConfig(configBaseName, GPUcount): - configFullName = configBaseName+str(GPUcount)+".cmd" - with open(configFullName,"w") as f: - f.write('#!/bin/bash\n') + configFullName = configBaseName + str(GPUcount) + ".cmd" + with open(configFullName, "w") as f: + f.write('#!/bin/bash\n') f.write('#SBATCH -t 01:00:00\n') f.write('#SBATCH -N '+str(GPUcount)+'\n') f.write('#SBATCH --ntasks-per-node=4\n') @@ -35,14 +40,15 @@ def createOneConfig(configBaseName, GPUcount): f.write('\n\n') f.write('module load anaconda\n') f.write('source activate PPPL\n') - f.write('module load cudatoolkit/8.0 cudann/cuda-8.0/5.1 openmpi/intel-17.0/1.10.2/64 intel/17.0/64/17.0.2.174\n') - + f.write(('module load cudatoolkit/8.0 cudann/cuda-8.0/5.1 ' + 'openmpi/intel-17.0/1.10.2/64 intel/17.0/64/17.0.2.174\n')) f.write('rm /tigress/alexeys/model_checkpoints/*\n') - f.write('srun python mpi_learn.py\n') + f.write('srun python mpi_learn.py\n') return configFullName -if __name__=='__main__': - gpuNodeCountGrid = [1,3,6,12,24,32,48] + +if __name__ == '__main__': + gpuNodeCountGrid = [1, 3, 6, 12, 24, 32, 48] configBaseName = "FRNN_TigerGPU" - checkAndSchedule(configBaseName,gpuNodeCountGrid,0) + checkAndSchedule(configBaseName, gpuNodeCountGrid, 0) diff --git a/examples/signal_influence.py b/examples/signal_influence.py index fc987177..3fbae01c 100644 --- a/examples/signal_influence.py +++ b/examples/signal_influence.py @@ -1,3 +1,13 @@ +from plasma.models.mpi_runner import ( + mpi_make_predictions + ) +from mpi4py import MPI +from plasma.preprocessor.preprocess import guarantee_preprocessed +from plasma.preprocessor.augment import ByShotAugmentator +from plasma.primitives.shots import ShotList +from plasma.models.loader import Loader +from plasma.conf import conf +from pprint import pprint ''' ######################################################### This file trains a deep learning model to predict @@ -16,10 +26,8 @@ ######################################################### ''' -from __future__ import print_function import os -import sys -import time +import sys import datetime import random import numpy as np @@ -29,39 +37,36 @@ import matplotlib matplotlib.use('Agg') -from pprint import pprint sys.setrecursionlimit(10000) -from plasma.conf import conf -from plasma.models.loader import Loader -from plasma.primitives.shots import ShotList -from plasma.preprocessor.normalize import Normalizer -from plasma.preprocessor.augment import ByShotAugmentator -from plasma.preprocessor.preprocess import guarantee_preprocessed if conf['model']['shallow']: - print("Shallow learning using MPI is not supported yet. set conf['model']['shallow'] to false.") + print( + "Shallow learning using MPI is not supported yet. ", + "Set conf['model']['shallow'] to False.") exit(1) if conf['data']['normalizer'] == 'minmax': from plasma.preprocessor.normalize import MinMaxNormalizer as Normalizer elif conf['data']['normalizer'] == 'meanvar': from plasma.preprocessor.normalize import MeanVarNormalizer as Normalizer elif conf['data']['normalizer'] == 'var': - from plasma.preprocessor.normalize import VarNormalizer as Normalizer #performs !much better than minmaxnormalizer + # performs !much better than minmaxnormalizer + from plasma.preprocessor.normalize import VarNormalizer as Normalizer elif conf['data']['normalizer'] == 'averagevar': - from plasma.preprocessor.normalize import AveragingVarNormalizer as Normalizer #performs !much better than minmaxnormalizer + # performs !much better than minmaxnormalizer + from plasma.preprocessor.normalize import ( + AveragingVarNormalizer as Normalizer + ) else: print('unkown normalizer. exiting') exit(1) -from mpi4py import MPI comm = MPI.COMM_WORLD task_index = comm.Get_rank() num_workers = comm.Get_size() NUM_GPUS = conf['num_gpus'] MY_GPU = task_index % NUM_GPUS -from plasma.models.mpi_runner import * np.random.seed(task_index) random.seed(task_index) @@ -73,102 +78,132 @@ if only_predict: custom_path = sys.argv[1] shot_num = int(sys.argv[2]) -print("predicting using path {} on shot {}".format(custom_path,shot_num)) +print("predicting using path {} on shot {}".format(custom_path, shot_num)) assert(only_predict) + ##################################################### -####################Normalization#################### +# NORMALIZATION # ##################################################### -if task_index == 0: #make sure preprocessing has been run, and is saved as a file - shot_list_train,shot_list_validate,shot_list_test = guarantee_preprocessed(conf) +# TODO(KGF): identical in at least 3x files in examples/ +# make sure preprocessing has been run, and is saved as a file +if task_index == 0: + # TODO(KGF): check tuple unpack + (shot_list_train, shot_list_validate, + shot_list_test) = guarantee_preprocessed(conf) comm.Barrier() -shot_list_train,shot_list_validate,shot_list_test = guarantee_preprocessed(conf) +(shot_list_train, shot_list_validate, + shot_list_test) = guarantee_preprocessed(conf) -shot_list = sum([l.filter_by_number([shot_num]) for l in [shot_list_train,shot_list_validate,shot_list_test]],ShotList()) +shot_list = sum([l.filter_by_number([shot_num]) + for l in [shot_list_train, shot_list_validate, + shot_list_test]], ShotList()) assert(len(shot_list) == 1) # for s in shot_list.shots: - # s.restore() +# s.restore() + def chunks(l, n): """Yield successive n-sized chunks from l.""" - return[ l[i:i + n] for i in range(0, len(l), n)] + return[l[i:i + n] for i in range(0, len(l), n)] + -def hide_signal_data(shot,t=0,sigs_to_hide=None): +def hide_signal_data(shot, t=0, sigs_to_hide=None): for sig in shot.signals: - if sigs_to_hide is None or (sigs_to_hide is not None and sig in sigs_to_hide): - shot.signals_dict[sig][t:,:] = shot.signals_dict[sig][t,:] + if sigs_to_hide is None or ( + sigs_to_hide is not None and sig in sigs_to_hide): + shot.signals_dict[sig][t:, :] = shot.signals_dict[sig][t, :] -def create_shot_list_tmp(original_shot,time_points,sigs=None): + +def create_shot_list_tmp(original_shot, time_points, sigs=None): shot_list_tmp = ShotList() T = len(original_shot.ttd) - t_range = np.linspace(0,T-1,time_points,dtype=np.int) + t_range = np.linspace(0, T-1, time_points, dtype=np.int) for t in t_range: new_shot = copy.copy(original_shot) - assert(new_shot.augmentation_fn == None) - new_shot.augmentation_fn = partial(hide_signal_data,t = t,sigs_to_hide=sigs) - #new_shot.number = original_shot.number + assert(new_shot.augmentation_fn is None) + new_shot.augmentation_fn = partial( + hide_signal_data, t=t, sigs_to_hide=sigs) + # new_shot.number = original_shot.number shot_list_tmp.append(new_shot) - return shot_list_tmp,t_range - -def get_importance_measure(original_shot,loader,custom_path,metric,time_points=10,sig=None): - shot_list_tmp,t_range = create_shot_list_tmp(original_shot,time_points,sigs) - y_prime,y_gold,disruptive = mpi_make_predictions(conf,shot_list_tmp,loader,custom_path) + return shot_list_tmp, t_range + + +def get_importance_measure( + original_shot, + loader, + custom_path, + metric, + time_points=10, + sig=None): + shot_list_tmp, t_range = create_shot_list_tmp( + original_shot, time_points, sigs) + y_prime, y_gold, disruptive = mpi_make_predictions( + conf, shot_list_tmp, loader, custom_path) shot_list_tmp.make_light() - return t_range,get_importance_measure_given_y_prime(y_prime,metric),y_prime[-1] + return t_range, get_importance_measure_given_y_prime( + y_prime, metric), y_prime[-1] -def difference_metric(y_prime,y_prime_orig): - idx = np.argmax(y_prime_orig) - return (np.max(y_prime_orig) - y_prime[idx])/(np.max(y_prime_orig) - np.min(y_prime_orig)) -def get_importance_measure_given_y_prime(y_prime,metric): - differences = [metric(y_prime[i],y_prime[-1]) for i in range(len(y_prime))] - return 1.0-np.array(differences)#/np.max(differences) +def difference_metric(y_prime, y_prime_orig): + idx = np.argmax(y_prime_orig) + return (np.max(y_prime_orig) - y_prime[idx]) / \ + (np.max(y_prime_orig) - np.min(y_prime_orig)) +def get_importance_measure_given_y_prime(y_prime, metric): + differences = [metric(y_prime[i], y_prime[-1]) + for i in range(len(y_prime))] + return 1.0-np.array(differences) # /np.max(differences) original_shot = shot_list[0] original_shot.augmentation_fn = None original_shot.restore(conf['paths']['processed_prepath']) -#remove original shot +# remove original shot -print("normalization",end='') +print("normalization", end='') normalizer = Normalizer(conf) normalizer.train() normalizer = ByShotAugmentator(normalizer) -loader = Loader(conf,normalizer) +loader = Loader(conf, normalizer) print("...done") # if not only_predict: # mpi_train(conf,shot_list_train,shot_list_validate,loader) -#load last model for testing +# load last model for testing loader.set_inference_mode(True) use_signals = copy.copy(conf['paths']['use_signals']) use_signals.append(None) importances = dict() y_prime = 0 -use_signals = [[s] for s in use_signals[:-3]] + [use_signals[-3:-1]] + [use_signals[-1]] +use_signals = [ + [s] for s in use_signals[:-3]] + [use_signals[-3:-1]] + [use_signals[-1]] print(use_signals) for sigs in use_signals: - t_range,measure,y_prime = get_importance_measure(original_shot,loader,custom_path,difference_metric,time_points=128,sig=sigs) + t_range, measure, y_prime = get_importance_measure(original_shot, loader, + custom_path, + difference_metric, + time_points=128, + sig=sigs) if sigs is None: idx = None else: idx = tuple(sorted(sigs)) - importances[idx] = (t_range,measure) - + importances[idx] = (t_range, measure) if task_index == 0: - save_str = 'signal_influence_results_{}_'.format(shot_num) + datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + save_str = 'signal_influence_results_{}_'.format( + shot_num) + datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") result_base_path = conf['paths']['results_prepath'] if not os.path.exists(result_base_path): os.makedirs(result_base_path) - np.savez(result_base_path+save_str, - original_shot=original_shot,importances=importances,y_prime=y_prime,conf = conf) + np.savez(result_base_path + save_str, original_shot=original_shot, + importances=importances, y_prime=y_prime, conf=conf) shot_list.make_light() sys.stdout.flush() diff --git a/examples/simple_augmentation.py b/examples/simple_augmentation.py index 3afd19c9..8be07b85 100644 --- a/examples/simple_augmentation.py +++ b/examples/simple_augmentation.py @@ -1,3 +1,13 @@ +from plasma.models.mpi_runner import ( + mpi_make_predictions_and_evaluate + ) +from mpi4py import MPI +from plasma.preprocessor.preprocess import guarantee_preprocessed +from plasma.preprocessor.augment import ByShotAugmentator +from plasma.primitives.shots import ShotList +from plasma.models.loader import Loader +from plasma.conf import conf +from pprint import pprint ''' ######################################################### This file trains a deep learning model to predict @@ -16,11 +26,7 @@ ######################################################### ''' -from __future__ import print_function -import os -import sys -import time -import datetime +import sys import random import numpy as np import copy @@ -29,39 +35,36 @@ import matplotlib matplotlib.use('Agg') -from pprint import pprint sys.setrecursionlimit(10000) -from plasma.conf import conf -from plasma.models.loader import Loader -from plasma.primitives.shots import ShotList -from plasma.preprocessor.normalize import Normalizer -from plasma.preprocessor.augment import ByShotAugmentator -from plasma.preprocessor.preprocess import guarantee_preprocessed if conf['model']['shallow']: - print("Shallow learning using MPI is not supported yet. set conf['model']['shallow'] to false.") + print( + "Shallow learning using MPI is not supported yet. ", + "Set conf['model']['shallow'] to False.") exit(1) if conf['data']['normalizer'] == 'minmax': from plasma.preprocessor.normalize import MinMaxNormalizer as Normalizer elif conf['data']['normalizer'] == 'meanvar': from plasma.preprocessor.normalize import MeanVarNormalizer as Normalizer elif conf['data']['normalizer'] == 'var': - from plasma.preprocessor.normalize import VarNormalizer as Normalizer #performs !much better than minmaxnormalizer + # performs !much better than minmaxnormalizer + from plasma.preprocessor.normalize import VarNormalizer as Normalizer elif conf['data']['normalizer'] == 'averagevar': - from plasma.preprocessor.normalize import AveragingVarNormalizer as Normalizer #performs !much better than minmaxnormalizer + # performs !much better than minmaxnormalizer + from plasma.preprocessor.normalize import ( + AveragingVarNormalizer as Normalizer + ) else: print('unkown normalizer. exiting') exit(1) -from mpi4py import MPI comm = MPI.COMM_WORLD task_index = comm.Get_rank() num_workers = comm.Get_size() NUM_GPUS = conf['num_gpus'] MY_GPU = task_index % NUM_GPUS -from plasma.models.mpi_runner import * np.random.seed(task_index) random.seed(task_index) @@ -75,86 +78,115 @@ print("predicting using path {}".format(custom_path)) assert(only_predict) + + ##################################################### -####################Normalization#################### +# NORMALIZATION # ##################################################### -if task_index == 0: #make sure preprocessing has been run, and is saved as a file - shot_list_train,shot_list_validate,shot_list_test = guarantee_preprocessed(conf) +# TODO(KGF): identical in at least 3x files in examples/ +# make sure preprocessing has been run, and is saved as a file +if task_index == 0: + # TODO(KGF): check tuple unpack + (shot_list_train, shot_list_validate, + shot_list_test) = guarantee_preprocessed(conf) comm.Barrier() -shot_list_train,shot_list_validate,shot_list_test = guarantee_preprocessed(conf) +(shot_list_train, shot_list_validate, + shot_list_test) = guarantee_preprocessed(conf) def chunks(l, n): """Yield successive n-sized chunks from l.""" - return[ l[i:i + n] for i in range(0, len(l), n)] + return[l[i:i + n] for i in range(0, len(l), n)] + -def hide_signal_data(shot,t=0,sigs_to_hide=None): +def hide_signal_data(shot, t=0, sigs_to_hide=None): for sig in shot.signals: - if sigs_to_hide is None or (sigs_to_hide is not None and sig in sigs_to_hide): - shot.signals_dict[sig][t:,:] = shot.signals_dict[sig][t,:] + if sigs_to_hide is None or ( + sigs_to_hide is not None and sig in sigs_to_hide): + shot.signals_dict[sig][t:, :] = shot.signals_dict[sig][t, :] -def create_shot_list_tmp(original_shot,time_points,sigs=None): + +def create_shot_list_tmp(original_shot, time_points, sigs=None): shot_list_tmp = ShotList() T = len(original_shot.ttd) - t_range = np.linspace(0,T-1,time_points,dtype=np.int) + t_range = np.linspace(0, T-1, time_points, dtype=np.int) for t in t_range: new_shot = copy.copy(original_shot) - assert(new_shot.augmentation_fn == None) - new_shot.augmentation_fn = partial(hide_signal_data,t = t,sigs_to_hide=sigs) + assert(new_shot.augmentation_fn is None) + new_shot.augmentation_fn = partial( + hide_signal_data, t=t, sigs_to_hide=sigs) #new_shot.number = original_shot.number shot_list_tmp.append(new_shot) - return shot_list_tmp,t_range - -def get_importance_measure(original_shot,loader,custom_path,metric,time_points=10,sig=None): - shot_list_tmp,t_range = create_shot_list_tmp(original_shot,time_points,sigs) - y_prime,y_gold,disruptive = mpi_make_predictions(conf,shot_list_tmp,loader,custom_path) + return shot_list_tmp, t_range + + +def get_importance_measure( + original_shot, + loader, + custom_path, + metric, + time_points=10, + sig=None): + shot_list_tmp, t_range = create_shot_list_tmp( + original_shot, time_points, sigs) + y_prime, y_gold, disruptive = mpi_make_predictions( + conf, shot_list_tmp, loader, custom_path) shot_list_tmp.make_light() - return t_range,get_importance_measure_given_y_prime(y_prime,metric),y_prime[-1] + return t_range, get_importance_measure_given_y_prime( + y_prime, metric), y_prime[-1] + -def difference_metric(y_prime,y_prime_orig): - idx = np.argmax(y_prime_orig) - return (np.max(y_prime_orig) - y_prime[idx])/(np.max(y_prime_orig) - np.min(y_prime_orig)) +def difference_metric(y_prime, y_prime_orig): + idx = np.argmax(y_prime_orig) + return (np.max(y_prime_orig) - y_prime[idx]) / \ + (np.max(y_prime_orig) - np.min(y_prime_orig)) -def get_importance_measure_given_y_prime(y_prime,metric): - differences = [metric(y_prime[i],y_prime[-1]) for i in range(len(y_prime))] - return 1.0-np.array(differences)#/np.max(differences) +def get_importance_measure_given_y_prime(y_prime, metric): + differences = [metric(y_prime[i], y_prime[-1]) + for i in range(len(y_prime))] + return 1.0-np.array(differences) # /np.max(differences) -print("normalization",end='') + +print("normalization", end='') normalizer = Normalizer(conf) normalizer.train() normalizer = ByShotAugmentator(normalizer) -loader = Loader(conf,normalizer) +loader = Loader(conf, normalizer) print("...done") # if not only_predict: # mpi_train(conf,shot_list_train,shot_list_validate,loader) -#load last model for testing +# load last model for testing loader.set_inference_mode(True) use_signals = copy.copy(conf['paths']['use_signals']) use_signals.append(None) - for shot in shot_list_test: - shot.augmentation_fn = None# partial(hide_signal_data,t = 0,sigs_to_hide = sigs_to_hide) + # partial(hide_signal_data,t = 0,sigs_to_hide = sigs_to_hide) + shot.augmentation_fn = None print("All signals:") -y_prime,y_gold,disruptive,roc,loss = mpi_make_predictions_and_evaluate(conf,shot_list_test,loader,custom_path) +y_prime, y_gold, disruptive, roc, loss = mpi_make_predictions_and_evaluate( + conf, shot_list_test, loader, custom_path) print(roc) print(loss) -#for sigs_to_hide in [[s] for s in use_signals[:-3]] + [use_signals[-3:-1]] + [use_signals[-1]]: -for sigs_to_hide in [[s] for s in use_signals[:-3]] + [[s] for s in use_signals[-3:-1]] + [use_signals[-3:-1]]:# + [use_signals[-1]]: +# for sigs_to_hide in [[s] for s in use_signals[:-3]] + +# [use_signals[-3:-1]] + [use_signals[-1]]: +for sigs_to_hide in [[s] for s in use_signals[:-3]] + [[s] + for s in use_signals[-3:-1]] + [use_signals[-3:-1]]: # + [use_signals[-1]]: for shot in shot_list_test: - shot.augmentation_fn = partial(hide_signal_data,t = 0,sigs_to_hide = sigs_to_hide) + shot.augmentation_fn = partial( + hide_signal_data, t=0, sigs_to_hide=sigs_to_hide) print("Hiding: {}".format(sigs_to_hide)) - y_prime,y_gold,disruptive,roc,loss = mpi_make_predictions_and_evaluate(conf,shot_list_test,loader,custom_path) + y_prime, y_gold, disruptive, roc, loss = mpi_make_predictions_and_evaluate( + conf, shot_list_test, loader, custom_path) print(roc) print(loss) - if task_index == 0: print('finished.') diff --git a/examples/submit_batch_job.py b/examples/submit_batch_job.py index d13f8213..facc2214 100644 --- a/examples/submit_batch_job.py +++ b/examples/submit_batch_job.py @@ -1,7 +1,10 @@ -from plasma.utils.batch_jobs import get_executable_name,generate_working_dirname,start_slurm_job,copy_files_to_environment -from pprint import pprint +from plasma.utils.batch_jobs import ( + get_executable_name, generate_working_dirname, + start_slurm_job, copy_files_to_environment + ) import yaml -import sys,os,getpass +import os +import getpass import plasma.conf # tunables = [] @@ -10,49 +13,62 @@ num_trials = 1 -run_directory = "{}/{}/batch_jobs/".format(plasma.conf.conf['fs_path'],getpass.getuser()) -template_path = os.environ['PWD'] #"/home/{}/plasma-python/examples/".format(getpass.getuser()) +run_directory = "{}/{}/batch_jobs/".format( + plasma.conf.conf['fs_path'], getpass.getuser()) +# "/home/{}/plasma-python/examples/".format(getpass.getuser()) +template_path = os.environ['PWD'] conf_name = "conf.yaml" -def copy_conf_file(shallow,template_path = "../",save_path = "./",conf_name="conf.yaml"): + +def copy_conf_file( + shallow, + template_path="../", + save_path="./", + conf_name="conf.yaml"): assert(template_path != save_path) - pathsrc = os.path.join(template_path,conf_name) - pathdst = os.path.join(save_path,conf_name) + pathsrc = os.path.join(template_path, conf_name) + pathdst = os.path.join(save_path, conf_name) with open(pathsrc, 'r') as yaml_file: conf = yaml.load(yaml_file) - conf['training']['hyperparam_tuning'] = True #make sure all files like checkpoints and normalization are done locally + # make sure all files like checkpoints and normalization are done locally + conf['training']['hyperparam_tuning'] = True with open(pathdst, 'w') as outfile: yaml.dump(conf, outfile, default_flow_style=False) -def get_conf(template_path,conf_name): - with open(os.path.join(template_path,conf_name), 'r') as yaml_file: + +def get_conf(template_path, conf_name): + with open(os.path.join(template_path, conf_name), 'r') as yaml_file: conf = yaml.load(yaml_file) return conf -conf = get_conf(template_path,conf_name) + +conf = get_conf(template_path, conf_name) shallow = conf['model']['shallow'] if shallow: num_nodes = 1 working_directory = generate_working_dirname(run_directory) os.makedirs(working_directory) -#copy conf and executable into directory -executable_name,_ = get_executable_name(conf) -os.system(" ".join(["cp -p",os.path.join(template_path,conf_name),working_directory])) -os.system(" ".join(["cp -p",os.path.join(template_path,executable_name),working_directory])) +# copy conf and executable into directory +executable_name, _ = get_executable_name(conf) +os.system(" ".join(["cp -p", os.path.join(template_path, conf_name), + working_directory])) +os.system(" ".join(["cp -p", os.path.join(template_path, executable_name), + working_directory])) os.chdir(working_directory) print("Going into {}".format(working_directory)) for i in range(num_trials): - subdir = working_directory + "/{}/".format(i) + subdir = working_directory + "/{}/".format(i) os.makedirs(subdir) copy_files_to_environment(subdir) print("Making modified conf") - copy_conf_file(shallow,working_directory,subdir,conf_name) + copy_conf_file(shallow, working_directory, subdir, conf_name) print("Starting job") - start_slurm_job(subdir,num_nodes,i,conf,shallow,conf['env']['name'],conf['env']['type']) + start_slurm_job(subdir, num_nodes, i, conf, + shallow, conf['env']['name'], conf['env']['type']) print("submitted {} jobs.".format(num_trials)) diff --git a/examples/test.py b/examples/test.py index 544ac257..3a3cfd22 100644 --- a/examples/test.py +++ b/examples/test.py @@ -1,65 +1,64 @@ -import keras -from keras.models import Sequential, Model +# import keras +from keras.models import Model # , Sequential from keras.layers import Input -from keras.layers.core import Dense, Activation, Dropout, Lambda, Reshape, Flatten, Permute -from keras.layers.recurrent import LSTM, SimpleRNN -from keras.layers.convolutional import Convolution1D -from keras.layers.pooling import MaxPooling1D -from keras.utils.data_utils import get_file -from keras.layers.wrappers import TimeDistributed -from keras.layers.merge import Concatenate -from keras.callbacks import Callback -from keras.optimizers import * -from keras.regularizers import l1,l2,l1_l2 +from keras.layers.core import Lambda, Reshape, Permute +# Dense, Activation, Dropout, Flatten +# from keras.layers.recurrent import LSTM, SimpleRNN +# from keras.layers.convolutional import Convolution1D +# from keras.layers.pooling import MaxPooling1D +# from keras.utils.data_utils import get_file +# from keras.layers.wrappers import TimeDistributed +# from keras.layers.merge import Concatenate +# from keras.callbacks import Callback +# from keras.optimizers import * +# from keras.regularizers import l1, l2, l1_l2 - -import keras.backend as K - -import dill -import re -import os,sys import numpy as np -from copy import deepcopy x = np.array(range(8)) num_signals = len(x) x = np.atleast_2d(x) -x = np.reshape(x,(1,num_signals)) +x = np.reshape(x, (1, num_signals)) print(x.shape) print(x) -indices_0d = np.array([0,1]) -indices_1d = np.array([2,3,4,5,6,7]) +indices_0d = np.array([0, 1]) +indices_1d = np.array([2, 3, 4, 5, 6, 7]) num_1D = 2 pre_rnn_input = Input(shape=(num_signals,)) -pre_rnn_1D = Lambda(lambda x: x[:,len(indices_0d):],output_shape=(len(indices_1d),))(pre_rnn_input) -pre_rnn_0D = Lambda(lambda x: x[:,:len(indices_0d)],output_shape=(len(indices_0d),))(pre_rnn_input)# slicer(x,indices_0d),lambda s: slicer_output_shape(s,indices_0d))(pre_rnn_input) -pre_rnn_1D = Reshape((num_1D,len(indices_1d)/num_1D)) (pre_rnn_1D) -pre_rnn_1D = Permute((2,1)) (pre_rnn_1D) - +pre_rnn_1D = Lambda(lambda x: x[:, len(indices_0d):], output_shape=( + len(indices_1d),))(pre_rnn_input) +pre_rnn_0D = Lambda(lambda x: x[:, :len(indices_0d)], + output_shape=(len(indices_0d),))(pre_rnn_input) +# slicer(x, indices_0d), +# lambda s: slicer_output_shape(s, indices_0d))(pre_rnn_input) +pre_rnn_1D = Reshape((num_1D, len(indices_1d)/num_1D))(pre_rnn_1D) +pre_rnn_1D = Permute((2, 1))(pre_rnn_1D) + # for i in range(model_conf['num_conv_layers']): -# pre_rnn_1D = Convolution1D(num_conv_filters,size_conv_filters,padding='valid',activation='relu') (pre_rnn_1D) +# pre_rnn_1D = Convolution1D(num_conv_filters, size_conv_filters, +# padding='valid',activation='relu') (pre_rnn_1D) # pre_rnn_1D = MaxPooling1D(pool_size) (pre_rnn_1D) # pre_rnn_1D = Flatten() (pre_rnn_1D) # pre_rnn = Concatenate() ([pre_rnn_0D,pre_rnn_1D]) -model = Model(inputs = pre_rnn_input,outputs=pre_rnn_1D) +model = Model(inputs=pre_rnn_input, outputs=pre_rnn_1D) # x_input = Input(batch_shape = batch_input_shape) # x_in = TimeDistributed(pre_rnn_model) (x_input) # if return_sequences: - #x_out = TimeDistributed(Dense(100,activation='tanh')) (x_in) - # x_out = TimeDistributed(Dense(1,activation=output_activation)) (x_in) +# x_out = TimeDistributed(Dense(100, activation='tanh')) (x_in) +# x_out = TimeDistributed(Dense(1, activation=output_activation)) (x_in) # else: - # x_out = Dense(1,activation=output_activation) (x_in) -model.compile(loss='mse',optimizer='sgd') +# x_out = Dense(1, activation=output_activation) (x_in) +model.compile(loss='mse', optimizer='sgd') y = model.predict(x) print(model.layers) print(x) print(y) print(y.shape) -print(y[0,:,0]) -#bug with tensorflow/Keras \ No newline at end of file +print(y[0, :, 0]) +# bug with tensorflow/Keras --- ????? diff --git a/examples/tune_hyperparams.py b/examples/tune_hyperparams.py index 56b8eb69..6d8f5d8f 100644 --- a/examples/tune_hyperparams.py +++ b/examples/tune_hyperparams.py @@ -1,8 +1,14 @@ -from plasma.primitives.hyperparameters import CategoricalHyperparam,ContinuousHyperparam,LogContinuousHyperparam,IntegerHyperparam -from plasma.utils.batch_jobs import create_slurm_script,create_slurm_header,start_slurm_job,generate_working_dirname,copy_files_to_environment -from pprint import pprint +from plasma.primitives.hyperparameters import ( + CategoricalHyperparam, ContinuousHyperparam, + LogContinuousHyperparam, IntegerHyperparam + ) +from plasma.utils.batch_jobs import ( + # create_slurm_script, create_slurm_header, + start_slurm_job, generate_working_dirname, copy_files_to_environment + ) import yaml -import sys,os,getpass +import os +import getpass import plasma.conf tunables = [] @@ -10,61 +16,107 @@ num_nodes = 1 num_trials = 20 -t_warn = CategoricalHyperparam(['data','T_warning'],[0.256,1.024,10.024]) -cut_ends = CategoricalHyperparam(['data','cut_shot_ends'],[False,True]) -#for shallow +t_warn = CategoricalHyperparam(['data', 'T_warning'], [0.256, 1.024, 10.024]) +cut_ends = CategoricalHyperparam(['data', 'cut_shot_ends'], [False, True]) +# for shallow if shallow: num_nodes = 1 - shallow_model = CategoricalHyperparam(['model','shallow_model','type'],["svm","random_forest","xgboost","mlp"]) - n_estimators = CategoricalHyperparam(['model','shallow_model','n_estimators'],[5,20,50,100,300,1000]) - max_depth = CategoricalHyperparam(['model','shallow_model','max_depth'],[None,3,6,10,30,100]) - C = LogContinuousHyperparam(['model','shallow_model','C'],1e-3,1e3) - kernel = CategoricalHyperparam(['model','shallow_model','kernel'],["rbf","sigmoid","linear","poly"]) - xg_learning_rate = ContinuousHyperparam(['model','shallow_model','learning_rate'],0,1) - scale_pos_weight = CategoricalHyperparam(['model','shallow_model','scale_pos_weight'],[1,10.0,100.0]) - num_samples = CategoricalHyperparam(['model','shallow_model','num_samples'],[30000,100000,1000000,2000000]) - hidden_size = CategoricalHyperparam(['model','shallow_model','final_hidden_layer_size'],[5,10,20]) - hidden_num = CategoricalHyperparam(['model','shallow_model','num_hidden_layers'],[2,4]) - mlp_learning_rate = CategoricalHyperparam(['model','shallow_model','learning_rate_mlp'],[0.001,0.0001,0.00001]) - mlp_regularization = CategoricalHyperparam(['model','shallow_model','mlp_regularization'],[0.1,0.003,0.0001]) - tunables = [shallow_model,n_estimators,max_depth,C,kernel,xg_learning_rate,scale_pos_weight,num_samples,hidden_num,hidden_size,mlp_learning_rate,mlp_regularization] #target + shallow_model = CategoricalHyperparam( + ['model', 'shallow_model', 'type'], + ["svm", "random_forest", "xgboost", "mlp"]) + n_estimators = CategoricalHyperparam( + ['model', 'shallow_model', 'n_estimators'], + [5, 20, 50, 100, 300, 1000]) + max_depth = CategoricalHyperparam( + ['model', 'shallow_model', 'max_depth'], + [None, 3, 6, 10, 30, 100]) + C = LogContinuousHyperparam(['model', 'shallow_model', 'C'], 1e-3, 1e3) + kernel = CategoricalHyperparam(['model', 'shallow_model', 'kernel'], [ + "rbf", "sigmoid", "linear", "poly"]) + xg_learning_rate = ContinuousHyperparam( + ['model', 'shallow_model', 'learning_rate'], 0, 1) + scale_pos_weight = CategoricalHyperparam( + ['model', 'shallow_model', 'scale_pos_weight'], [1, 10.0, 100.0]) + num_samples = CategoricalHyperparam( + ['model', 'shallow_model', 'num_samples'], + [30000, 100000, 1000000, 2000000]) + hidden_size = CategoricalHyperparam( + ['model', 'shallow_model', 'final_hidden_layer_size'], [5, 10, 20]) + hidden_num = CategoricalHyperparam( + ['model', 'shallow_model', 'num_hidden_layers'], [2, 4]) + mlp_learning_rate = CategoricalHyperparam( + ['model', 'shallow_model', 'learning_rate_mlp'], + [0.001, 0.0001, 0.00001]) + mlp_regularization = CategoricalHyperparam( + ['model', 'shallow_model', 'mlp_regularization'], [0.1, 0.003, 0.0001]) + tunables = [ + shallow_model, + n_estimators, + max_depth, + C, + kernel, + xg_learning_rate, + scale_pos_weight, + num_samples, + hidden_num, + hidden_size, + mlp_learning_rate, + mlp_regularization] # target else: - #for DL - lr = LogContinuousHyperparam(['model','lr'],1e-7,1e-4) - lr_decay = CategoricalHyperparam(['model','lr_decay'],[0.97,0.985,1.0]) - fac = CategoricalHyperparam(['data','positive_example_penalty'],[1.0,4.0,16.0]) - target = CategoricalHyperparam(['target'],['maxhinge','hinge','ttdinv','ttd']) - #target = CategoricalHyperparam(['target'],['hinge','ttdinv','ttd']) - batch_size = CategoricalHyperparam(['training','batch_size'],[128,256]) - dropout_prob = CategoricalHyperparam(['model','dropout_prob'],[0.01,0.05,0.1]) - conv_filters = CategoricalHyperparam(['model','num_conv_filters'],[128,256]) - conv_layers = IntegerHyperparam(['model','num_conv_layers'],2,4) - rnn_layers = IntegerHyperparam(['model','rnn_layers'],1,3) - rnn_size = CategoricalHyperparam(['model','rnn_size'],[128,256]) - dense_size = CategoricalHyperparam(['model','dense_size'],[128,256]) - extra_dense_input = CategoricalHyperparam(['model','extra_dense_input'],[False,True]) - equalize_classes = CategoricalHyperparam(['data','equalize_classes'],[False,True]) - #rnn_length = CategoricalHyperparam(['model','length'],[32,128]) - #tunables = [lr,lr_decay,fac,target,batch_size,dropout_prob] - tunables = [lr,lr_decay,fac,target,batch_size,equalize_classes,dropout_prob] - tunables += [conv_filters,conv_layers,rnn_layers,rnn_size,dense_size,extra_dense_input] -tunables += [cut_ends,t_warn] + # for DL + lr = LogContinuousHyperparam(['model', 'lr'], 1e-7, 1e-4) + lr_decay = CategoricalHyperparam(['model', 'lr_decay'], [0.97, 0.985, 1.0]) + fac = CategoricalHyperparam( + ['data', 'positive_example_penalty'], [1.0, 4.0, 16.0]) + target = CategoricalHyperparam( + ['target'], ['maxhinge', 'hinge', 'ttdinv', 'ttd']) + # target = CategoricalHyperparam(['target'],['hinge','ttdinv','ttd']) + batch_size = CategoricalHyperparam(['training', 'batch_size'], [128, 256]) + dropout_prob = CategoricalHyperparam( + ['model', 'dropout_prob'], [0.01, 0.05, 0.1]) + conv_filters = CategoricalHyperparam( + ['model', 'num_conv_filters'], [128, 256]) + conv_layers = IntegerHyperparam(['model', 'num_conv_layers'], 2, 4) + rnn_layers = IntegerHyperparam(['model', 'rnn_layers'], 1, 3) + rnn_size = CategoricalHyperparam(['model', 'rnn_size'], [128, 256]) + dense_size = CategoricalHyperparam(['model', 'dense_size'], [128, 256]) + extra_dense_input = CategoricalHyperparam( + ['model', 'extra_dense_input'], [False, True]) + equalize_classes = CategoricalHyperparam( + ['data', 'equalize_classes'], [False, True]) + # rnn_length = CategoricalHyperparam(['model', 'length'], [32, 128]) + # tunables = [lr, lr_decay, fac, target, batch_size, dropout_prob] + tunables = [lr, lr_decay, fac, target, batch_size, equalize_classes, + dropout_prob] + tunables += [conv_filters, conv_layers, rnn_layers, + rnn_size, dense_size, extra_dense_input] +tunables += [cut_ends, t_warn] -run_directory = "{}/{}/hyperparams/".format(plasma.conf.conf['fs_path'],getpass.getuser()) -template_path = os.environ['PWD'] #"/home/{}/plasma-python/examples/".format(getpass.getuser()) +run_directory = "{}/{}/hyperparams/".format( + plasma.conf.conf['fs_path'], getpass.getuser()) +# "/home/{}/plasma-python/examples/".format(getpass.getuser()) +template_path = os.environ['PWD'] conf_name = "conf.yaml" -def generate_conf_file(tunables,shallow,template_path = "../",save_path = "./",conf_name="conf.yaml"): + +def generate_conf_file( + tunables, + shallow, + template_path="../", + save_path="./", + conf_name="conf.yaml"): assert(template_path != save_path) - with open(os.path.join(template_path,conf_name), 'r') as yaml_file: + with open(os.path.join(template_path, conf_name), 'r') as yaml_file: conf = yaml.load(yaml_file) for tunable in tunables: - tunable.assign_to_conf(conf,save_path) - conf['training']['num_epochs'] = 1000 #rely on early stopping to terminate training - conf['training']['hyperparam_tuning'] = True #rely on early stopping to terminate training + tunable.assign_to_conf(conf, save_path) + # rely on early stopping to terminate training + conf['training']['num_epochs'] = 1000 + # rely on early stopping to terminate training + conf['training']['hyperparam_tuning'] = True conf['model']['shallow'] = shallow - with open(os.path.join(save_path,conf_name), 'w') as outfile: + with open(os.path.join(save_path, conf_name), 'w') as outfile: yaml.dump(conf, outfile, default_flow_style=False) return conf @@ -77,27 +129,30 @@ def get_executable_name_imposed_shallow(shallow): else: executable_name = conf['paths']['executable'] use_mpi = True - return executable_name,use_mpi - + return executable_name, use_mpi working_directory = generate_working_dirname(run_directory) os.makedirs(working_directory) -executable_name,_ = get_executable_name_imposed_shallow(shallow) -os.system(" ".join(["cp -p",os.path.join(template_path,conf_name),working_directory])) -os.system(" ".join(["cp -p",os.path.join(template_path,executable_name),working_directory])) +executable_name, _ = get_executable_name_imposed_shallow(shallow) +os.system(" ".join(["cp -p", os.path.join(template_path, conf_name), + working_directory])) +os.system(" ".join(["cp -p", os.path.join(template_path, executable_name), + working_directory])) os.chdir(working_directory) print("Going into {}".format(working_directory)) for i in range(num_trials): - subdir = working_directory + "/{}/".format(i) + subdir = working_directory + "/{}/".format(i) os.makedirs(subdir) copy_files_to_environment(subdir) print("Making modified conf") - conf = generate_conf_file(tunables,shallow,working_directory,subdir,conf_name) + conf = generate_conf_file(tunables, shallow, working_directory, + subdir, conf_name) print("Starting job") - start_slurm_job(subdir,num_nodes,i,conf,shallow,conf['env']['name'],conf['env']['type']) + start_slurm_job(subdir, num_nodes, i, conf, shallow, + conf['env']['name'], conf['env']['type']) print("submitted {} jobs.".format(num_trials)) From 33681eb2d5464c680f339c0bed58d42fb9c91db1 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 24 Sep 2019 11:30:33 -0500 Subject: [PATCH 086/272] Clean up files in data/ --- data/d3d_signals.py | 108 +++---- data/gadata.py | 172 ++++++----- data/get_mdsplus_data.py | 15 +- data/jet_signals.py | 238 +++++++------- data/signals.py | 652 ++++++++++++++++++++++++--------------- plasma/conf.py | 11 +- plasma/version.py | 4 +- 7 files changed, 693 insertions(+), 507 deletions(-) diff --git a/data/d3d_signals.py b/data/d3d_signals.py index 1ae4b1a7..2845b570 100644 --- a/data/d3d_signals.py +++ b/data/d3d_signals.py @@ -1,59 +1,59 @@ -#JET signal hierarchy -#------------------------------------------------------------------------# -#User only needs to look at 1st and last sections +# D3D signal hierarchy +# ------------------------------------------------------------------------ +# User only needs to look at 1st and last sections # - conf.py only needs to import signals_dirs and signals_masks # - get_mdsplus_data.py only needs signals_dirs and download_masks # - performance_analysis_utils.py needs : # - signals_dirs, plot_masks, ppf_labels, jpf_labels -#------------------------------------------------------------------------# +# ------------------------------------------------------------------------ ################ # Signal names # ################ -#This section contains all the exact JET signal strings and their -#groupings by type and dimensionality. -#User should not touch this. Use for reference +# This section contains all the exact D3D signal strings and their +# groupings by type and dimensionality. +# User should not touch this. Use for reference -### 0D signals ### +# 0D signals # signal_paths = [ -'efsli', #Internal Inductance -'ipsip', #Plasma Current -'efsbetan', #Normalized Beta -'efswmhd', #Stored Energy -'nssampn1l', #Tearing Mode Amplitude (rotating 2/1) -'nssfrqn1l', #Tearing Mode Frequency (rotating 2/1) -'nssampn2l', #Tearing Mode Amplitude (rotating 3/2) -'nssfrqn2l', #Tearing Mode Frequency (rotating 3/2) -'dusbradial', #LM Amplitude -'dssdenest', #Plasma Density -r'\bol_l15_p', #Radiated Power core -r'\bol_l03_p', #Radiated Power Edge -'bmspinj', #Total Beam Power -'bmstinj',] #Total Beam Torque -#'pcechpwrf'] #Total ECH Power Not always on! + 'efsli', # Internal Inductance + 'ipsip', # Plasma Current + 'efsbetan', # Normalized Beta + 'efswmhd', # Stored Energy + 'nssampn1l', # Tearing Mode Amplitude (rotating 2/1) + 'nssfrqn1l', # Tearing Mode Frequency (rotating 2/1) + 'nssampn2l', # Tearing Mode Amplitude (rotating 3/2) + 'nssfrqn2l', # Tearing Mode Frequency (rotating 3/2) + 'dusbradial', # LM Amplitude + 'dssdenest', # Plasma Density + r'\bol_l15_p', # Radiated Power core + r'\bol_l03_p', # Radiated Power Edge + 'bmspinj', # Total Beam Power + 'bmstinj', ] # Total Beam Torque +# 'pcechpwrf'] #Total ECH Power Not always on! signal_paths = ['d3d/' + path for path in signal_paths] -### 0D EFIT signals ### +# 0D EFIT signals signal_paths += ['EFIT01/RESULTS.AEQDSK.Q95'] - -### 1D EFIT signals ### -#signal_paths += [ -#'AOT/EQU.t_e', #electron temperature profile vs rho (uniform mapping over time) -#'AOT/EQU.dens_e'] #electron density profile vs rho (uniform mapping over time) -#these signals seem to give more reliable data +# 1D EFIT signals +# signal_paths += [ +# 'AOT/EQU.t_e', # electron temperature profile vs rho +# 'AOT/EQU.dens_e'] # electron density profile vs rho + +# these signals seem to give more reliable data signal_paths += [ -'ZIPFIT01/PROFILES.ETEMPFIT', #electron temperature profile vs rho (uniform mapping over time) -'ZIPFIT01/PROFILES.EDENSFIT'] #electron density profile vs rho (uniform mapping over time) + 'ZIPFIT01/PROFILES.ETEMPFIT', # electron temperature profile vs rho + 'ZIPFIT01/PROFILES.EDENSFIT'] # electron density profile vs rho -#make into list of lists format to be consistent with jet_signals.py +# make into list of lists format to be consistent with jet_signals.py signal_paths = [[path] for path in signal_paths] -#format : 'tree/signal_path' for each path +# format : 'tree/signal_path' for each path signals_dirs = signal_paths - + ################################################## # USER SELECTIONS # ################################################## @@ -63,37 +63,39 @@ # Select signals for downloading # ################################## -#Default pass to get_mdsplus_data.py: download all above signals +# Default pass to get_mdsplus_data.py: download all above signals download_masks = [[True]*len(sig_list) for sig_list in signals_dirs] -# download_masks[-1] = [False] # enable/disable temperature profile -# download_masks[-2] = [False] # enable/disable density profile +# download_masks[-1] = [False] # enable/disable temperature profile +# download_masks[-2] = [False] # enable/disable density profile ####################################### # Select signals for training/testing # ####################################### -#Default pass to conf.py: train with all above signals +# Default pass to conf.py: train with all above signals signals_masks = [[True]*len(sig_list) for sig_list in signals_dirs] -signals_masks[-1] = [False] # enable/disable temperature profile -signals_masks[-2] = [False] # enable/disable density profile +signals_masks[-1] = [False] # enable/disable temperature profile +signals_masks[-2] = [False] # enable/disable density profile + +# num_signals = sum([group.count(True) for i, group in +# enumerate(jet_signals.signals_masks)] -#num_signals = sum([group.count(True) for i,group in enumerate(jet_signals.signals_masks)] ########################################### # Select signals for performance analysis # ########################################### -#User selects these by signal name +# User selects these by signal name plot_masks = [[True]*len(sig_list) for sig_list in signals_dirs] -#LaTeX strings for performance analysis, sorted in lists by signal_group +# LaTeX strings for performance analysis, sorted in lists by signal_group group_labels = [[r' $I_{plasma}$ [A]'], - [r' Mode L. A. [A]'], - [r' $P_{radiated}$ [W]'], #0d radiation, db/ - [r' $P_{radiated}$ [W]'],#1d radiation, db/ - [r' $\rho_{plasma}$ [m^-2]'], - [r' $L_{plasma,internal}$'], - [r'$\frac{d}{dt} E_{D}$ [W]'], - [r' $P_{input}$ [W]'], - [r'$E_{D}$'], -#ppf signal labels + [r' Mode L. A. [A]'], + [r' $P_{radiated}$ [W]'], # 0d radiation, db/ + [r' $P_{radiated}$ [W]'], # 1d radiation, db/ + [r' $\rho_{plasma}$ [m^-2]'], + [r' $L_{plasma,internal}$'], + [r'$\frac{d}{dt} E_{D}$ [W]'], + [r' $P_{input}$ [W]'], + [r'$E_{D}$'], + # ppf signal labels [r'ECE unit?']] diff --git a/data/gadata.py b/data/gadata.py index 9494e87c..1906ea58 100644 --- a/data/gadata.py +++ b/data/gadata.py @@ -1,94 +1,108 @@ -import MDSplus +import MDSplus import numpy -import time -import sys +# import time + class gadata: - """GA Data Obj""" - def __init__(self,signal,shot,tree=None,connection=None,nomds=False): + """GA Data Obj""" + + def __init__(self, signal, shot, tree=None, connection=None, nomds=False): - # Save object values - self.signal = signal - self.shot = shot - self.zdata = -1 - self.xdata = -1 - self.ydata = -1 - self.zunits = '' - self.xunits = '' - self.yunits = '' - self.rank = -1 - self.connection = connection - + # Save object values + self.signal = signal + self.shot = shot + self.zdata = -1 + self.xdata = -1 + self.ydata = -1 + self.zunits = '' + self.xunits = '' + self.yunits = '' + self.rank = -1 + self.connection = connection - ## Retrieve Data - t0 = time.time() - self.found = False + # Retrieve Data + # t0 = time.time() + self.found = False - # Create the MDSplus connection (thin) if not passed in - if self.connection is None: - self.connection = MDSplus.Connection('atlas.gat.com') + # Create the MDSplus connection (thin) if not passed in + if self.connection is None: + self.connection = MDSplus.Connection('atlas.gat.com') - # Retrieve data from MDSplus (thin) - if nomds == False: - #first try, retrieve directly from tree and tag - try: - #print 'trying direct using tree and tag' - if tree != None: - tag = self.signal - fstree = tree - else: - tag = self.connection.get('findsig("'+self.signal+'",_fstree)').value - fstree = self.connection.get('_fstree').value + # Retrieve data from MDSplus (thin) + if not nomds: + # first try, retrieve directly from tree and tag + try: + # print('trying direct using tree and tag') + if tree is not None: + tag = self.signal + fstree = tree + else: + tag = self.connection.get( + 'findsig("' + self.signal + '",_fstree)').value + fstree = self.connection.get('_fstree').value - self.connection.openTree(fstree,shot) - self.zdata = self.connection.get('_s = '+tag).data() - self.zunits = self.connection.get('units_of(_s)').data() - self.rank = numpy.ndim(self.zdata) - if self.rank > 1: - self.xdata = self.connection.get('dim_of(_s,1)').data() - self.xunits = self.connection.get('units_of(dim_of(_s,1))').data() - if self.xunits == '' or self.xunits == ' ': - self.xunits = self.connection.get('units(dim_of(_s,1))').data() + self.connection.openTree(fstree, shot) + self.zdata = self.connection.get('_s = ' + tag).data() + self.zunits = self.connection.get('units_of(_s)').data() + self.rank = numpy.ndim(self.zdata) + if self.rank > 1: + self.xdata = self.connection.get('dim_of(_s,1)').data() + self.xunits = self.connection.get( + 'units_of(dim_of(_s,1))').data() + if self.xunits == '' or self.xunits == ' ': + self.xunits = self.connection.get( + 'units(dim_of(_s,1))').data() - self.ydata = self.connection.get('dim_of(_s)').data() - self.yunits = self.connection.get('units_of(dim_of(_s))').data() - if self.yunits == '' or self.yunits == ' ': - self.yunits = self.connection.get('units(dim_of(_s))').data() - else: - self.xdata = self.connection.get('dim_of(_s)').data() - self.xunits = self.connection.get('units_of(dim_of(_s))').data() - if self.xunits == '' or self.xunits == ' ': - self.xunits = self.connection.get('units(dim_of(_s))').data() - #print 'zdata: ' + str(self.zdata) - self.found = True + self.ydata = self.connection.get('dim_of(_s)').data() + self.yunits = self.connection.get( + 'units_of(dim_of(_s))').data() + if self.yunits == '' or self.yunits == ' ': + self.yunits = self.connection.get( + 'units(dim_of(_s))').data() + else: + self.xdata = self.connection.get('dim_of(_s)').data() + self.xunits = self.connection.get( + 'units_of(dim_of(_s))').data() + if self.xunits == '' or self.xunits == ' ': + self.xunits = self.connection.get( + 'units(dim_of(_s))').data() + # print('zdata: ' + str(self.zdata)) + self.found = True - # MDSplus seems to return 2-D arrays transposed. Change them back. - if numpy.ndim(self.zdata) == 2: self.zdata = numpy.transpose(self.zdata) - if numpy.ndim(self.ydata) == 2: self.ydata = numpy.transpose(self.ydata) - if numpy.ndim(self.xdata) == 2: self.xdata = numpy.transpose(self.xdata) + # MDSplus seems to return 2-D arrays transposed. Change them + # back. + if numpy.ndim(self.zdata) == 2: + self.zdata = numpy.transpose(self.zdata) + if numpy.ndim(self.ydata) == 2: + self.ydata = numpy.transpose(self.ydata) + if numpy.ndim(self.xdata) == 2: + self.xdata = numpy.transpose(self.xdata) - except Exception as e: - pass + except Exception as e: + print(e) + pass - # Retrieve data from PTDATA if node not found - if not self.found: - #print 'Trying ptdata: %s' % (signal,) - self.zdata = self.connection.get('_s = ptdata2("'+signal+'",'+str(shot)+')') - if len(self.zdata) != 1: - self.xdata = self.connection.get('dim_of(_s)') - self.rank = 1 - self.found = True + # Retrieve data from PTDATA if node not found + if not self.found: + # print('Trying ptdata: %s' % (signal,)) + self.zdata = self.connection.get( + '_s = ptdata2("' + signal+'",' + str(shot)+')') + if len(self.zdata) != 1: + self.xdata = self.connection.get('dim_of(_s)') + self.rank = 1 + self.found = True - # Retrieve data from Pseudo-pointname if not in ptdata - if not self.found: - #print ' Signal not in PTDATA: %s' % (signal,) - self.zdata = self.connection.get('_s = pseudo("'+signal+'",'+str(shot)+')') - if len(self.zdata) != 1: - self.xdata = self.connection.get('dim_of(_s)') - self.rank = 1 - self.found = True + # Retrieve data from Pseudo-pointname if not in ptdata + if not self.found: + # print(' Signal not in PTDATA: %s' % (signal,)) + self.zdata = self.connection.get( + '_s = pseudo("' + signal+'",' + str(shot)+')') + if len(self.zdata) != 1: + self.xdata = self.connection.get('dim_of(_s)') + self.rank = 1 + self.found = True - if not self.found: #this means the signal wasn't found - pass + if not self.found: # this means the signal wasn't found + pass - return + return diff --git a/data/get_mdsplus_data.py b/data/get_mdsplus_data.py index 077b0c85..81f14c53 100644 --- a/data/get_mdsplus_data.py +++ b/data/get_mdsplus_data.py @@ -1,19 +1,18 @@ from plasma.utils.downloading import download_all_shot_numbers -from data.signals import * from plasma.conf import conf -prepath = '/p/datad2/' #'/cscratch/share/frnn/'#'/p/datad2/' +prepath = '/p/datad2/' # '/cscratch/share/frnn/' shot_numbers_path = 'shot_lists/' save_path = 'signal_data_new/' -machine = conf['paths']['all_machines'][0]# d3d#jet#d3d #should match with data set from conf.yaml -signals = conf['paths']['all_signals']#all_signals#jet_signals#d3d_signals +# d3d, jet # should match with data set from conf.yaml +machine = conf['paths']['all_machines'][0] +signals = conf['paths']['all_signals'] # jet_signals, d3d_signals print('using signals: ') print(signals) # shot_list_files = plasma.conf.jet_full -#shot_list_files = plasma.conf.d3d_full -shot_list_files = conf['paths']['shot_files'][0]#plasma.conf.d3d_100 - -download_all_shot_numbers(prepath,save_path,shot_list_files,signals) +# shot_list_files = plasma.conf.d3d_full +shot_list_files = conf['paths']['shot_files'][0] # plasma.conf.d3d_100 +download_all_shot_numbers(prepath, save_path, shot_list_files, signals) diff --git a/data/jet_signals.py b/data/jet_signals.py index 8add5273..1efea0e1 100644 --- a/data/jet_signals.py +++ b/data/jet_signals.py @@ -1,90 +1,101 @@ -#JET signal hierarchy -#------------------------------------------------------------------------# -#User only needs to look at 1st and last sections +# JET signal hierarchy +# ------------------------------------------------------------------------ +# User only needs to look at 1st and last sections # - conf.py only needs to import signals_dirs and signals_masks # - get_mdsplus_data.py only needs signals_dirs and download_masks # - performance_analysis_utils.py needs : # - signals_dirs, plot_masks, ppf_labels, jpf_labels -#------------------------------------------------------------------------# +# ------------------------------------------------------------------------ ################ # Signal names # ################ -#This section contains all the exact JET signal strings and their -#groupings by type and dimensionality. -#User should not touch this. Use for reference - -#### 0D current signals #### -da_current = ['c2-ipla'] # Plasma Current [A] -da_lock =['c2-loca'] # Mode Lock Amplitude [A] - -#### Radiation signals #### -#0D -db_out = ['b5r-ptot>out'] #Radiated Power [W] -#1D vertical signals, don't use signal 16 and 23 -db =[] -db += ['b5vr-pbol:{:03d}'.format(i) for i in range(1,28) if (i != 16 and i != 23)] -#1D horizontal signals -db += ['b5hr-pbol:{:03d}'.format(i) for i in range(1,24)] - -#### 1D density signals [m^-2] #### -df = [] -#4 vertical channels and 4 horizontal channels, dont use signal 1 -df += ['g1r-lid:{:03d}'.format(i) for i in range(2,9)] - -#### 0D signals et al #### -gs_inductance = ['bl-liout'] # Radiated Power [W] +# 1D vertical signals, don't use signal 16 and 23 +db = [] +db += ['b5vr-pbol:{:03d}'.format(i) + for i in range(1, 28) if (i != 16 and i != 23)] +# 1D horizontal signals +db += ['b5hr-pbol:{:03d}'.format(i) for i in range(1, 24)] + +#################################### +# 1D density signals [m^-2] # +#################################### +df = [] +# 4 vertical channels and 4 horizontal channels, dont use signal 1 +df += ['g1r-lid:{:03d}'.format(i) for i in range(2, 9)] + +########################### +# 0D signals et al # +########################### +gs_inductance = ['bl-li 1: - xdata = c.get('dim_of(_s,1)').data() - xunits = get_units('dim_of(_s,1)') - ydata = c.get('dim_of(_s)').data() - yunits = get_units('dim_of(_s)') - else: - xdata = c.get('dim_of(_s)').data() - xunits = get_units('dim_of(_s)') - - # MDSplus seems to return 2-D arrays transposed. Change them back. - if np.ndim(data) == 2: data = np.transpose(data) - if np.ndim(ydata) == 2: ydata = np.transpose(ydata) - if np.ndim(xdata) == 2: xdata = np.transpose(xdata) - - # print ' GADATA Retrieval Time : ',time.time() - t0 - xdata = xdata*1e-3#time is measued in ms - return xdata,data,ydata,found - - -def fetch_jet_data(signal_path,shot_num,c): - found = False - time = np.array([0]) - ydata = None - data = np.array([0]) - try: - data = c.get('_sig=jet("{}/",{})'.format(signal_path,shot_num)).data() - if np.ndim(data) == 2: - data = np.transpose(data) - time = c.get('_sig=dim_of(jet("{}/",{}),1)'.format(signal_path,shot_num)).data() - ydata = c.get('_sig=dim_of(jet("{}/",{}),0)'.format(signal_path,shot_num)).data() - else: - time = c.get('_sig=dim_of(jet("{}/",{}))'.format(signal_path,shot_num)).data() - found = True - except Exception as e: - print(e) - sys.stdout.flush() - #pass - return time,data,ydata,found - -def fetch_nstx_data(signal_path,shot_num,c): - tree,tag = get_tree_and_tag(signal_path) - c.openTree(tree,shot_num) - data = c.get(tag).data() - time = c.get('dim_of('+tag+')').data() - found = True - return time,data,None,found - - - - - -d3d = Machine("d3d","atlas.gat.com",fetch_d3d_data,max_cores=32,current_threshold=2e-1) -jet = Machine("jet","mdsplus.jet.efda.org",fetch_jet_data,max_cores=8,current_threshold=1e5) -nstx = Machine("nstx","skylark.pppl.gov:8501::",fetch_nstx_data,max_cores=8) - -all_machines = [d3d,jet] + spl = path.split('/') + tree = spl[0] + tag = spl[1] + return tree, tag + + +def fetch_d3d_data(signal_path, shot, c=None): + tree, signal = get_tree_and_tag_no_backslash(signal_path) + if tree is None: + signal = c.get('findsig("'+signal+'",_fstree)').value + tree = c.get('_fstree').value + # if c is None: + # c = MDSplus.Connection('atlas.gat.com') + + # Retrieve data + found = False + xdata = np.array([0]) + ydata = None + data = np.array([0]) + + # Retrieve data from MDSplus (thin) + # first try, retrieve directly from tree andsignal + def get_units(str): + units = c.get('units_of('+str+')').data() + if units == '' or units == ' ': + units = c.get('units('+str+')').data() + return units + + try: + c.openTree(tree, shot) + data = c.get('_s = '+signal).data() + # data_units = c.get('units_of(_s)').data() + rank = np.ndim(data) + found = True + + except Exception as e: + print(e) + sys.stdout.flush() + pass + + # Retrieve data from PTDATA if node not found + if not found: + # print("not in full path {}".format(signal)) + data = c.get('_s = ptdata2("'+signal+'",'+str(shot)+')').data() + if len(data) != 1: + rank = np.ndim(data) + found = True + # Retrieve data from Pseudo-pointname if not in ptdata + if not found: + # print("not in PTDATA {}".format(signal)) + data = c.get('_s = pseudo("'+signal+'",'+str(shot)+')').data() + if len(data) != 1: + rank = np.ndim(data) + found = True + # this means the signal wasn't found + if not found: + print("No such signal: {}".format(signal)) + pass + + # get time base + if found: + if rank > 1: + xdata = c.get('dim_of(_s,1)').data() + # xunits = get_units('dim_of(_s,1)') + ydata = c.get('dim_of(_s)').data() + # yunits = get_units('dim_of(_s)') + else: + xdata = c.get('dim_of(_s)').data() + # xunits = get_units('dim_of(_s)') + + # MDSplus seems to return 2-D arrays transposed. Change them back. + if np.ndim(data) == 2: + data = np.transpose(data) + if np.ndim(ydata) == 2: + ydata = np.transpose(ydata) + if np.ndim(xdata) == 2: + xdata = np.transpose(xdata) + + # print(' GADATA Retrieval Time : ', time.time() - t0) + xdata = xdata*1e-3 # time is measued in ms + return xdata, data, ydata, found + + +def fetch_jet_data(signal_path, shot_num, c): + found = False + time = np.array([0]) + ydata = None + data = np.array([0]) + try: + data = c.get('_sig=jet("{}/",{})'.format(signal_path, shot_num)).data() + if np.ndim(data) == 2: + data = np.transpose(data) + time = c.get( + '_sig=dim_of(jet("{}/",{}),1)'.format( + signal_path, shot_num)).data() + ydata = c.get( + '_sig=dim_of(jet("{}/",{}),0)'.format( + signal_path, shot_num)).data() + else: + time = c.get( + '_sig=dim_of(jet("{}/",{}))'.format( + signal_path, shot_num)).data() + found = True + except Exception as e: + print(e) + sys.stdout.flush() + # pass + return time, data, ydata, found + + +def fetch_nstx_data(signal_path, shot_num, c): + tree, tag = get_tree_and_tag(signal_path) + c.openTree(tree, shot_num) + data = c.get(tag).data() + time = c.get('dim_of(' + tag + ')').data() + found = True + return time, data, None, found + + +d3d = Machine( + "d3d", + "atlas.gat.com", + fetch_d3d_data, + max_cores=32, + current_threshold=2e-1) +jet = Machine( + "jet", + "mdsplus.jet.efda.org", + fetch_jet_data, + max_cores=8, + current_threshold=1e5) +nstx = Machine("nstx", "skylark.pppl.gov:8501::", fetch_nstx_data, max_cores=8) + +all_machines = [d3d, jet] profile_num_channels = 64 -#ZIPFIT comes from actual measurements -#etemp_profile = ProfileSignal("Electron temperature profile",["ppf/hrts/te","ZIPFIT01/PROFILES.ETEMPFIT"],[jet,d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) -#edens_profile = ProfileSignal("Electron density profile",["ppf/hrts/ne","ZIPFIT01/PROFILES.EDENSFIT"],[jet,d3d],mapping_paths=["ppf/hrts/rho",None],causal_shifts=[0,10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.05,0.02]) - -etemp_profile = ProfileSignal("Electron temperature profile",["ZIPFIT01/PROFILES.ETEMPFIT"],[d3d],mapping_paths=[None],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) -edens_profile = ProfileSignal("Electron density profile",["ZIPFIT01/PROFILES.EDENSFIT"],[d3d],mapping_paths=[None],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) -itemp_profile = ProfileSignal("Ion temperature profile",["ZIPFIT01/PROFILES.ITEMPFIT"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) -zdens_profile = ProfileSignal("Impurity density profile",["ZIPFIT01/PROFILES.ZDENSFIT"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) -trot_profile = ProfileSignal("Rotation profile",["ZIPFIT01/PROFILES.TROTFIT"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) -pthm_profile = ProfileSignal("Thermal pressure profile",["ZIPFIT01/PROFILES.PTHMFIT"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02])# thermal pressure doesn't include fast ions -neut_profile = ProfileSignal("Neutrals profile",["ZIPFIT01/PROFILES.NEUTFIT"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) - -q_profile = ProfileSignal("Q profile",["ZIPFIT01/PROFILES.BOOTSTRAP.QRHO"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02])#compare to just q95 -bootstrap_current_profile = ProfileSignal("Rotation profile",["ZIPFIT01/PROFILES.BOOTSTRAP.JBS_SAUTER"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) - -#equilibrium_image = 2DSignal("2D Magnetic Equilibrium",["EFIT01/RESULTS.GEQDSK.PSIRZ"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) - -#EFIT is the inverse problem from external magnetic measurements -#pressure_profile = ProfileSignal("Pressure profile",["EFIT01/RESULTS.GEQDSK.PRES"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02])# pressure might be unphysical since it is not constrained by measurements, only the EFIT which does not know about density and temperature -q_psi_profile = ProfileSignal("Q(psi) profile",["EFIT01/RESULTS.GEQDSK.QPSI"],[d3d],causal_shifts=[10],mapping_range=(0,1),num_channels=profile_num_channels,data_avail_tolerances=[0.02]) - - -# epress_profile_spatial = ProfileSignal("Electron pressure profile",["ppf/hrts/pe/"],[jet],causal_shifts=[25],mapping_range=(2,4),num_channels=profile_num_channels) -etemp_profile_spatial = ProfileSignal("Electron temperature profile",["ppf/hrts/te"],[jet],causal_shifts=[25],mapping_range=(2,4),num_channels=profile_num_channels,data_avail_tolerances=[0.05]) -edens_profile_spatial = ProfileSignal("Electron density profile",["ppf/hrts/ne"],[jet],causal_shifts=[25],mapping_range=(2,4),num_channels=profile_num_channels,data_avail_tolerances=[0.05]) -rho_profile_spatial = ProfileSignal("Rho at spatial positions",["ppf/hrts/rho"],[jet],causal_shifts=[25],mapping_range=(2,4),num_channels=profile_num_channels,data_avail_tolerances=[0.05]) - -etemp = Signal("electron temperature",["ppf/hrtx/te0"],[jet],causal_shifts=[25],data_avail_tolerances=[0.05]) -# epress = Signal("electron pressure",["ppf/hrtx/pe0/"],[jet],causal_shifts=[25]) - -q95 = Signal("q95 safety factor",['ppf/efit/q95',"EFIT01/RESULTS.AEQDSK.Q95"],[jet,d3d],causal_shifts=[15,10],normalize=False,data_avail_tolerances=[0.03,0.02]) - -ip = Signal("plasma current",["jpf/da/c2-ipla","d3d/ipspr15V"],[jet,d3d],is_ip=True) #"d3d/ipsip" was used before, ipspr15V seems to be available for a superset of shots. -iptarget = Signal("plasma current target",["d3d/ipsiptargt"],[d3d]) -iperr = Signal("plasma current error",["d3d/ipeecoil"],[d3d]) - -li = Signal("internal inductance",["jpf/gs/bl-liout'],[jet]) -#pradcore = ChannelSignal("Radiated Power Core",[ 'd3d/'+r'\bol_l15_p'],[d3d]) -#pradedge = ChannelSignal("Radiated Power Edge",['d3d/'+r'\bol_l03_p'],[d3d]) -pradcore = ChannelSignal("Radiated Power Core",['ppf/bolo/kb5h/channel14', 'd3d/'+r'\bol_l15_p'],[jet,d3d]) -pradedge = ChannelSignal("Radiated Power Edge",['ppf/bolo/kb5h/channel10','d3d/'+r'\bol_l03_p'],[jet,d3d]) -# pechin = Signal("ECH input power, not always on",['d3d/pcechpwrf'],[d3d]) -pechin = Signal("ECH input power, not always on",['RF/ECH.TOTAL.ECHPWRC'],[d3d]) - -#betan = Signal("Normalized Beta",['jpf/gs/bl-bndiaout'], [jet]) +# pradcore = ChannelSignal("Radiated Power Core", [ 'd3d/' + r'\bol_l15_p'] +# ,[d3d]) +# pradedge = ChannelSignal("Radiated Power Edge", ['d3d/' + r'\bol_l03_p'], +# [d3d]) +pradcore = ChannelSignal("Radiated Power Core", + ['ppf/bolo/kb5h/channel14', 'd3d/' + r'\bol_l15_p'], + [jet, d3d]) +pradedge = ChannelSignal("Radiated Power Edge", + ['ppf/bolo/kb5h/channel10', 'd3d/' + r'\bol_l03_p'], + [jet, d3d]) +# pechin = Signal("ECH input power, not always on", ['d3d/pcechpwrf'], [d3d]) +pechin = Signal("ECH input power, not always on", + ['RF/ECH.TOTAL.ECHPWRC'], [d3d]) + +# betan = Signal("Normalized Beta", ['jpf/gs/bl-bndia 1) } - -d3d_signals = {sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if sig.is_defined_on_machine(d3d)} -d3d_signals_0D = {sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if (sig.is_defined_on_machine(d3d) and sig.num_channels == 1)} -d3d_signals_1D = {sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if (sig.is_defined_on_machine(d3d) and sig.num_channels > 1)} - -jet_signals = {sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if sig.is_defined_on_machine(jet)} -jet_signals_0D = {sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if (sig.is_defined_on_machine(jet) and sig.num_channels == 1)} -jet_signals_1D = {sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if (sig.is_defined_on_machine(jet) and sig.num_channels > 1)} - -#['pcechpwrf'] #Total ECH Power Not always on! -### 0D EFIT signals ### -#signal_paths += ['EFIT02/RESULTS.AEQDSK.Q95'] - -### 1D EFIT signals ### -#the other signals give more reliable data -#signal_paths += [ -#'AOT/EQU.t_e', #electron temperature profile vs rho (uniform mapping over time) -#'AOT/EQU.dens_e'] #electron density profile vs rho (uniform mapping over time) +fully_defined_signals = { + sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if ( + sig.is_defined_on_machines(all_machines)) +} +fully_defined_signals_0D = { + sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if ( + sig.is_defined_on_machines(all_machines) and sig.num_channels == 1) +} +fully_defined_signals_1D = { + sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if ( + sig.is_defined_on_machines(all_machines) and sig.num_channels > 1) +} +d3d_signals = { + sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if ( + sig.is_defined_on_machine(d3d)) +} +d3d_signals_0D = { + sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if ( + (sig.is_defined_on_machine(d3d) and sig.num_channels == 1)) +} +d3d_signals_1D = { + sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if ( + (sig.is_defined_on_machine(d3d) and sig.num_channels > 1)) +} + +jet_signals = { + sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if ( + sig.is_defined_on_machine(jet)) +} +jet_signals_0D = { + sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if ( + (sig.is_defined_on_machine(jet) and sig.num_channels == 1)) +} +jet_signals_1D = { + sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if ( + (sig.is_defined_on_machine(jet) and sig.num_channels > 1)) +} + +# ['pcechpwrf'] #Total ECH Power Not always on! +# ## 0D EFIT signals ### +# signal_paths += ['EFIT02/RESULTS.AEQDSK.Q95'] + +# ## 1D EFIT signals ### +# the other signals give more reliable data +# signal_paths += [ +# # Note, the following signals are uniformly mapped over time +# 'AOT/EQU.t_e', # electron temperature profile vs rho +# 'AOT/EQU.dens_e'] # electron density profile vs rho # [[' $I_{plasma}$ [A]'], -#[' Mode L. A. [A]'], -#[' $P_{radiated}$ [W]'], -#[' $P_{radiated}$ [W]'], -#[' $\rho_{plasma}$ [m^-2]'], -#[' $L_{plasma,internal}$'], -#['$\frac{d}{dt} E_{D}$ [W]'], -#[' $P_{input}$ [W]'], -#['$E_{D}$'], -##ppf signal labels -#['ECE unit?']] +# [' Mode L. A. [A]'], +# [' $P_{radiated}$ [W]'], +# [' $P_{radiated}$ [W]'], +# [' $\rho_{plasma}$ [m^-2]'], +# [' $L_{plasma,internal}$'], +# ['$\frac{d}{dt} E_{D}$ [W]'], +# [' $P_{input}$ [W]'], +# ['$E_{D}$'], +# ppf signal labels +# ['ECE unit?']] diff --git a/plasma/conf.py b/plasma/conf.py index 26ed01b8..84b6ce49 100644 --- a/plasma/conf.py +++ b/plasma/conf.py @@ -2,13 +2,16 @@ import os import errno -if os.path.exists(os.path.join(os.path.abspath(os.path.dirname(__file__)), '../examples/conf.yaml')): - conf = parameters(os.path.join(os.path.abspath(os.path.dirname(__file__)), '../examples/conf.yaml')) +if os.path.exists(os.path.join(os.path.abspath(os.path.dirname(__file__)), + '../examples/conf.yaml')): + conf = parameters(os.path.join(os.path.abspath(os.path.dirname(__file__)), + '../examples/conf.yaml')) elif os.path.exists('./conf.yaml'): conf = parameters('./conf.yaml') elif os.path.exists('./examples/conf.yaml'): conf = parameters('./examples/conf.yaml') elif os.path.exists('../examples/conf.yaml'): - conf = parameters('../examples/conf.yaml') + conf = parameters('../examples/conf.yaml') else: - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), 'conf.yaml') + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), + 'conf.yaml') diff --git a/plasma/version.py b/plasma/version.py index 758ad1a2..b7c9168c 100644 --- a/plasma/version.py +++ b/plasma/version.py @@ -8,9 +8,11 @@ specification = ".".join(version_info[:2]) + def compatible(serializedVersion): selfMajor, selfMinor = map(int, version_info[:2]) - otherMajor, otherMinor = map(int, re.split(r"[-\.]", serializedVersion)[:2]) + otherMajor, otherMinor = map(int, + re.split(r"[-\.]", serializedVersion)[:2]) if selfMajor >= otherMajor: return True elif selfMinor >= otherMinor: From f23865aed4d4ad44a0723e229e73adc16fb076cc Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 24 Sep 2019 11:38:37 -0500 Subject: [PATCH 087/272] Use qualified import for data.signals module in conf_parser.py --- plasma/conf_parser.py | 337 +++++++++++++++++++++++++++++------------- 1 file changed, 234 insertions(+), 103 deletions(-) diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index 97ec81ae..050d48cc 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -1,47 +1,62 @@ from plasma.primitives.shots import ShotListFiles -from data.signals import * - +import data.signals as sig +# from data.signals import ( +# all_signals, fully_defined_signals_1D, +# jet, d3d) # nstx import getpass -import uuid import yaml import hashlib + def parameters(input_file): """Parse yaml file of configuration parameters.""" - from plasma.models.targets import HingeTarget, MaxHingeTarget, BinaryTarget, TTDTarget, TTDInvTarget, TTDLinearTarget - + from plasma.models.targets import ( + HingeTarget, MaxHingeTarget, BinaryTarget, + TTDTarget, TTDInvTarget, TTDLinearTarget + ) with open(input_file, 'r') as yaml_file: params = yaml.load(yaml_file) - params['user_name'] = getpass.getuser() output_path = params['fs_path'] + "/" + params['user_name'] base_path = output_path params['paths']['base_path'] = base_path - params['paths']['signal_prepath'] = base_path + params['paths']['signal_prepath'] - params['paths']['shot_list_dir'] = base_path + params['paths']['shot_list_dir'] + params['paths']['signal_prepath'] = base_path + \ + params['paths']['signal_prepath'] + params['paths']['shot_list_dir'] = base_path + \ + params['paths']['shot_list_dir'] params['paths']['output_path'] = output_path - h = get_unique_signal_hash(all_signals.values()) - params['paths']['global_normalizer_path'] = output_path + '/normalization/normalization_signal_group_{}.npz'.format(h) + h = get_unique_signal_hash(sig.all_signals.values()) + params['paths']['global_normalizer_path'] = output_path + \ + '/normalization/normalization_signal_group_{}.npz'.format(h) if params['training']['hyperparam_tuning']: - # params['paths']['saved_shotlist_path'] = './normalization/shot_lists.npz' - params['paths']['normalizer_path'] = './normalization/normalization_signal_group_{}.npz'.format(h) + # params['paths']['saved_shotlist_path'] = + # './normalization/shot_lists.npz' + params['paths']['normalizer_path'] = ( + './normalization/normalization_signal_group_{}.npz'.format(h)) params['paths']['model_save_path'] = './model_checkpoints/' params['paths']['csvlog_save_path'] = './csv_logs/' params['paths']['results_prepath'] = './results/' else: - # params['paths']['saved_shotlist_path'] = output_path +'/normalization/shot_lists.npz' - params['paths']['normalizer_path'] = params['paths']['global_normalizer_path'] - params['paths']['model_save_path'] = output_path + '/model_checkpoints/' + # params['paths']['saved_shotlist_path'] = output_path + + # '/normalization/shot_lists.npz' + params['paths']['normalizer_path'] = ( + params['paths']['global_normalizer_path']) + params['paths']['model_save_path'] = ( + output_path + '/model_checkpoints/') params['paths']['csvlog_save_path'] = output_path + '/csv_logs/' params['paths']['results_prepath'] = output_path + '/results/' - params['paths']['tensorboard_save_path'] = output_path + params['paths']['tensorboard_save_path'] - params['paths']['saved_shotlist_path'] = params['paths']['base_path'] + '/processed_shotlists/' + params['paths']['data'] + '/shot_lists_signal_group_{}.npz'.format(h) - params['paths']['processed_prepath'] = output_path +'/processed_shots/' + 'signal_group_{}/'.format(h) - - #ensure shallow model has +1 -1 target. + params['paths']['tensorboard_save_path'] = output_path + \ + params['paths']['tensorboard_save_path'] + params['paths']['saved_shotlist_path'] = ( + params['paths']['base_path'] + '/processed_shotlists/' + + params['paths']['data'] + + '/shot_lists_signal_group_{}.npz'.format(h)) + params['paths']['processed_prepath'] = ( + output_path + '/processed_shots/' + 'signal_group_{}/'.format(h)) + # ensure shallow model has +1 -1 target. if params['model']['shallow'] or params['target'] == 'hinge': params['data']['target'] = HingeTarget elif params['target'] == 'maxhinge': @@ -59,163 +74,279 @@ def parameters(input_file): print('Unkown type of target. Exiting') exit(1) - #params['model']['output_activation'] = params['data']['target'].activation - #binary crossentropy performs slightly better? - #params['model']['loss'] = params['data']['target'].loss - - #signals - params['paths']['all_signals_dict'] = all_signals - #assert order q95,li,ip,lm,betan,energy,dens,pradcore,pradedge,pin,pechin,torquein,ipdirect,etemp_profile,edens_profile + # params['model']['output_activation'] = + # params['data']['target'].activation + # binary crossentropy performs slightly better? + # params['model']['loss'] = params['data']['target'].loss + + # signals + params['paths']['all_signals_dict'] = sig.all_signals + # assert order + # q95, li, ip, lm, betan, energy, dens, pradcore, pradedge, pin, + # pechin, torquein, ipdirect, etemp_profile, edens_profile + + # shot lists + jet_carbon_wall = ShotListFiles( + sig.jet, params['paths']['shot_list_dir'], + ['CWall_clear.txt', 'CFC_unint.txt'], 'jet carbon wall data') + jet_iterlike_wall = ShotListFiles( + sig.jet, params['paths']['shot_list_dir'], + ['ILW_unint.txt', 'BeWall_clear.txt'], 'jet iter like wall data') + + jenkins_jet_carbon_wall = ShotListFiles( + sig.jet, params['paths']['shot_list_dir'], + ['jenkins_CWall_clear.txt', 'jenkins_CFC_unint.txt'], + 'Subset of jet carbon wall data for Jenkins tests') + jenkins_jet_iterlike_wall = ShotListFiles( + sig.jet, params['paths']['shot_list_dir'], + ['jenkins_ILW_unint.txt', 'jenkins_BeWall_clear.txt'], + 'Subset of jet iter like wall data for Jenkins tests') + + jet_full = ShotListFiles( + sig.jet, params['paths']['shot_list_dir'], + ['ILW_unint.txt', 'BeWall_clear.txt', 'CWall_clear.txt', + 'CFC_unint.txt'], 'jet full data') + + # d3d_10000 = ShotListFiles( + # sig.d3d, params['paths']['shot_list_dir'], + # ['d3d_clear_10000.txt', 'd3d_disrupt_10000.txt'], + # 'd3d data 10000 ND and D shots') + # d3d_1000 = ShotListFiles( + # sig.d3d, params['paths']['shot_list_dir'], + # ['d3d_clear_1000.txt', 'd3d_disrupt_1000.txt'], + # 'd3d data 1000 ND and D shots') + # d3d_100 = ShotListFiles( + # sig.d3d, params['paths']['shot_list_dir'], + # ['d3d_clear_100.txt', 'd3d_disrupt_100.txt'], + # 'd3d data 100 ND and D shots') + d3d_full = ShotListFiles( + sig.d3d, params['paths']['shot_list_dir'], + ['d3d_clear_data_avail.txt', 'd3d_disrupt_data_avail.txt'], + 'd3d data since shot 125500') + d3d_jenkins = ShotListFiles( + sig.d3d, params['paths']['shot_list_dir'], + ['jenkins_d3d_clear.txt', 'jenkins_d3d_disrupt.txt'], + 'Subset of d3d data for Jenkins test') + # d3d_jb_full = ShotListFiles( + # sig.d3d, params['paths']['shot_list_dir'], + # ['shotlist_JaysonBarr_clear.txt', + # 'shotlist_JaysonBarr_disrupt.txt'], + # 'd3d shots since 160000-170000') + + # nstx_full = ShotListFiles( + # nstx, params['paths']['shot_list_dir'], + # ['disrupt_nstx.txt'], 'nstx shots (all are disruptive') - #shot lists - jet_carbon_wall = ShotListFiles(jet,params['paths']['shot_list_dir'],['CWall_clear.txt','CFC_unint.txt'],'jet carbon wall data') - jet_iterlike_wall = ShotListFiles(jet,params['paths']['shot_list_dir'],['ILW_unint.txt','BeWall_clear.txt'],'jet iter like wall data') - - jenkins_jet_carbon_wall = ShotListFiles(jet,params['paths']['shot_list_dir'],['jenkins_CWall_clear.txt','jenkins_CFC_unint.txt'],'Subset of jet carbon wall data for Jenkins tests') - jenkins_jet_iterlike_wall = ShotListFiles(jet,params['paths']['shot_list_dir'],['jenkins_ILW_unint.txt','jenkins_BeWall_clear.txt'],'Subset of jet iter like wall data for Jenkins tests') - - jet_full = ShotListFiles(jet,params['paths']['shot_list_dir'],['ILW_unint.txt','BeWall_clear.txt','CWall_clear.txt','CFC_unint.txt'],'jet full data') - - d3d_10000 = ShotListFiles(d3d,params['paths']['shot_list_dir'],['d3d_clear_10000.txt','d3d_disrupt_10000.txt'],'d3d data 10000 ND and D shots') - d3d_1000 = ShotListFiles(d3d,params['paths']['shot_list_dir'],['d3d_clear_1000.txt','d3d_disrupt_1000.txt'],'d3d data 1000 ND and D shots') - d3d_100 = ShotListFiles(d3d,params['paths']['shot_list_dir'],['d3d_clear_100.txt','d3d_disrupt_100.txt'],'d3d data 100 ND and D shots') - d3d_full = ShotListFiles(d3d,params['paths']['shot_list_dir'],['d3d_clear_data_avail.txt','d3d_disrupt_data_avail.txt'],'d3d data since shot 125500') - d3d_jenkins = ShotListFiles(d3d,params['paths']['shot_list_dir'],['jenkins_d3d_clear.txt','jenkins_d3d_disrupt.txt'],'Subset of d3d data for Jenkins test') - d3d_jb_full = ShotListFiles(d3d,params['paths']['shot_list_dir'],['shotlist_JaysonBarr_clear.txt','shotlist_JaysonBarr_disrupt.txt'],'d3d shots since 160000-170000') - - nstx_full = ShotListFiles(nstx,params['paths']['shot_list_dir'],['disrupt_nstx.txt'],'nstx shots (all are disruptive') - if params['paths']['data'] == 'jet_data': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] - params['paths']['use_signals_dict'] = jet_signals + params['paths']['use_signals_dict'] = sig.jet_signals elif params['paths']['data'] == 'jet_data_0D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] - params['paths']['use_signals_dict'] = jet_signals_0D + params['paths']['use_signals_dict'] = sig.jet_signals_0D elif params['paths']['data'] == 'jet_data_1D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] - params['paths']['use_signals_dict'] = jet_signals_1D + params['paths']['use_signals_dict'] = sig.jet_signals_1D elif params['paths']['data'] == 'jet_carbon_data': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [] - params['paths']['use_signals_dict'] = jet_signals + params['paths']['use_signals_dict'] = sig.jet_signals elif params['paths']['data'] == 'jet_mixed_data': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [] - params['paths']['use_signals_dict'] = jet_signals + params['paths']['use_signals_dict'] = sig.jet_signals elif params['paths']['data'] == 'jenkins_jet': params['paths']['shot_files'] = [jenkins_jet_carbon_wall] params['paths']['shot_files_test'] = [jenkins_jet_iterlike_wall] - params['paths']['use_signals_dict'] = jet_signals - elif params['paths']['data'] == 'jet_data_fully_defined': #jet data but with fully defined signals + params['paths']['use_signals_dict'] = sig.jet_signals + # jet data but with fully defined signals + elif params['paths']['data'] == 'jet_data_fully_defined': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] - params['paths']['use_signals_dict'] = fully_defined_signals - elif params['paths']['data'] == 'jet_data_fully_defined_0D': #jet data but with fully defined signals + params['paths']['use_signals_dict'] = sig.fully_defined_signals + # jet data but with fully defined signals + elif params['paths']['data'] == 'jet_data_fully_defined_0D': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] - params['paths']['use_signals_dict'] = fully_defined_signals_0D - - + params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D elif params['paths']['data'] == 'd3d_data': params['paths']['shot_files'] = [d3d_full] - params['paths']['shot_files_test'] = [] - params['paths']['use_signals_dict'] = {'q95':q95,'li':li,'ip':ip,'lm':lm,'betan':betan,'energy':energy,'dens':dens,'pradcore':pradcore,'pradedge':pradedge,'pin':pin,'torquein':torquein,'ipdirect':ipdirect,'iptarget':iptarget,'iperr':iperr, -'etemp_profile':etemp_profile ,'edens_profile':edens_profile} + params['paths']['shot_files_test'] = [] + params['paths']['use_signals_dict'] = { + 'q95': sig.q95, + 'li': sig.li, + 'ip': sig.ip, + 'lm': sig.lm, + 'betan': sig.betan, + 'energy': sig.energy, + 'dens': sig.dens, + 'pradcore': sig.pradcore, + 'pradedge': sig.pradedge, + 'pin': sig.pin, + 'torquein': sig.torquein, + 'ipdirect': sig.ipdirect, + 'iptarget': sig.iptarget, + 'iperr': sig.iperr, + 'etemp_profile': sig.etemp_profile, + 'edens_profile': sig.edens_profile, + } elif params['paths']['data'] == 'd3d_data_1D': params['paths']['shot_files'] = [d3d_full] - params['paths']['shot_files_test'] = [] - params['paths']['use_signals_dict'] = {'ipdirect':ipdirect,'etemp_profile':etemp_profile ,'edens_profile':edens_profile} + params['paths']['shot_files_test'] = [] + params['paths']['use_signals_dict'] = { + 'ipdirect': sig.ipdirect, + 'etemp_profile': sig.etemp_profile, + 'edens_profile': sig.edens_profile, + } elif params['paths']['data'] == 'd3d_data_all_profiles': params['paths']['shot_files'] = [d3d_full] - params['paths']['shot_files_test'] = [] - params['paths']['use_signals_dict'] = {'ipdirect':ipdirect,'etemp_profile':etemp_profile ,'edens_profile':edens_profile,'itemp_profile':itemp_profile,'zdens_profile':zdens_profile,'trot_profile':trot_profile,'pthm_profile':pthm_profile,'neut_profile':neut_profile,'q_profile':q_profile,'bootstrap_current_profile':bootstrap_current_profile,'q_psi_profile':q_psi_profile} + params['paths']['shot_files_test'] = [] + params['paths']['use_signals_dict'] = { + 'ipdirect': sig.ipdirect, + 'etemp_profile': sig.etemp_profile, + 'edens_profile': sig.edens_profile, + 'itemp_profile': sig.itemp_profile, + 'zdens_profile': sig.zdens_profile, + 'trot_profile': sig.trot_profile, + 'pthm_profile': sig.pthm_profile, + 'neut_profile': sig.neut_profile, + 'q_profile': sig.q_profile, + 'bootstrap_current_profile': sig.bootstrap_current_profile, + 'q_psi_profile': sig.q_psi_profile, + } elif params['paths']['data'] == 'd3d_data_0D': params['paths']['shot_files'] = [d3d_full] - params['paths']['shot_files_test'] = [] - params['paths']['use_signals_dict'] = {'q95':q95,'li':li,'ip':ip,'lm':lm,'betan':betan,'energy':energy,'dens':dens,'pradcore':pradcore,'pradedge':pradedge,'pin':pin,'torquein':torquein,'ipdirect':ipdirect,'iptarget':iptarget,'iperr':iperr} + params['paths']['shot_files_test'] = [] + params['paths']['use_signals_dict'] = { + 'q95': sig.q95, + 'li': sig.li, + 'ip': sig.ip, + 'lm': sig.lm, + 'betan': sig.betan, + 'energy': sig.energy, + 'dens': sig.dens, + 'pradcore': sig.pradcore, + 'pradedge': sig.pradedge, + 'pin': sig.pin, + 'torquein': sig.torquein, + 'ipdirect': sig.ipdirect, + 'iptarget': sig.iptarget, + 'iperr': sig.iperr, + } elif params['paths']['data'] == 'd3d_data_all': params['paths']['shot_files'] = [d3d_full] - params['paths']['shot_files_test'] = [] - params['paths']['use_signals_dict'] = d3d_signals + params['paths']['shot_files_test'] = [] + params['paths']['use_signals_dict'] = sig.d3d_signals elif params['paths']['data'] == 'jenkins_d3d': params['paths']['shot_files'] = [d3d_jenkins] params['paths']['shot_files_test'] = [] - params['paths']['use_signals_dict'] = {'q95':q95,'li':li,'ip':ip,'lm':lm,'betan':betan,'energy':energy,'dens':dens,'pradcore':pradcore,'pradedge':pradedge,'pin':pin,'torquein':torquein,'ipdirect':ipdirect,'iptarget':iptarget,'iperr':iperr, -'etemp_profile':etemp_profile ,'edens_profile':edens_profile} - elif params['paths']['data'] == 'd3d_data_fully_defined': #jet data but with fully defined signals + params['paths']['use_signals_dict'] = { + 'q95': sig.q95, + 'li': sig.li, + 'ip': sig.ip, + 'lm': sig.lm, + 'betan': sig.betan, + 'energy': sig.energy, + 'dens': sig.dens, + 'pradcore': sig.pradcore, + 'pradedge': sig.pradedge, + 'pin': sig.pin, + 'torquein': sig.torquein, + 'ipdirect': sig.ipdirect, + 'iptarget': sig.iptarget, + 'iperr': sig.iperr, + 'etemp_profile': sig.etemp_profile, + 'edens_profile': sig.edens_profile, + } + # jet data but with fully defined signals + elif params['paths']['data'] == 'd3d_data_fully_defined': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] - params['paths']['use_signals_dict'] = fully_defined_signals - elif params['paths']['data'] == 'd3d_data_fully_defined_0D': #jet data but with fully defined signals + params['paths']['use_signals_dict'] = sig.fully_defined_signals + # jet data but with fully defined signals + elif params['paths']['data'] == 'd3d_data_fully_defined_0D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] - params['paths']['use_signals_dict'] = fully_defined_signals_0D + params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D - #cross-machine + # cross-machine elif params['paths']['data'] == 'jet_to_d3d_data': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] - params['paths']['use_signals_dict'] = fully_defined_signals + params['paths']['use_signals_dict'] = sig.fully_defined_signals elif params['paths']['data'] == 'd3d_to_jet_data': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] - params['paths']['use_signals_dict'] = fully_defined_signals + params['paths']['use_signals_dict'] = sig.fully_defined_signals elif params['paths']['data'] == 'jet_to_d3d_data_0D': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] - params['paths']['use_signals_dict'] = fully_defined_signals_0D + params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D elif params['paths']['data'] == 'd3d_to_jet_data_0D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] - params['paths']['use_signals_dict'] = fully_defined_signals_0D + params['paths']['use_signals_dict'] = sig.fully_defined_signals_0D elif params['paths']['data'] == 'jet_to_d3d_data_1D': params['paths']['shot_files'] = [jet_full] params['paths']['shot_files_test'] = [d3d_full] - params['paths']['use_signals_dict'] = fully_defined_signals_1D + params['paths']['use_signals_dict'] = sig.fully_defined_signals_1D elif params['paths']['data'] == 'd3d_to_jet_data_1D': params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [jet_iterlike_wall] - params['paths']['use_signals_dict'] = fully_defined_signals_1D - + params['paths']['use_signals_dict'] = sig.fully_defined_signals_1D - - else: + else: print("Unkown data set {}".format(params['paths']['data'])) exit(1) if len(params['paths']['specific_signals']): - for sig in params['paths']['specific_signals']: - if sig not in params['paths']['use_signals_dict'].keys(): - print("Signal {} is not fully defined for {} machine. Skipping...".format(sig,params['paths']['data'].split("_")[0])) - params['paths']['specific_signals'] = list(filter(lambda x: x in params['paths']['use_signals_dict'].keys(), params['paths']['specific_signals'])) - selected_signals = {k: params['paths']['use_signals_dict'][k] for k in params['paths']['specific_signals']} - params['paths']['use_signals'] = sort_by_channels(list(selected_signals.values())) + for s in params['paths']['specific_signals']: + if s not in params['paths']['use_signals_dict'].keys(): + print("Signal {} is not fully defined for {} machine. ", + "Skipping...".format( + s, params['paths']['data'].split("_")[0])) + params['paths']['specific_signals'] = list( + filter( + lambda x: x in params['paths']['use_signals_dict'].keys(), + params['paths']['specific_signals'])) + selected_signals = {k: params['paths']['use_signals_dict'][k] + for k in params['paths']['specific_signals']} + params['paths']['use_signals'] = sort_by_channels( + list(selected_signals.values())) else: - #default case - params['paths']['use_signals'] = sort_by_channels(list(params['paths']['use_signals_dict'].values())) + # default case + params['paths']['use_signals'] = sort_by_channels( + list(params['paths']['use_signals_dict'].values())) - params['paths']['all_signals'] = sort_by_channels(list(params['paths']['all_signals_dict'].values())) + params['paths']['all_signals'] = sort_by_channels( + list(params['paths']['all_signals_dict'].values())) - print("Selected signals (determines which signals training is run on):\n{}".format(params['paths']['use_signals'])) + print("Selected signals (determines which signals are used for ", + "training):\n{}".format(params['paths']['use_signals'])) - params['paths']['shot_files_all'] = params['paths']['shot_files']+params['paths']['shot_files_test'] - params['paths']['all_machines'] = list(set([file.machine for file in params['paths']['shot_files_all']])) + params['paths']['shot_files_all'] = ( + params['paths']['shot_files'] + params['paths']['shot_files_test']) + params['paths']['all_machines'] = list( + set([file.machine for file in params['paths']['shot_files_all']])) - #type assertations - assert type(params['data']['signal_to_augment']) == str or type(params['data']['signal_to_augment']) == None - assert type(params['data']['augment_during_training']) == bool + # type assertations + assert (isinstance(params['data']['signal_to_augment'], str) + or isinstance(params['data']['signal_to_augment'], None)) + assert isinstance(params['data']['augment_during_training'], bool) return params + def get_unique_signal_hash(signals): - return int(hashlib.md5(''.join(tuple(map(lambda x: x.description, sorted(signals)))).encode('utf-8')).hexdigest(),16) + return int(hashlib.md5(''.join( + tuple(map(lambda x: x.description, sorted(signals)))).encode( + 'utf-8')).hexdigest(), 16) -#make sure 1D signals come last! This is necessary for model builder. -def sort_by_channels(list_of_signals): - return sorted(list_of_signals,key = lambda x: x.num_channels) +def sort_by_channels(list_of_signals): + # make sure 1D signals come last! This is necessary for model builder. + return sorted(list_of_signals, key=lambda x: x.num_channels) From deddb520c861e6693811c90b718fcc843d395113 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 24 Sep 2019 12:49:52 -0500 Subject: [PATCH 088/272] Fix style for files in plasma/models --- plasma/models/builder.py | 331 +++++---- plasma/models/custom_loss.py | 28 +- plasma/models/loader.py | 617 +++++++++------- plasma/models/mpi_runner.py | 1199 +++++++++++++++++-------------- plasma/models/runner.py | 442 +++++++----- plasma/models/shallow_runner.py | 449 +++++++----- plasma/models/targets.py | 138 ++-- 7 files changed, 1849 insertions(+), 1355 deletions(-) diff --git a/plasma/models/builder.py b/plasma/models/builder.py index e1913b85..dc2b75c6 100644 --- a/plasma/models/builder.py +++ b/plasma/models/builder.py @@ -1,28 +1,32 @@ from __future__ import division -import keras from keras.models import Sequential, Model from keras.layers import Input -from keras.layers.core import Dense, Activation, Dropout, Lambda, Reshape, Flatten, Permute, RepeatVector +from keras.layers.core import ( + Dense, Activation, Dropout, Lambda, + Reshape, Flatten, Permute, # RepeatVector + ) from keras.layers import LSTM, SimpleRNN, Bidirectional, BatchNormalization from keras.layers.convolutional import Convolution1D from keras.layers.pooling import MaxPooling1D -from keras.utils.data_utils import get_file +# from keras.utils.data_utils import get_file from keras.layers.wrappers import TimeDistributed from keras.layers.merge import Concatenate from keras.callbacks import Callback -from keras.regularizers import l1,l2,l1_l2 +from keras.regularizers import l2 # l1, l1_l2 import keras.backend as K import dill import re -import os,sys +import os +import sys import numpy as np from copy import deepcopy from plasma.utils.downloading import makedirs_process_safe import hashlib + class LossHistory(Callback): def on_train_begin(self, logs=None): self.losses = [] @@ -32,44 +36,49 @@ def on_batch_end(self, batch, logs=None): class ModelBuilder(object): - def __init__(self,conf): + def __init__(self, conf): self.conf = conf def get_unique_id(self): - num_epochs = self.conf['training']['num_epochs'] + # num_epochs = self.conf['training']['num_epochs'] this_conf = deepcopy(self.conf) - #don't make hash dependent on number of epochs. + # don't make hash dependent on number of epochs. this_conf['training']['num_epochs'] = 0 - unique_id = int(hashlib.md5((dill.dumps(this_conf).decode('unicode_escape')).encode('utf-8')).hexdigest(),16) + unique_id = int(hashlib.md5((dill.dumps(this_conf).decode( + 'unicode_escape')).encode('utf-8')).hexdigest(), 16) return unique_id def get_0D_1D_indices(self): - #make sure all 1D indices are contiguous in the end! + # make sure all 1D indices are contiguous in the end! use_signals = self.conf['paths']['use_signals'] indices_0d = [] indices_1d = [] num_0D = 0 num_1D = 0 curr_idx = 0 - is_1D_region = use_signals[0].num_channels > 1#do we have any 1D indices? + # do we have any 1D indices? + is_1D_region = use_signals[0].num_channels > 1 for sig in use_signals: num_channels = sig.num_channels - indices = range(curr_idx,curr_idx+num_channels) + indices = range(curr_idx, curr_idx+num_channels) if num_channels > 1: indices_1d += indices num_1D += 1 is_1D_region = True else: - assert(not is_1D_region), "make sure all use_signals are ordered such that 1D signals come last!" + assert(not is_1D_region) + # , "make sure all use_signals are ordered such that 1D signals + # come last!" assert(num_channels == 1) indices_0d += indices num_0D += 1 is_1D_region = False curr_idx += num_channels - return np.array(indices_0d).astype(np.int32), np.array(indices_1d).astype(np.int32),num_0D,num_1D - + return np.array(indices_0d).astype( + np.int32), np.array(indices_1d).astype( + np.int32), num_0D, num_1D - def build_model(self,predict,custom_batch_size=None): + def build_model(self, predict, custom_batch_size=None): conf = self.conf model_conf = conf['model'] use_bidirectional = model_conf['use_bidirectional'] @@ -82,25 +91,25 @@ def build_model(self,predict,custom_batch_size=None): dropout_prob = model_conf['dropout_prob'] length = model_conf['length'] pred_length = model_conf['pred_length'] - skip = model_conf['skip'] + # skip = model_conf['skip'] stateful = model_conf['stateful'] return_sequences = model_conf['return_sequences'] - output_activation = conf['data']['target'].activation#model_conf['output_activation'] + # model_conf['output_activation'] + output_activation = conf['data']['target'].activation use_signals = conf['paths']['use_signals'] num_signals = sum([sig.num_channels for sig in use_signals]) num_conv_filters = model_conf['num_conv_filters'] - num_conv_layers = model_conf['num_conv_layers'] + # num_conv_layers = model_conf['num_conv_layers'] size_conv_filters = model_conf['size_conv_filters'] pool_size = model_conf['pool_size'] dense_size = model_conf['dense_size'] - batch_size = self.conf['training']['batch_size'] if predict: batch_size = self.conf['model']['pred_batch_size'] - #so we can predict with one time point at a time! + # so we can predict with one time point at a time! if return_sequences: - length =pred_length + length = pred_length else: length = 1 @@ -110,20 +119,20 @@ def build_model(self,predict,custom_batch_size=None): if rnn_type == 'LSTM': rnn_model = LSTM elif rnn_type == 'SimpleRNN': - rnn_model =SimpleRNN + rnn_model = SimpleRNN else: print('Unkown Model Type, exiting.') exit(1) - - batch_input_shape=(batch_size,length, num_signals) - batch_shape_non_temporal=(batch_size,num_signals) - indices_0d,indices_1d,num_0D,num_1D = self.get_0D_1D_indices() + batch_input_shape = (batch_size, length, num_signals) + # batch_shape_non_temporal = (batch_size, num_signals) + + indices_0d, indices_1d, num_0D, num_1D = self.get_0D_1D_indices() - def slicer(x,indices): - return x[:,indices] + def slicer(x, indices): + return x[:, indices] - def slicer_output_shape(input_shape,indices): + def slicer_output_shape(input_shape, indices): shape_curr = list(input_shape) assert len(shape_curr) == 2 # only valid for 3D tensors shape_curr[-1] = len(indices) @@ -132,112 +141,178 @@ def slicer_output_shape(input_shape,indices): pre_rnn_input = Input(shape=(num_signals,)) if num_1D > 0: - pre_rnn_1D = Lambda(lambda x: x[:,len(indices_0d):],output_shape=(len(indices_1d),))(pre_rnn_input) - pre_rnn_0D = Lambda(lambda x: x[:,:len(indices_0d)],output_shape=(len(indices_0d),))(pre_rnn_input)# slicer(x,indices_0d),lambda s: slicer_output_shape(s,indices_0d))(pre_rnn_input) - pre_rnn_1D = Reshape((num_1D,len(indices_1d)//num_1D)) (pre_rnn_1D) - pre_rnn_1D = Permute((2,1)) (pre_rnn_1D) - + pre_rnn_1D = Lambda(lambda x: x[:, len(indices_0d):], + output_shape=(len(indices_1d),))(pre_rnn_input) + pre_rnn_0D = Lambda(lambda x: x[:, :len(indices_0d)], + output_shape=(len(indices_0d),))(pre_rnn_input) + # slicer(x,indices_0d),lambda s: + # slicer_output_shape(s,indices_0d))(pre_rnn_input) + pre_rnn_1D = Reshape((num_1D, len(indices_1d)//num_1D))(pre_rnn_1D) + pre_rnn_1D = Permute((2, 1))(pre_rnn_1D) + for i in range(model_conf['num_conv_layers']): div_fac = 2**i - '''The first conv layer learns `num_conv_filters//div_fac` filters (aka kernels), - each of size `(size_conv_filters, num1D)``. Its output will have shape - (None, len(indices_1d)//num_1D - size_conv_filters + 1, num_conv_filters//div_fac), - i.e., for each position in the input spatial series (direction along radius), - the activation of each filter at that position.''' + '''The first conv layer learns `num_conv_filters//div_fac` + filters (aka kernels), each of size + `(size_conv_filters, num1D)`. Its output will have shape + (None, len(indices_1d)//num_1D - size_conv_filters + 1, + num_conv_filters//div_fac), i.e., for + each position in the input spatial series (direction along + radius), the activation of each filter at that position. + + ''' '''For i=1 first conv layer would get: - (None, (len(indices_1d)//num_1D - size_conv_filters + 1)/pool_size-size_conv_filters+1,num_conv_filters//div_fac)''' - pre_rnn_1D = Convolution1D(num_conv_filters//div_fac,size_conv_filters,padding='valid') (pre_rnn_1D) - if use_batch_norm: pre_rnn_1D = BatchNormalization()(pre_rnn_1D) + (None, (len(indices_1d)//num_1D - size_conv_filters + + 1)/pool_size-size_conv_filters + 1,num_conv_filters//div_fac) + + ''' + pre_rnn_1D = Convolution1D( + num_conv_filters//div_fac, size_conv_filters, + padding='valid')(pre_rnn_1D) + if use_batch_norm: + pre_rnn_1D = BatchNormalization()(pre_rnn_1D) pre_rnn_1D = Activation('relu')(pre_rnn_1D) - '''The output of the second conv layer will have shape - (None, len(indices_1d)//num_1D - size_conv_filters + 1, num_conv_filters//div_fac), - i.e., for each position in the input spatial series (direction along radius), - the activation of each filter at that position. - - for i=1 second layer would output - (None, (len(indices_1d)//num_1D - size_conv_filters + 1)/pool_size-size_conv_filters+1,num_conv_filters//div_fac)''' - pre_rnn_1D = Convolution1D(num_conv_filters//div_fac,1,padding='valid') (pre_rnn_1D) - if use_batch_norm: pre_rnn_1D = BatchNormalization()(pre_rnn_1D) + '''The output of the second conv layer will have shape + (None, len(indices_1d)//num_1D - size_conv_filters + 1, + num_conv_filters//div_fac), + i.e., for each position in the input spatial series + (direction along radius), the activation of each filter + at that position. + + For i=1, the second layer would output + (None, (len(indices_1d)//num_1D - size_conv_filters + 1)/ + pool_size-size_conv_filters + 1,num_conv_filters//div_fac) + ''' + pre_rnn_1D = Convolution1D( + num_conv_filters//div_fac, 1, padding='valid')(pre_rnn_1D) + if use_batch_norm: + pre_rnn_1D = BatchNormalization()(pre_rnn_1D) pre_rnn_1D = Activation('relu')(pre_rnn_1D) - '''Outputs (None, (len(indices_1d)//num_1D - size_conv_filters + 1)/pool_size, num_conv_filters//div_fac) - - for i=1 pooling layer would output: - (None,((len(indices_1d)//num_1D- size_conv_filters + 1)/pool_size-size_conv_filters+1)/pool_size,num_conv_filters//div_fac)''' - pre_rnn_1D = MaxPooling1D(pool_size) (pre_rnn_1D) - pre_rnn_1D = Flatten() (pre_rnn_1D) - pre_rnn_1D = Dense(dense_size,kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn_1D) - if use_batch_norm: pre_rnn_1D = BatchNormalization()(pre_rnn_1D) + '''Outputs (None, (len(indices_1d)//num_1D - size_conv_filters + + 1)/pool_size, num_conv_filters//div_fac) + + For i=1, the pooling layer would output: + (None,((len(indices_1d)//num_1D- size_conv_filters + + 1)/pool_size-size_conv_filters+1)/pool_size, + num_conv_filters//div_fac) + + ''' + pre_rnn_1D = MaxPooling1D(pool_size)(pre_rnn_1D) + pre_rnn_1D = Flatten()(pre_rnn_1D) + pre_rnn_1D = Dense( + dense_size, + kernel_regularizer=l2(dense_regularization), + bias_regularizer=l2(dense_regularization), + activity_regularizer=l2(dense_regularization))(pre_rnn_1D) + if use_batch_norm: + pre_rnn_1D = BatchNormalization()(pre_rnn_1D) pre_rnn_1D = Activation('relu')(pre_rnn_1D) - pre_rnn_1D = Dense(dense_size//4,kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn_1D) - if use_batch_norm: pre_rnn_1D = BatchNormalization()(pre_rnn_1D) + pre_rnn_1D = Dense( + dense_size//4, + kernel_regularizer=l2(dense_regularization), + bias_regularizer=l2(dense_regularization), + activity_regularizer=l2(dense_regularization))(pre_rnn_1D) + if use_batch_norm: + pre_rnn_1D = BatchNormalization()(pre_rnn_1D) pre_rnn_1D = Activation('relu')(pre_rnn_1D) - pre_rnn = Concatenate() ([pre_rnn_0D,pre_rnn_1D]) + pre_rnn = Concatenate()([pre_rnn_0D, pre_rnn_1D]) else: - pre_rnn = pre_rnn_input + pre_rnn = pre_rnn_input if model_conf['rnn_layers'] == 0 or model_conf['extra_dense_input']: - pre_rnn = Dense(dense_size,activation='relu',kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn) - pre_rnn = Dense(dense_size//2,activation='relu',kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn) - pre_rnn = Dense(dense_size//4,activation='relu',kernel_regularizer=l2(dense_regularization),bias_regularizer=l2(dense_regularization),activity_regularizer=l2(dense_regularization)) (pre_rnn) - - pre_rnn_model = Model(inputs = pre_rnn_input,outputs=pre_rnn) - #pre_rnn_model.summary() - x_input = Input(batch_shape = batch_input_shape) - x_in = TimeDistributed(pre_rnn_model) (x_input) + pre_rnn = Dense( + dense_size, + activation='relu', + kernel_regularizer=l2(dense_regularization), + bias_regularizer=l2(dense_regularization), + activity_regularizer=l2(dense_regularization))(pre_rnn) + pre_rnn = Dense( + dense_size//2, + activation='relu', + kernel_regularizer=l2(dense_regularization), + bias_regularizer=l2(dense_regularization), + activity_regularizer=l2(dense_regularization))(pre_rnn) + pre_rnn = Dense( + dense_size//4, + activation='relu', + kernel_regularizer=l2(dense_regularization), + bias_regularizer=l2(dense_regularization), + activity_regularizer=l2(dense_regularization))(pre_rnn) + + pre_rnn_model = Model(inputs=pre_rnn_input, outputs=pre_rnn) + # pre_rnn_model.summary() + x_input = Input(batch_shape=batch_input_shape) + x_in = TimeDistributed(pre_rnn_model)(x_input) if use_bidirectional: for _ in range(model_conf['rnn_layers']): - x_in = Bidirectional(rnn_model(rnn_size, return_sequences=return_sequences, - stateful=stateful,kernel_regularizer=l2(regularization),recurrent_regularizer=l2(regularization), - bias_regularizer=l2(regularization),dropout=dropout_prob,recurrent_dropout=dropout_prob)) (x_in) - x_in = Dropout(dropout_prob) (x_in) + x_in = Bidirectional( + rnn_model( + rnn_size, + return_sequences=return_sequences, + stateful=stateful, + kernel_regularizer=l2(regularization), + recurrent_regularizer=l2(regularization), + bias_regularizer=l2(regularization), + dropout=dropout_prob, + recurrent_dropout=dropout_prob))(x_in) + x_in = Dropout(dropout_prob)(x_in) else: for _ in range(model_conf['rnn_layers']): - x_in = rnn_model(rnn_size, return_sequences=return_sequences,#batch_input_shape=batch_input_shape, - stateful=stateful,kernel_regularizer=l2(regularization),recurrent_regularizer=l2(regularization), - bias_regularizer=l2(regularization),dropout=dropout_prob,recurrent_dropout=dropout_prob) (x_in) - x_in = Dropout(dropout_prob) (x_in) + x_in = rnn_model( + rnn_size, + return_sequences=return_sequences, + # batch_input_shape=batch_input_shape, + stateful=stateful, + kernel_regularizer=l2(regularization), + recurrent_regularizer=l2(regularization), + bias_regularizer=l2(regularization), + dropout=dropout_prob, + recurrent_dropout=dropout_prob)(x_in) + x_in = Dropout(dropout_prob)(x_in) if return_sequences: - #x_out = TimeDistributed(Dense(100,activation='tanh')) (x_in) - x_out = TimeDistributed(Dense(1,activation=output_activation)) (x_in) + # x_out = TimeDistributed(Dense(100,activation='tanh')) (x_in) + x_out = TimeDistributed( + Dense(1, activation=output_activation))(x_in) else: - x_out = Dense(1,activation=output_activation) (x_in) - model = Model(inputs=x_input,outputs=x_out) - #bug with tensorflow/Keras - if conf['model']['backend'] == 'tf' or conf['model']['backend'] == 'tensorflow': - first_time = "tensorflow" not in sys.modules - import tensorflow as tf - if first_time: - K.get_session().run(tf.global_variables_initializer()) + x_out = Dense(1, activation=output_activation)(x_in) + model = Model(inputs=x_input, outputs=x_out) + # bug with tensorflow/Keras + if (conf['model']['backend'] == 'tf' + or conf['model']['backend'] == 'tensorflow'): + first_time = "tensorflow" not in sys.modules + import tensorflow as tf + if first_time: + K.get_session().run(tf.global_variables_initializer()) model.reset_states() return model def build_train_test_models(self): - return self.build_model(False),self.build_model(True) + return self.build_model(False), self.build_model(True) - def save_model_weights(self,model,epoch): + def save_model_weights(self, model, epoch): save_path = self.get_save_path(epoch) - model.save_weights(save_path,overwrite=True) + model.save_weights(save_path, overwrite=True) - def delete_model_weights(self,model,epoch): + def delete_model_weights(self, model, epoch): save_path = self.get_save_path(epoch) assert(os.path.exists(save_path)) os.remove(save_path) - - def get_save_path(self,epoch): + def get_save_path(self, epoch): unique_id = self.get_unique_id() - return self.conf['paths']['model_save_path'] + 'model.{}._epoch_.{}.h5'.format(unique_id,epoch) + return (self.conf['paths']['model_save_path'] + + 'model.{}._epoch_.{}.h5'.format(unique_id, epoch)) def ensure_save_directory(self): prepath = self.conf['paths']['model_save_path'] makedirs_process_safe(prepath) - def load_model_weights(self,model,custom_path=None): - if custom_path == None: + def load_model_weights(self, model, custom_path=None): + if custom_path is None: epochs = self.get_all_saved_files() if len(epochs) == 0: print('no previous checkpoint found') @@ -248,7 +323,8 @@ def load_model_weights(self,model,custom_path=None): model.load_weights(self.get_save_path(max_epoch)) return max_epoch else: - epoch = self.extract_id_and_epoch_from_filename(os.path.basename(custom_path))[1] + epoch = self.extract_id_and_epoch_from_filename( + os.path.basename(custom_path))[1] model.load_weights(custom_path) print("Loading from custom epoch {}".format(epoch)) return epoch @@ -263,14 +339,12 @@ def get_latest_save_path(self): print('loading from epoch {}'.format(max_epoch)) return self.get_save_path(max_epoch) - - def extract_id_and_epoch_from_filename(self,filename): + def extract_id_and_epoch_from_filename(self, filename): regex = re.compile(r'-?\d+') numbers = [int(x) for x in regex.findall(filename)] - assert(len(numbers) == 3) #id,epoch number and extension - assert(numbers[2] == 5) #.h5 extension - return numbers[0],numbers[1] - + assert(len(numbers) == 3) # id,epoch number and extension + assert(numbers[2] == 5) # .h5 extension + return numbers[0], numbers[1] def get_all_saved_files(self): self.ensure_save_directory() @@ -278,16 +352,15 @@ def get_all_saved_files(self): filenames = os.listdir(self.conf['paths']['model_save_path']) epochs = [] for file in filenames: - curr_id,epoch = self.extract_id_and_epoch_from_filename(file) + curr_id, epoch = self.extract_id_and_epoch_from_filename(file) if curr_id == unique_id: epochs.append(epoch) return epochs - - #FIXME this is essentially the ModelBuilder.build_model - #in the long run we want to replace the space dictionary with the - #regular conf file - I am sure there is a way to accomodate - def hyper_build_model(self,space,predict,custom_batch_size=None): + # FIXME this is essentially the ModelBuilder.build_model + # in the long run we want to replace the space dictionary with the + # regular conf file - I am sure there is a way to accomodate + def hyper_build_model(self, space, predict, custom_batch_size=None): conf = self.conf model_conf = conf['model'] rnn_size = model_conf['rnn_size'] @@ -297,19 +370,19 @@ def hyper_build_model(self,space,predict,custom_batch_size=None): dropout_prob = model_conf['dropout_prob'] length = model_conf['length'] pred_length = model_conf['pred_length'] - skip = model_conf['skip'] + # skip = model_conf['skip'] stateful = model_conf['stateful'] return_sequences = model_conf['return_sequences'] - output_activation = conf['data']['target'].activation#model_conf['output_activation'] + # model_conf['output_activation'] + output_activation = conf['data']['target'].activation num_signals = conf['data']['num_signals'] - batch_size = self.conf['training']['batch_size'] if predict: batch_size = self.conf['model']['pred_batch_size'] - #so we can predict with one time point at a time! + # so we can predict with one time point at a time! if return_sequences: - length =pred_length + length = pred_length else: length = 1 @@ -319,23 +392,31 @@ def hyper_build_model(self,space,predict,custom_batch_size=None): if rnn_type == 'LSTM': rnn_model = LSTM elif rnn_type == 'SimpleRNN': - rnn_model =SimpleRNN + rnn_model = SimpleRNN else: print('Unkown Model Type, exiting.') exit(1) - - batch_input_shape=(batch_size,length, num_signals) + + batch_input_shape = (batch_size, length, num_signals) model = Sequential() for _ in range(model_conf['rnn_layers']): - model.add(rnn_model(rnn_size, return_sequences=return_sequences,batch_input_shape=batch_input_shape, - stateful=stateful,kernel_regularizer=l2(regularization),recurrent_regularizer=l2(regularization), - bias_regularizer=l2(regularization),dropout=dropout_prob,recurrent_dropout=dropout_prob)) + model.add( + rnn_model( + rnn_size, + return_sequences=return_sequences, + batch_input_shape=batch_input_shape, + stateful=stateful, + kernel_regularizer=l2(regularization), + recurrent_regularizer=l2(regularization), + bias_regularizer=l2(regularization), + dropout=dropout_prob, + recurrent_dropout=dropout_prob)) model.add(Dropout(space['Dropout'])) if return_sequences: - model.add(TimeDistributed(Dense(1,activation=output_activation))) + model.add(TimeDistributed(Dense(1, activation=output_activation))) else: - model.add(Dense(1,activation=output_activation)) + model.add(Dense(1, activation=output_activation)) model.reset_states() return model diff --git a/plasma/models/custom_loss.py b/plasma/models/custom_loss.py index aaaa03b5..bdc18a9e 100644 --- a/plasma/models/custom_loss.py +++ b/plasma/models/custom_loss.py @@ -1,27 +1,30 @@ import numpy as np -from keras import objectives +# from keras import objectives from keras import backend as K -from keras.losses import hinge, squared_hinge +from keras.losses import squared_hinge _EPSILON = K.epsilon() + def _loss_tensor(y_true, y_pred): - max_val = K.max(y_pred,axis=-2) #temporal axis! - max_val = K.repeat(max_val,K.shape(y_pred)[-2]) + max_val = K.max(y_pred, axis=-2) # temporal axis! + max_val = K.repeat(max_val, K.shape(y_pred)[-2]) print(K.eval(max_val)) - mask = K.cast(K.equal(max_val,y_pred),K.floatx()) + mask = K.cast(K.equal(max_val, y_pred), K.floatx()) y_pred = mask * y_pred + (1-mask) * y_true - return squared_hinge(y_true,y_pred) + return squared_hinge(y_true, y_pred) + def _loss_np(y_true, y_pred): print(y_pred.shape) - max_val = np.max(y_pred,axis=-2) #temporal axis! - max_val = np.reshape(max_val,max_val.shape[:-1] + (1,) + (max_val.shape[-1],)) - max_val = np.tile(max_val,(1,y_pred.shape[-2],1)) + max_val = np.max(y_pred, axis=-2) # temporal axis! + max_val = np.reshape( + max_val, max_val.shape[:-1] + (1,) + (max_val.shape[-1],)) + max_val = np.tile(max_val, (1, y_pred.shape[-2], 1)) print(max_val.shape) print(max_val) - mask = np.equal(max_val,y_pred) + mask = np.equal(max_val, y_pred) mask = mask.astype(np.float32) y_pred = mask * y_pred + (1-mask) * y_true return np.mean(np.square(np.maximum(1. - y_true * y_pred, 0.)), axis=-1) @@ -38,7 +41,7 @@ def check_loss(_shape): shape = (9, 8, 5, 6, 7) y_a = 1.0*np.ones(shape) - y_b = 0.5+np.random.random(shape) + y_b = 0.5 + np.random.random(shape) print(y_a) print(y_b) @@ -55,10 +58,11 @@ def check_loss(_shape): def test_loss(): - shape_list = ['3d']#, '3d', '4d', '5d'] + shape_list = ['3d'] # , '3d', '4d', '5d'] for _shape in shape_list: check_loss(_shape) print('======================') + if __name__ == '__main__': test_loss() diff --git a/plasma/models/loader.py b/plasma/models/loader.py index 56ba0021..cdc63937 100644 --- a/plasma/models/loader.py +++ b/plasma/models/loader.py @@ -14,45 +14,55 @@ from plasma.primitives.shots import Shot import multiprocessing as mp -import pdb +# import pdb + class Loader(object): - ''' - A Python class to ... + '''A Python class to ... - The length of shots in e.g. JET data varies by orders of magnitude. For data parallel - synchronous training it is essential that amounds of train data passed to the model replica is about the same size. - Therefore, a patching technique is introduced. + The length of shots in e.g. JET data varies by orders of magnitude. For + data parallel synchronous training it is essential that amounds of train + data passed to the model replica is about the same size. Therefore, a + patching technique is introduced. + + A patch is a subset of shot's time/signal profile having a fixed length, + equal among all patches. Patch size is approximately equal to the minimum + shot length. More precisely: it is equal to the max(1, + min_len//rnn_length)*rnn_length - the largest number less or equal to the + minimum shot length divisible by the LSTM model length. If minimum shot + length is less than the rnn_length, then the patch length is equal to the + rnn_length - A patch is a subset of shot's time/signal profile having a fixed length, equal among all patches. - Patch size is approximately equal to the minimum shot length. More precisely: it is equal - to the max(1, min_len//rnn_length)*rnn_length - the largest number less or equal to the minimum shot length divisible by the LSTM model length. If minimum shot length is less than the rnn_length, then the patch length is equal to the rnn_length ''' - def __init__(self,conf,normalizer=None): + def __init__(self, conf, normalizer=None): self.conf = conf self.stateful = conf['model']['stateful'] self.normalizer = normalizer self.verbose = True - def set_inference_mode(self,val): + def set_inference_mode(self, val): self.normalizer.set_inference_mode(val) - def training_batch_generator(self,shot_list): - """ - The method implements a training batch generator as a Python generator with a while-loop. - It iterates indefinitely over the data set and returns one mini-batch of data at a time. + def training_batch_generator(self, shot_list): + """The method implements a training batch generator as a Python + generator with a while-loop. It iterates indefinitely over the + data set and returns one mini-batch of data at a time. - NOTE: Can be inefficient during distributed training because one process loading data will - cause all other processes to stall. + NOTE: Can be inefficient during distributed training because one + process loading data will cause all other processes to stall. - Argument list: + Argument list: - shot_list: - Returns: - - One mini-batch of data and label as a Numpy array: X[start:end],y[start:end] - - reset_states_now: boolean flag indicating when to reset state during stateful RNN training - - num_so_far,num_total: number of samples generated so far and the total dataset size as per shot_list + Returns: + - One mini-batch of data and label as a Numpy array: X[start:end], + y[start:end] + - reset_states_now: boolean flag indicating when to reset state + during stateful RNN training + - num_so_far,num_total: number of samples generated so far and the + total dataset size as per shot_list + """ batch_size = self.conf['training']['batch_size'] num_at_once = self.conf['training']['num_shots_at_once'] @@ -60,193 +70,238 @@ def training_batch_generator(self,shot_list): num_so_far = 0 while True: # the list of all shots - shot_list.shuffle() - # split the list into equal-length sublists (random shots will be reused to make them equal length). - shot_sublists = shot_list.sublists(num_at_once,equal_size=True) + shot_list.shuffle() + # split the list into equal-length sublists (random shots will be + # reused to make them equal length). + shot_sublists = shot_list.sublists(num_at_once, equal_size=True) num_total = len(shot_list) - for (i,shot_sublist) in enumerate(shot_sublists): - #produce a list of equal-length chunks from this set of shots - X_list,y_list = self.load_as_X_y_list(shot_sublist) - #Each chunk will be a multiple of the batch size - for j,(X,y) in enumerate(zip(X_list,y_list)): + for (i, shot_sublist) in enumerate(shot_sublists): + # produce a list of equal-length chunks from this set of shots + X_list, y_list = self.load_as_X_y_list(shot_sublist) + # Each chunk will be a multiple of the batch size + for j, (X, y) in enumerate(zip(X_list, y_list)): num_examples = X.shape[0] assert(num_examples % batch_size == 0) num_chunks = num_examples//batch_size """ - The method produces batch-sized training data X and labels y as Numpy arrays to feed during training. - Mini-batch dimensions are (num_examples, num_timesteps, num_dimensions_of_data) - also num_examples has to be divisible by the batch_size. The i-th example and the - (batchsize + 1)-th example are consecutive in time, so we do not reset the - RNN internal state unless we start a new chunk. + The method produces batch-sized training data X and labels + y as Numpy arrays to feed during training. + + Mini-batch dimensions are (num_examples, num_timesteps, + num_dimensions_of_data) also num_examples has to be + divisible by the batch_size. The i-th example and the + (batchsize + 1)-th example are consecutive in time, so we + do not reset the RNN internal state unless we start a new + chunk. + """ for k in range(num_chunks): - #epoch_end = (i == len(shot_sublists) - 1 and j == len(X_list) -1 and k == num_chunks - 1) + # epoch_end = (i == len(shot_sublists) - 1 and j == + # len(X_list) -1 and k == num_chunks - 1) reset_states_now = (k == 0) start = k*batch_size end = (k + 1)*batch_size - num_so_far += 1.0*len(shot_sublist)/(len(X_list)*num_chunks) - yield X[start:end],y[start:end],reset_states_now,num_so_far,num_total + num_so_far += 1.0 * \ + len(shot_sublist)/(len(X_list)*num_chunks) + yield X[start:end], y[start:end], reset_states_now, + num_so_far, num_total epoch += 1 - def fill_training_buffer(self,Xbuff,Ybuff,end_indices,shot,is_first_fill=False): - sig,res = self.get_signal_result_from_shot(shot) + def fill_training_buffer( + self, + Xbuff, + Ybuff, + end_indices, + shot, + is_first_fill=False): + sig, res = self.get_signal_result_from_shot(shot) length = self.conf['model']['length'] - if is_first_fill:#cut signal to random position + if is_first_fill: # cut signal to random position cut_idx = np.random.randint(res.shape[0]-length+1) sig = sig[cut_idx:] res = res[cut_idx:] sig_len = res.shape[0] - sig_len = (sig_len // length)*length #make divisible by lenth + sig_len = (sig_len // length)*length # make divisible by lenth assert(sig_len > 0) batch_idx = np.where(end_indices == 0)[0][0] if sig_len > Xbuff.shape[1]: - Xbuff = self.resize_buffer(Xbuff,sig_len+length) - Ybuff = self.resize_buffer(Ybuff,sig_len+length) - Xbuff[batch_idx,:sig_len,:] = sig[-sig_len:] - Ybuff[batch_idx,:sig_len,:] = res[-sig_len:] + Xbuff = self.resize_buffer(Xbuff, sig_len+length) + Ybuff = self.resize_buffer(Ybuff, sig_len+length) + Xbuff[batch_idx, :sig_len, :] = sig[-sig_len:] + Ybuff[batch_idx, :sig_len, :] = res[-sig_len:] end_indices[batch_idx] += sig_len - #print("Filling buffer at index {}".format(batch_idx)) - return Xbuff,Ybuff,batch_idx + # print("Filling buffer at index {}".format(batch_idx)) + return Xbuff, Ybuff, batch_idx - def return_from_training_buffer(self,Xbuff,Ybuff,end_indices): + def return_from_training_buffer(self, Xbuff, Ybuff, end_indices): length = self.conf['model']['length'] end_indices -= length assert(np.all(end_indices >= 0)) - X = 1.0*Xbuff[:,:length,:] - Y = 1.0*Ybuff[:,:length,:] - self.shift_buffer(Xbuff,length) - self.shift_buffer(Ybuff,length) - return X,Y - - def shift_buffer(self,buff,length): - buff[:,:-length,:] = buff[:,length:,:] + X = 1.0*Xbuff[:, :length, :] + Y = 1.0*Ybuff[:, :length, :] + self.shift_buffer(Xbuff, length) + self.shift_buffer(Ybuff, length) + return X, Y + def shift_buffer(self, buff, length): + buff[:, :-length, :] = buff[:, length:, :] - def resize_buffer(self,buff,new_length): + def resize_buffer(self, buff, new_length): old_length = buff.shape[1] batch_size = buff.shape[0] num_signals = buff.shape[2] - new_buff = np.empty((batch_size,new_length,num_signals),dtype=self.conf['data']['floatx']) - new_buff[:,:old_length,:] = buff - #print("Resizing buffer to new length {}".format(new_length)) + new_buff = np.empty( + (batch_size, + new_length, + num_signals), + dtype=self.conf['data']['floatx']) + new_buff[:, :old_length, :] = buff + # print("Resizing buffer to new length {}".format(new_length)) return new_buff - - - - def training_batch_generator_partial_reset(self,shot_list): + def training_batch_generator_partial_reset(self, shot_list): """ - The method implements a training batch generator as a Python generator with a while-loop. - It iterates indefinitely over the data set and returns one mini-batch of data at a time. + The method implements a training batch generator as a Python generator + with a while-loop. It iterates indefinitely over the data set and + returns one mini-batch of data at a time. - NOTE: Can be inefficient during distributed training because one process loading data will - cause all other processes to stall. + NOTE: Can be inefficient during distributed training because one + process loading data will cause all other processes to stall. - Argument list: + Argument list: - shot_list: - Returns: - - One mini-batch of data and label as a Numpy array: X[start:end],y[start:end] - - reset_states_now: boolean flag indicating when to reset state during stateful RNN training - - num_so_far,num_total: number of samples generated so far and the total dataset size as per shot_list + Returns: + - One mini-batch of data and label as a Numpy array: X[start:end], + y[start:end] + - reset_states_now: boolean flag indicating when to reset state + during stateful RNN training + - num_so_far,num_total: number of samples generated so far and the + total dataset size as per shot_list """ batch_size = self.conf['training']['batch_size'] - length = self.conf['model']['length'] - sig,res = self.get_signal_result_from_shot(shot_list.shots[0]) - Xbuff = np.empty((batch_size,) + sig.shape,dtype=self.conf['data']['floatx']) - Ybuff = np.empty((batch_size,) + res.shape,dtype=self.conf['data']['floatx']) - end_indices = np.zeros(batch_size,dtype=np.int) - batches_to_reset = np.ones(batch_size,dtype=np.bool) + # length = self.conf['model']['length'] + sig, res = self.get_signal_result_from_shot(shot_list.shots[0]) + Xbuff = np.empty((batch_size,) + sig.shape, + dtype=self.conf['data']['floatx']) + Ybuff = np.empty((batch_size,) + res.shape, + dtype=self.conf['data']['floatx']) + end_indices = np.zeros(batch_size, dtype=np.int) + batches_to_reset = np.ones(batch_size, dtype=np.bool) # epoch = 0 num_total = len(shot_list) num_so_far = 0 returned = False num_steps = 0 warmup_steps = self.conf['training']['batch_generator_warmup_steps'] - is_warmup_period = num_steps < warmup_steps + is_warmup_period = num_steps < warmup_steps is_first_fill = num_steps < batch_size while True: # the list of all shots - shot_list.shuffle() + shot_list.shuffle() for i in range(len(shot_list)): if self.conf['training']['ranking_difficulty_fac'] == 1.0: if self.conf['data']['equalize_classes']: shot = shot_list.sample_equal_classes() else: shot = shot_list.shots[i] - else: #draw the shot weighted + else: # draw the shot weighted shot = shot_list.sample_weighted() while not np.any(end_indices == 0): - X,Y = self.return_from_training_buffer(Xbuff,Ybuff,end_indices) - yield X,Y,batches_to_reset,num_so_far,num_total,is_warmup_period + X, Y = self.return_from_training_buffer( + Xbuff, Ybuff, end_indices) + yield (X, Y, batches_to_reset, num_so_far, num_total, + is_warmup_period) returned = True num_steps += 1 is_warmup_period = num_steps < warmup_steps is_first_fill = num_steps < batch_size batches_to_reset[:] = False - Xbuff,Ybuff,batch_idx = self.fill_training_buffer(Xbuff,Ybuff,end_indices,shot,is_first_fill) + Xbuff, Ybuff, batch_idx = self.fill_training_buffer( + Xbuff, Ybuff, end_indices, shot, is_first_fill) batches_to_reset[batch_idx] = True if returned and not is_warmup_period: num_so_far += 1 # epoch += 1 - def fill_batch_queue(self,shot_list,queue): + def fill_batch_queue(self, shot_list, queue): print("Starting thread to fill queue") gen = self.training_batch_generator_partial_reset(shot_list) while True: ret = next(gen) - queue.put(ret,block=True,timeout=-1) + queue.put(ret, block=True, timeout=-1) - - def training_batch_generator_process(self,shot_list): + def training_batch_generator_process(self, shot_list): queue = mp.Queue() - proc = mp.Process(target = self.fill_batch_queue,args=(shot_list,queue)) + proc = mp.Process( + target=self.fill_batch_queue, args=( + shot_list, queue)) proc.start() while True: yield queue.get(True) proc.join() queue.close() - def load_as_X_y_list(self,shot_list,verbose=False,prediction_mode=False): + def load_as_X_y_list( + self, + shot_list, + verbose=False, + prediction_mode=False): """ - The method turns a ShotList into a set of equal-sized patches which contain a number of examples - that is a multiple of the batch size. - Initially, shots are "light" meaning signal amd disruption related attributes are not filled. - By invoking Loader.get_signals_results_from_shotlist the shot information is filled and stored in - the object in memory. Next, patches are made, finally patches are arranged into batch input shape expected by RNN model. - - Performs calls to: get_signals_results_from_shotlist, make_patches, arange_patches + The method turns a ShotList into a set of equal-sized patches which + contain a number of examples that is a multiple of the batch size. + Initially, shots are "light" meaning signal amd disruption related + attributes are not filled. + By invoking Loader.get_signals_results_from_shotlist the + shot information is filled and stored in the object in memory. Next, + patches are made, finally patches are arranged into batch input shape + expected by RNN model. + + Performs calls to: get_signals_results_from_shotlist, make_patches, + arange_patches Argument list: - shot_list: a ShotList - verbose: TO BE DEPRECATED, self.verbose data member is used instead - prediction_mode: unused - + Returns: - X_list,y_list: lists of Numpy arrays of batch input shape + """ - signals,results,total_length = self.get_signals_results_from_shotlist(shot_list) - sig_patches, res_patches = self.make_patches(signals,results) + # TODO(KGF): check tuple unpack + (signals, results, + total_length) = self.get_signals_results_from_shotlist(shot_list) + sig_patches, res_patches = self.make_patches(signals, results) - X_list,y_list = self.arange_patches(sig_patches,res_patches) + X_list, y_list = self.arange_patches(sig_patches, res_patches) effective_length = len(res_patches)*len(res_patches[0]) if self.verbose: - print('multiplication factor: {}'.format(1.0*effective_length/total_length)) - print('effective/total length : {}/{}'.format(effective_length,total_length)) - print('patch length: {} num patches: {}'.format(len(res_patches[0]),len(res_patches))) - return X_list,y_list - - def load_as_X_y_pred(self,shot_list,verbose=False,custom_batch_size=None): - signals,results,shot_lengths,disruptive = self.get_signals_results_from_shotlist(shot_list,prediction_mode=True) - sig_patches, res_patches = self.make_prediction_patches(signals,results) - X,y = self.arange_patches_single(sig_patches,res_patches,prediction_mode=True,custom_batch_size=custom_batch_size) - return X,y,shot_lengths,disruptive - - - def get_signals_results_from_shotlist(self,shot_list,prediction_mode=False): + print('multiplication factor: {}'.format( + 1.0 * effective_length / total_length)) + print('effective/total length : {}/{}'.format( + effective_length, total_length)) + print('patch length: {} num patches: {}'.format( + len(res_patches[0]), len(res_patches))) + return X_list, y_list + + def load_as_X_y_pred(self, shot_list, verbose=False, + custom_batch_size=None): + (signals, results, shot_lengths, + disruptive) = self.get_signals_results_from_shotlist( + shot_list, prediction_mode=True) + sig_patches, res_patches = self.make_prediction_patches(signals, + results) + X, y = self.arange_patches_single(sig_patches, res_patches, + prediction_mode=True, + custom_batch_size=custom_batch_size) + return X, y, shot_lengths, disruptive + + def get_signals_results_from_shotlist(self, shot_list, + prediction_mode=False): prepath = self.conf['paths']['processed_prepath'] use_signals = self.conf['paths']['use_signals'] signals = [] @@ -255,20 +310,20 @@ def get_signals_results_from_shotlist(self,shot_list,prediction_mode=False): shot_lengths = [] total_length = 0 for shot in shot_list: - assert(isinstance(shot,Shot)) + assert(isinstance(shot, Shot)) assert(shot.valid) shot.restore(prepath) if self.normalizer is not None: self.normalizer.apply(shot) else: - print('Warning, no normalization. Training data may be poorly conditioned') - - + print('Warning, no normalization. ', + 'Training data may be poorly conditioned') if self.conf['training']['use_mock_data']: - signal,ttd = self.get_mock_data() - ttd,signal = shot.get_data_arrays(use_signals,self.conf['data']['floatx']) + signal, ttd = self.get_mock_data() + ttd, signal = shot.get_data_arrays( + use_signals, self.conf['data']['floatx']) if len(ttd) < self.conf['model']['length']: print(ttd) print(shot) @@ -279,29 +334,31 @@ def get_signals_results_from_shotlist(self,shot_list,prediction_mode=False): shot_lengths.append(len(ttd)) disruptive.append(shot.is_disruptive) if len(ttd.shape) == 1: - results.append(np.expand_dims(ttd,axis=1)) + results.append(np.expand_dims(ttd, axis=1)) else: results.append(ttd) shot.make_light() if not prediction_mode: - return signals,results,total_length + return signals, results, total_length else: - return signals,results,shot_lengths,disruptive + return signals, results, shot_lengths, disruptive - def get_signal_result_from_shot(self,shot,prediction_mode=False): + def get_signal_result_from_shot(self, shot, prediction_mode=False): prepath = self.conf['paths']['processed_prepath'] use_signals = self.conf['paths']['use_signals'] - assert(isinstance(shot,Shot)) + assert(isinstance(shot, Shot)) assert(shot.valid) shot.restore(prepath) if self.normalizer is not None: self.normalizer.apply(shot) else: - print('Warning, no normalization. Training data may be poorly conditioned') + print('Warning, no normalization. ', + 'Training data may be poorly conditioned') if self.conf['training']['use_mock_data']: - signal,ttd = self.get_mock_data() - ttd,signal = shot.get_data_arrays(use_signals,self.conf['data']['floatx']) + signal, ttd = self.get_mock_data() + ttd, signal = shot.get_data_arrays( + use_signals, self.conf['data']['floatx']) if len(ttd) < self.conf['model']['length']: print(ttd) print(shot) @@ -310,15 +367,14 @@ def get_signal_result_from_shot(self,shot,prediction_mode=False): exit(1) if len(ttd.shape) == 1: - ttd = np.expand_dims(ttd,axis=1) + ttd = np.expand_dims(ttd, axis=1) shot.make_light() if not prediction_mode: - return signal,ttd + return signal, ttd else: - return signal,ttd,shot.is_disruptive + return signal, ttd, shot.is_disruptive - - def batch_output_to_array(self,output,batch_size = None): + def batch_output_to_array(self, output, batch_size=None): if batch_size is None: batch_size = self.conf['model']['pred_batch_size'] assert(output.shape[0] % batch_size == 0) @@ -328,152 +384,171 @@ def batch_output_to_array(self,output,batch_size = None): outs = [] for patch_idx in range(batch_size): - out = np.empty((num_chunks*num_timesteps,feature_size)) + out = np.empty((num_chunks*num_timesteps, feature_size)) for chunk in range(num_chunks): - out[chunk*num_timesteps:(chunk+1)*num_timesteps,:] = output[chunk*batch_size+patch_idx,:,:] + out[chunk*num_timesteps:(chunk + 1)*num_timesteps, :] = output[ + chunk * batch_size + patch_idx, :, :] outs.append(out) - return outs - + return outs - def make_deterministic_patches(self,signals,results): + def make_deterministic_patches(self, signals, results): num_timesteps = self.conf['model']['length'] sig_patches = [] res_patches = [] - min_len = self.get_min_len(signals,num_timesteps) - for sig,res in zip(signals,results): - sig_patch, res_patch = self.make_deterministic_patches_from_single_array(sig,res,min_len) + min_len = self.get_min_len(signals, num_timesteps) + for sig, res in zip(signals, results): + (sig_patch, + res_patch) = self.make_deterministic_patches_from_single_array( + sig, res, min_len) sig_patches += sig_patch res_patches += res_patch return sig_patches, res_patches - def make_deterministic_patches_from_single_array(self,sig,res,min_len): + def make_deterministic_patches_from_single_array(self, sig, res, min_len): sig_patches = [] res_patches = [] if len(sig) <= min_len: print('signal length: {}'.format(len(sig))) assert(min_len <= len(sig)) - for start in range(0,len(sig)-min_len,min_len): + for start in range(0, len(sig)-min_len, min_len): sig_patches.append(sig[start:start+min_len]) res_patches.append(res[start:start+min_len]) sig_patches.append(sig[-min_len:]) res_patches.append(res[-min_len:]) - return sig_patches,res_patches + return sig_patches, res_patches - def make_random_patches(self,signals,results,num): + def make_random_patches(self, signals, results, num): num_timesteps = self.conf['model']['length'] sig_patches = [] res_patches = [] - min_len = self.get_min_len(signals,num_timesteps) + min_len = self.get_min_len(signals, num_timesteps) for i in range(num): - idx= np.random.randint(len(signals)) - sig_patch, res_patch = self.make_random_patch_from_array(signals[idx],results[idx],min_len) + idx = np.random.randint(len(signals)) + sig_patch, res_patch = self.make_random_patch_from_array( + signals[idx], results[idx], min_len) sig_patches.append(sig_patch) res_patches.append(res_patch) - return sig_patches,res_patches + return sig_patches, res_patches - def make_random_patch_from_array(self,sig,res,min_len): + def make_random_patch_from_array(self, sig, res, min_len): start = np.random.randint(len(sig) - min_len+1) - return sig[start:start+min_len],res[start:start+min_len] - + return sig[start:start+min_len], res[start:start+min_len] - def get_min_len(self,arrs,length): - min_len = min([len(a) for a in arrs] + [self.conf['training']['max_patch_length']]) - min_len = max(1,min_len // length) * length + def get_min_len(self, arrs, length): + min_len = min([len(a) for a in arrs] + + [self.conf['training']['max_patch_length']]) + min_len = max(1, min_len // length) * length return min_len - - def get_max_len(self,arrs,length): + def get_max_len(self, arrs, length): max_len = max([len(a) for a in arrs]) - max_len = int(np.ceil(1.0*max_len / length) * length ) + max_len = int(np.ceil(1.0*max_len / length) * length) return max_len - def make_patches(self,signals,results): - """ - A patch is a subset of shot's time/signal profile having a fixed length, equal among all patches. - Patch size is approximately equal to the minimum shot length. More precisely: it is equal - to the max(1, min_len//rnn_length)*rnn_length - the largest number less or equal to the minimum shot length divisible by the LSTM model length. If minimum shot length is less than the rnn_length, then the patch length is equal to the rnn_length + def make_patches(self, signals, results): + """A patch is a subset of shot's time/signal profile having a fixed + length, equal among all patches. Patch size is approximately equal to + the minimum shot length. More precisely: it is equal to the max(1, + min_len//rnn_length)*rnn_length - the largest number less or equal to + the minimum shot length divisible by the LSTM model length. If minimum + shot length is less than the rnn_length, then the patch length is equal + to the rnn_length - Since shot lengthes are not multiples of the minimum shot length in general, - some non-deterministic fraction of patches is created. See: + Since shot lengthes are not multiples of the minimum shot length in + general, some non-deterministic fraction of patches is created. See: Deterministic patching: Random patching: - Argument list: - - signals: a list of 1D Numpy array of doubles containing signal values (a plasma property). - Numpy arrays are shot-sized - - results: a list of 1D Numpy array of doubles containing disruption times or -1 if a shot - is non-disruptive. Numpy arrays are shot-sized + Argument list: + - signals: a list of 1D Numpy array of doubles containing signal + values (a plasma property). Numpy arrays are shot-sized + - results: a list of 1D Numpy array of doubles containing disruption + times or -1 if a shot is non-disruptive. Numpy arrays are shot-sized - NOTE: signals and results are parallel lists. Since Arrays are shot-sized, the shape veries across the list + NOTE: signals and results are parallel lists. Since Arrays are + shot-sized, the shape veries across the list + + + Returns: + - sig_patches_det + sig_patches_rand: (concatenated) list of 1D Numpy + arrays of doubles containing signal values. Numpy arrays are + patch-sized + - res_patches_det + res_patches_rand: (concatenated) a list of 1D + Numpy array of doubles containing disruption times or -1 if a shot is + non-disruptive. Numpy arrays are patch-sized + NOTE: sig_patches_det + sig_patches_rand and res_patches_det + + res_patches_rand are prallel lists. All arrays in the list have + identical shapes. - Returns: - - sig_patches_det + sig_patches_rand: (concatenated) list of 1D Numpy arrays of doubles containing signal values. - Numpy arrays are patch-sized - - res_patches_det + res_patches_rand: (concatenated) a list of 1D Numpy array of doubles containing disruption times - or -1 if a shot is non-disruptive. Numpy arrays are patch-sized - NOTE: sig_patches_det + sig_patches_rand and res_patches_det + res_patches_rand are prallel lists - All arrays in the list have identical shapes. """ - total_num = self.conf['training']['batch_size'] - sig_patches_det,res_patches_det = self.make_deterministic_patches(signals,results) + total_num = self.conf['training']['batch_size'] + sig_patches_det, res_patches_det = self.make_deterministic_patches( + signals, results) num_already = len(sig_patches_det) - + total_num = int(np.ceil(1.0 * num_already / total_num)) * total_num - + num_additional = total_num - num_already assert(num_additional >= 0) - sig_patches_rand,res_patches_rand = self.make_random_patches(signals,results,num_additional) + sig_patches_rand, res_patches_rand = self.make_random_patches( + signals, results, num_additional) if self.verbose: - print('random to deterministic ratio: {}/{}'.format(num_additional,num_already)) - return sig_patches_det + sig_patches_rand,res_patches_det + res_patches_rand - - - def make_prediction_patches(self,signals,results): - #total_num = self.conf['training']['batch_size'] + print( + 'random to deterministic ratio: {}/{}'.format(num_additional, + num_already)) + return (sig_patches_det + sig_patches_rand, + res_patches_det + res_patches_rand) + + def make_prediction_patches(self, signals, results): + # total_num = self.conf['training']['batch_size'] num_timesteps = self.conf['model']['pred_length'] sig_patches = [] res_patches = [] - max_len = self.get_max_len(signals,num_timesteps) - for sig,res in zip(signals,results): - sig_patches.append(Loader.pad_array_to_length(sig,max_len)) - res_patches.append(Loader.pad_array_to_length(res,max_len)) + max_len = self.get_max_len(signals, num_timesteps) + for sig, res in zip(signals, results): + sig_patches.append(Loader.pad_array_to_length(sig, max_len)) + res_patches.append(Loader.pad_array_to_length(res, max_len)) return sig_patches, res_patches @staticmethod - def pad_array_to_length(arr,length): - dlength = max(0,length - arr.shape[0]) - tuples = [(0,dlength)] + def pad_array_to_length(arr, length): + dlength = max(0, length - arr.shape[0]) + tuples = [(0, dlength)] for l in arr.shape[1:]: - tuples.append((0,0)) - return np.pad(arr,tuples,mode='constant',constant_values=0) - - + tuples.append((0, 0)) + return np.pad(arr, tuples, mode='constant', constant_values=0) - - def arange_patches(self,sig_patches,res_patches): + def arange_patches(self, sig_patches, res_patches): num_timesteps = self.conf['model']['length'] batch_size = self.conf['training']['batch_size'] - assert(len(sig_patches) % batch_size == 0) #fixed number of batches - assert(len(sig_patches[0]) % num_timesteps == 0) #divisible by length of RNN sequence + assert(len(sig_patches) % batch_size == 0) # fixed number of batches + # divisible by length of RNN sequence + assert(len(sig_patches[0]) % num_timesteps == 0) num_batches = len(sig_patches) // batch_size - #patch_length = len(sig_patches[0]) + # patch_length = len(sig_patches[0]) - zipped = list(zip(sig_patches,res_patches)) + zipped = list(zip(sig_patches, res_patches)) np.random.shuffle(zipped) - sig_patches, res_patches = zip(*zipped) + sig_patches, res_patches = zip(*zipped) X_list = [] y_list = [] for i in range(num_batches): - X,y = self.arange_patches_single(sig_patches[i*batch_size:(i+1)*batch_size], - res_patches[i*batch_size:(i+1)*batch_size]) + X, y = self.arange_patches_single( + sig_patches[i*batch_size:(i+1)*batch_size], + res_patches[i*batch_size:(i+1)*batch_size]) X_list.append(X) y_list.append(y) - return X_list,y_list - - def arange_patches_single(self,sig_patches,res_patches,prediction_mode=False,custom_batch_size=None): + return X_list, y_list + + def arange_patches_single( + self, + sig_patches, + res_patches, + prediction_mode=False, + custom_batch_size=None): if prediction_mode: num_timesteps = self.conf['model']['pred_length'] batch_size = self.conf['model']['pred_batch_size'] @@ -492,27 +567,32 @@ def arange_patches_single(self,sig_patches,res_patches,prediction_mode=False,cus num_answers = 1 else: num_answers = res_patches[0].shape[1] - - X = np.zeros((num_chunks*batch_size,num_timesteps,num_dimensions_of_data)) + + X = np.zeros( + (num_chunks*batch_size, + num_timesteps, + num_dimensions_of_data)) if return_sequences: - y = np.zeros((num_chunks*batch_size,num_timesteps,num_answers)) + y = np.zeros((num_chunks*batch_size, num_timesteps, num_answers)) else: - y = np.zeros((num_chunks*batch_size,num_answers)) + y = np.zeros((num_chunks*batch_size, num_answers)) - for chunk_idx in range(num_chunks): src_start = chunk_idx*num_timesteps src_end = (chunk_idx+1)*num_timesteps for patch_idx in range(batch_size): - X[chunk_idx*batch_size + patch_idx,:,:] = sig_patches[patch_idx][src_start:src_end] + X[chunk_idx*batch_size + patch_idx, :, + :] = sig_patches[patch_idx][src_start:src_end] if return_sequences: - y[chunk_idx*batch_size + patch_idx,:,:] = res_patches[patch_idx][src_start:src_end] + y[chunk_idx*batch_size + patch_idx, :, + :] = res_patches[patch_idx][src_start:src_end] else: - y[chunk_idx*batch_size + patch_idx,:] = res_patches[patch_idx][src_end-1] - return X,y + y[chunk_idx*batch_size + patch_idx, + :] = res_patches[patch_idx][src_end-1] + return X, y - def load_as_X_y(self,shot,verbose=False,prediction_mode=False): - assert(isinstance(shot,Shot)) + def load_as_X_y(self, shot, verbose=False, prediction_mode=False): + assert(isinstance(shot, Shot)) assert(shot.valid) prepath = self.conf['paths']['processed_prepath'] return_sequences = self.conf['model']['return_sequences'] @@ -521,111 +601,116 @@ def load_as_X_y(self,shot,verbose=False,prediction_mode=False): if self.normalizer is not None: self.normalizer.apply(shot) else: - print('Warning, no normalization. Training data may be poorly conditioned') - + print('Warning, no normalization. ', + 'Training data may be poorly conditioned') signals = shot.signals ttd = shot.ttd if self.conf['training']['use_mock_data']: - signals,ttd = self.get_mock_data() + signals, ttd = self.get_mock_data() # if not self.stateful: # X,y = self.array_to_path_and_external_pred(signals,ttd) # else: - X,y = self.array_to_path_and_external_pred_cut(signals,ttd, - return_sequences=return_sequences,prediction_mode=prediction_mode) + X, y = self.array_to_path_and_external_pred_cut( + signals, ttd, return_sequences=return_sequences, + prediction_mode=prediction_mode) shot.make_light() - return X,y#X,y + return X, y # X,y def get_mock_data(self): - signals = linspace(0,4*pi,10000) - rand_idx = randint(6000) - lgth = randint(1000,3000) + signals = np.linspace(0, 4*np.pi, 10000) + rand_idx = np.randint(6000) + lgth = np.randint(1000, 3000) signals = signals[rand_idx:rand_idx+lgth] - #ttd[-100:] = 1 - signals = vstack([signals]*8) + # ttd[-100:] = 1 + signals = np.vstack([signals]*8) signals = signals.T - signals[:,0] = 0.5 + 0.5*sin(signals[:,0]) - signals[:,1] = 0.5# + 0.5*cos(signals[:,1]) - signals[:,2] = 0.5 + 0.5*sin(2*signals[:,2]) - signals[:,3:] *= 0 + signals[:, 0] = 0.5 + 0.5*np.sin(signals[:, 0]) + signals[:, 1] = 0.5 # + 0.5*cos(signals[:,1]) + signals[:, 2] = 0.5 + 0.5*np.sin(2*signals[:, 2]) + signals[:, 3:] *= 0 offset = 100 - ttd = 0.0*signals[:,0] - ttd[offset:] = 1.0*signals[:-offset,0] - mask = ttd > mean(ttd) + ttd = 0.0*signals[:, 0] + ttd[offset:] = 1.0*signals[:-offset, 0] + mask = ttd > np.mean(ttd) ttd[~mask] = 0 - #mean(signals[:,:2],1) - return signals,ttd - - def array_to_path_and_external_pred_cut(self,arr,res,return_sequences=False,prediction_mode=False): + # mean(signals[:,:2],1) + return signals, ttd + + def array_to_path_and_external_pred_cut( + self, + arr, + res, + return_sequences=False, + prediction_mode=False): num_timesteps = self.conf['model']['length'] skip = self.conf['model']['skip'] if prediction_mode: num_timesteps = self.conf['model']['pred_length'] if not return_sequences: num_timesteps = 1 - skip = num_timesteps #batchsize = 1! - assert(shape(arr)[0] == shape(res)[0]) + skip = num_timesteps # batchsize = 1! + assert(np.shape(arr)[0] == np.shape(res)[0]) num_chunks = len(arr) // num_timesteps arr = arr[-num_chunks*num_timesteps:] res = res[-num_chunks*num_timesteps:] - assert(shape(arr)[0] == shape(res)[0]) + assert(np.shape(arr)[0] == np.shape(res)[0]) X = [] y = [] i = 0 chunk_range = range(num_chunks-1) - i_range = range(1,num_timesteps+1,skip) + i_range = range(1, num_timesteps+1, skip) if prediction_mode: chunk_range = range(num_chunks) i_range = range(1) - for chunk in chunk_range: for i in i_range: start = chunk*num_timesteps + i assert(start + num_timesteps <= len(arr)) - X.append(arr[start:start+num_timesteps,:]) + X.append(arr[start:start+num_timesteps, :]) if return_sequences: y.append(res[start:start+num_timesteps]) else: y.append(res[start+num_timesteps-1:start+num_timesteps]) - X = array(X) - y = array(y) - if len(shape(X)) == 1: - X = np.expand_dims(X,axis=len(shape(X))) + X = np.array(X) + y = np.array(y) + if len(np.shape(X)) == 1: + X = np.expand_dims(X, axis=len(np.shape(X))) if return_sequences: - y = np.expand_dims(y,axis=len(shape(y))) - return X,y + y = np.expand_dims(y, axis=len(np.shape(y))) + return X, y @staticmethod - def get_batch_size(batch_size,prediction_mode): + def get_batch_size(batch_size, prediction_mode): if prediction_mode: return 1 else: - return batch_size#Loader.get_num_skips(length,skip) + return batch_size # Loader.get_num_skips(length,skip) @staticmethod - def get_num_skips(length,skip): + def get_num_skips(length, skip): return 1 + (length-1)//skip class ProcessGenerator(object): - def __init__(self,generator): + def __init__(self, generator): self.generator = generator self.proc = mp.Process(target=self.fill_batch_queue) self.queue = mp.Queue() self.proc.start() - + def fill_batch_queue(self): print("Starting process to fetch data") while True: - self.queue.put(next(self.generator),True) + self.queue.put(next(self.generator), True) def __next__(self): return self.queue.get(True) - + def next(self): return self.__next__() diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 35181d68..e092d8b7 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -1,3 +1,16 @@ +from __future__ import print_function +from plasma.primitives.ops import mpi_sum_f16 +from plasma.utils.performance import PerformanceAnalyzer +from plasma.utils.processing import concatenate_sublists +from plasma.utils.evaluation import get_loss_from_list +from plasma.models import builder +from plasma.models.loader import ProcessGenerator +from plasma.utils.state_reset import reset_states # , get_states +from plasma.conf import conf +from pprint import pprint +from mpi4py import MPI +# import mpi4py +# import getpass ''' ######################################################### This file trains a deep learning model to predict @@ -14,9 +27,8 @@ ######################################################### ''' -from __future__ import print_function import os -import sys +import sys import time import datetime import numpy as np @@ -25,20 +37,14 @@ from functools import partial import socket sys.setrecursionlimit(10000) -import getpass -#import keras sequentially because it otherwise reads from ~/.keras/keras.json with too many threads. -#from mpi_launch_tensorflow import get_mpi_task_index -import mpi4py -from mpi4py import MPI +# import keras sequentially because it otherwise reads from ~/.keras/keras.json +# with too many threads: +# from mpi_launch_tensorflow import get_mpi_task_index comm = MPI.COMM_WORLD task_index = comm.Get_rank() num_workers = comm.Get_size() -from pprint import pprint -from plasma.conf import conf -from plasma.utils.state_reset import reset_states,get_states -from plasma.models.loader import ProcessGenerator NUM_GPUS = conf['num_gpus'] MY_GPU = task_index % NUM_GPUS @@ -46,511 +52,621 @@ backend = conf['model']['backend'] if backend == 'tf' or backend == 'tensorflow': - if NUM_GPUS > 1: os.environ['CUDA_VISIBLE_DEVICES'] = '{}'.format(MY_GPU)#,mode=NanGuardMode' + if NUM_GPUS > 1: + os.environ['CUDA_VISIBLE_DEVICES'] = '{}'.format( + MY_GPU) # ,mode=NanGuardMode' os.environ['KERAS_BACKEND'] = 'tensorflow' import tensorflow as tf from keras.backend.tensorflow_backend import set_session - gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.95, allow_growth=True) + gpu_options = tf.GPUOptions( + per_process_gpu_memory_fraction=0.95, + allow_growth=True) config = tf.ConfigProto(gpu_options=gpu_options) set_session(tf.Session(config=config)) else: os.environ['KERAS_BACKEND'] = 'theano' - base_compile_dir = '{}/tmp/{}-{}'.format(conf['paths']['output_path'],socket.gethostname(),task_index) - os.environ['THEANO_FLAGS'] = 'device=gpu{},floatX=float32,base_compiledir={}'.format(MY_GPU,base_compile_dir)#,mode=NanGuardMode' - import theano -#import keras + base_compile_dir = '{}/tmp/{}-{}'.format( + conf['paths']['output_path'], socket.gethostname(), task_index) + os.environ['THEANO_FLAGS'] = ( + 'device=gpu{},floatX=float32,base_compiledir={}'.format( + MY_GPU, base_compile_dir)) # ,mode=NanGuardMode' + # import theano +# import keras for i in range(num_workers): - comm.Barrier() - if i == task_index: - print('[{}] importing Keras'.format(task_index)) - from keras import backend as K - from keras.optimizers import * - from keras.utils.generic_utils import Progbar - import keras.callbacks as cbks + comm.Barrier() + if i == task_index: + print('[{}] importing Keras'.format(task_index)) + from keras import backend as K + # from keras.optimizers import * + from keras.utils.generic_utils import Progbar + import keras.callbacks as cbks -from plasma.models import builder -from plasma.utils.evaluation import get_loss_from_list -from plasma.utils.processing import concatenate_sublists -from plasma.utils.performance import PerformanceAnalyzer -from plasma.primitives.ops import mpi_sum_f16 if task_index == 0: pprint(conf) - class MPIOptimizer(object): - def __init__(self,lr): - self.lr = lr - self.iterations = 0 + def __init__(self, lr): + self.lr = lr + self.iterations = 0 + + def get_deltas(self, raw_deltas): + raise NotImplementedError - def get_deltas(self,raw_deltas): - raise NotImplementedError + def set_lr(self, lr): + self.lr = lr - def set_lr(self,lr): - self.lr = lr class MPISGD(MPIOptimizer): - def __init__(self,lr): - super(MPISGD,self).__init__(lr) + def __init__(self, lr): + super(MPISGD, self).__init__(lr) + + def get_deltas(self, raw_deltas): + deltas = [] + for g in raw_deltas: + deltas.append(self.lr*g) - def get_deltas(self,raw_deltas): - deltas = [] - for g in raw_deltas: - deltas.append(self.lr*g) + self.iterations += 1 + return deltas - self.iterations += 1 - return deltas class MPIMomentumSGD(MPIOptimizer): def __init__(self, lr): super(MPIMomentumSGD, self).__init__(lr) self.momentum = 0.9 - def get_deltas(self, raw_deltas): + def get_deltas(self, raw_deltas): deltas = [] if self.iterations == 0: self.velocity_list = [np.zeros_like(g) for g in raw_deltas] - for (i,g) in enumerate(raw_deltas): - self.velocity_list[i] = self.momentum * self.velocity_list[i] + self.lr * g + for (i, g) in enumerate(raw_deltas): + self.velocity_list[i] = ( + self.momentum * self.velocity_list[i] + self.lr * g) deltas.append(self.velocity_list[i]) self.iterations += 1 return deltas + class MPIAdam(MPIOptimizer): - def __init__(self,lr): - super(MPIAdam,self).__init__(lr) - self.beta_1 = 0.9 - self.beta_2 = 0.999 - self.eps = 1e-8 + def __init__(self, lr): + super(MPIAdam, self).__init__(lr) + self.beta_1 = 0.9 + self.beta_2 = 0.999 + self.eps = 1e-8 - def get_deltas(self,raw_deltas): + def get_deltas(self, raw_deltas): - if self.iterations == 0: - self.m_list = [np.zeros_like(g) for g in raw_deltas] - self.v_list = [np.zeros_like(g) for g in raw_deltas] + if self.iterations == 0: + self.m_list = [np.zeros_like(g) for g in raw_deltas] + self.v_list = [np.zeros_like(g) for g in raw_deltas] - t = self.iterations + 1 - lr_t = self.lr * np.sqrt(1-self.beta_2**t)/(1-self.beta_1**t) - deltas = [] - for (i,g) in enumerate(raw_deltas): - m_t = (self.beta_1 * self.m_list[i]) + (1 - self.beta_1) * g - v_t = (self.beta_2 * self.v_list[i]) + (1 - self.beta_2) * (g**2) - delta_t = lr_t * m_t / (np.sqrt(v_t) + self.eps) - deltas.append(delta_t) - self.m_list[i] = m_t - self.v_list[i] = v_t + t = self.iterations + 1 + lr_t = self.lr * np.sqrt(1-self.beta_2**t)/(1-self.beta_1**t) + deltas = [] + for (i, g) in enumerate(raw_deltas): + m_t = (self.beta_1 * self.m_list[i]) + (1 - self.beta_1) * g + v_t = (self.beta_2 * self.v_list[i]) + (1 - self.beta_2) * (g**2) + delta_t = lr_t * m_t / (np.sqrt(v_t) + self.eps) + deltas.append(delta_t) + self.m_list[i] = m_t + self.v_list[i] = v_t - self.iterations += 1 + self.iterations += 1 - return deltas + return deltas class Averager(object): - def __init__(self): - self.steps = 0 - self.val = 0.0 + def __init__(self): + self.steps = 0 + self.val = 0.0 - def add_val(self,val): - self.val = (self.steps * self.val + 1.0 * val)/(self.steps + 1.0) - self.steps += 1 + def add_val(self, val): + self.val = (self.steps * self.val + 1.0 * val)/(self.steps + 1.0) + self.steps += 1 - def get_val(self): - return self.val + def get_val(self): + return self.val class MPIModel(): - def __init__(self,model,optimizer,comm,batch_iterator,batch_size,num_replicas=None,warmup_steps=1000,lr=0.01,num_batches_minimum=100): - random.seed(task_index) - np.random.seed(task_index) - self.start_time = time.time() - self.epoch = 0 - self.num_so_far = 0 - self.num_so_far_accum = 0 - self.num_so_far_indiv = 0 - self.model = model - self.optimizer = optimizer - self.max_lr = 0.1 - self.DUMMY_LR = 0.001 - self.comm = comm - self.batch_size = batch_size - self.batch_iterator = batch_iterator - self.set_batch_iterator_func() - self.warmup_steps=warmup_steps - self.num_batches_minimum=num_batches_minimum - self.num_workers = comm.Get_size() - self.task_index = comm.Get_rank() - self.history = cbks.History() - self.model.stop_training = False - if num_replicas is None or num_replicas < 1 or num_replicas > self.num_workers: - self.num_replicas = self.num_workers - else: - self.num_replicas = num_replicas - self.lr = lr/(1.0+self.num_replicas/100.0) if (lr < self.max_lr) else self.max_lr/(1.0+self.num_replicas/100.0) - - - def set_batch_iterator_func(self): - self.batch_iterator_func = ProcessGenerator(self.batch_iterator()) - - def close(self): - self.batch_iterator_func.__exit__() - - def set_lr(self,lr): - self.lr = lr - - def save_weights(self,path,overwrite=False): - self.model.save_weights(path,overwrite=overwrite) - - def load_weights(self,path): - self.model.load_weights(path) - - def compile(self,optimizer,clipnorm,loss='mse'): - if optimizer == 'sgd': - optimizer_class = SGD(lr=self.DUMMY_LR,clipnorm=clipnorm) - elif optimizer == 'momentum_sgd': - optimizer_class = SGD(lr=self.DUMMY_LR, clipnorm=clipnorm, decay=1e-6, momentum=0.9) - elif optimizer == 'tf_momentum_sgd': - optimizer_class = TFOptimizer(tf.train.MomentumOptimizer(learning_rate=self.DUMMY_LR,momentum=0.9)) - elif optimizer == 'adam': - optimizer_class = Adam(lr=self.DUMMY_LR,clipnorm=clipnorm) - elif optimizer == 'tf_adam': - optimizer_class = TFOptimizer(tf.train.AdamOptimizer(learning_rate=self.DUMMY_LR)) - elif optimizer == 'rmsprop': - optimizer_class = RMSprop(lr=self.DUMMY_LR,clipnorm=clipnorm) - elif optimizer == 'nadam': - optimizer_class = Nadam(lr=self.DUMMY_LR,clipnorm=clipnorm) - else: - print("Optimizer not implemented yet") - exit(1) - self.model.compile(optimizer=optimizer_class,loss=loss) - self.ensure_equal_weights() - - def ensure_equal_weights(self): - if task_index == 0: - new_weights = self.model.get_weights() - else: - new_weights = None - nw = comm.bcast(new_weights,root=0) - self.model.set_weights(nw) - - - - def train_on_batch_and_get_deltas(self,X_batch,Y_batch,verbose=False): - ''' - The purpose of the method is to perform a single gradient update over one mini-batch for one model replica. - Given a mini-batch, it first accesses the current model weights, performs single gradient update over one mini-batch, - gets new model weights, calculates weight updates (deltas) by subtracting weight scalars, applies the learning rate. - - It performs calls to: subtract_params, multiply_params - - Argument list: - - X_batch: input data for one mini-batch as a Numpy array - - Y_batch: labels for one mini-batch as a Numpy array - - verbose: set verbosity level (currently unused) - - Returns: - - deltas: a list of model weight updates - - loss: scalar training loss - ''' - weights_before_update = self.model.get_weights() - - loss = self.model.train_on_batch(X_batch,Y_batch) - - weights_after_update = self.model.get_weights() - self.model.set_weights(weights_before_update) - - #unscale before subtracting - weights_before_update = multiply_params(weights_before_update,1.0/self.DUMMY_LR) - weights_after_update = multiply_params(weights_after_update,1.0/self.DUMMY_LR) - - deltas = subtract_params(weights_after_update,weights_before_update) - - #unscale loss - if conf['model']['loss_scale_factor'] != 1.0: - deltas = multiply_params(deltas,1.0/conf['model']['loss_scale_factor']) - - return deltas,loss - - - def get_new_weights(self,deltas): - return add_params(self.model.get_weights(),deltas) - - def mpi_average_gradients(self,arr,num_replicas=None): - if num_replicas == None: - num_replicas = self.num_workers - if self.task_index >= num_replicas: - arr *= 0.0 - arr_global = np.empty_like(arr) - if K.floatx() == 'float16': - self.comm.Allreduce(arr,arr_global,op=mpi_sum_f16) - else: - self.comm.Allreduce(arr,arr_global,op=MPI.SUM) - arr_global /= num_replicas - return arr_global - - - - def mpi_average_scalars(self,val,num_replicas=None): - ''' - The purpose of the method is to calculate a simple scalar arithmetic mean over num_replicas. - - It performs calls to: MPIModel.mpi_sum_scalars - - Argument list: - - val: value averaged, scalar - - num_replicas: the size of the ensemble an average is perfromed over - - Returns: - - val_global: scalar arithmetic mean over num_replicas - ''' - val_global = self.mpi_sum_scalars(val,num_replicas) - val_global /= num_replicas - return val_global - - - def mpi_sum_scalars(self,val,num_replicas=None): - ''' - The purpose of the method is to calculate a simple scalar arithmetic mean over num_replicas using MPI allreduce action with fixed op=MPI.SIM - - Argument list: - - val: value averaged, scalar - - num_replicas: the size of the ensemble an average is perfromed over - - Returns: - - val_global: scalar arithmetic mean over num_replicas - ''' - if num_replicas == None: - num_replicas = self.num_workers - if self.task_index >= num_replicas: - val *= 0.0 - val_global = 0.0 - val_global = self.comm.allreduce(val,op=MPI.SUM) - return val_global - - - - def sync_deltas(self,deltas,num_replicas=None): - global_deltas = [] - #default is to reduce the deltas from all workers - for delta in deltas: - global_deltas.append(self.mpi_average_gradients(delta,num_replicas)) - return global_deltas - - def set_new_weights(self,deltas,num_replicas=None): - global_deltas = self.sync_deltas(deltas,num_replicas) - effective_lr = self.get_effective_lr(num_replicas) - - self.optimizer.set_lr(effective_lr) - global_deltas = self.optimizer.get_deltas(global_deltas) - - new_weights = self.get_new_weights(global_deltas) - self.model.set_weights(new_weights) - - def build_callbacks(self,conf,callbacks_list): - ''' - The purpose of the method is to set up logging and history. It is based on Keras Callbacks - https://github.com/fchollet/keras/blob/fbc9a18f0abc5784607cd4a2a3886558efa3f794/keras/callbacks.py - - Currently used callbacks include: BaseLogger, CSVLogger, EarlyStopping. - Other possible callbacks to add in future: RemoteMonitor, LearningRateScheduler - - Argument list: - - conf: There is a "callbacks" section in conf.yaml file. Relevant parameters are: - list: Parameter specifying additional callbacks, read in the driver script and passed as an argument of type list (see next arg) - metrics: List of quantities monitored during training and validation - mode: one of {auto, min, max}. The decision to overwrite the current save file is made based on either the maximization or the minimization of the monitored quantity. For val_acc, this should be max, for val_loss this should be min, etc. In auto mode, the direction is automatically inferred from the name of the monitored quantity. - monitor: Quantity used for early stopping, has to be from the list of metrics - patience: Number of epochs used to decide on whether to apply early stopping or continue training - - callbacks_list: uses callbacks.list configuration parameter, specifies the list of additional callbacks - Returns: modified list of callbacks - ''' - - mode = conf['callbacks']['mode'] - monitor = conf['callbacks']['monitor'] - patience = conf['callbacks']['patience'] - csvlog_save_path = conf['paths']['csvlog_save_path'] - #CSV callback is on by default - if not os.path.exists(csvlog_save_path): - os.makedirs(csvlog_save_path) - - callbacks_list = conf['callbacks']['list'] - - callbacks = [cbks.BaseLogger()] - callbacks += [self.history] - callbacks += [cbks.CSVLogger("{}callbacks-{}.log".format(csvlog_save_path,datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")))] - - if "earlystop" in callbacks_list: - callbacks += [cbks.EarlyStopping(patience=patience, monitor=monitor, mode=mode)] - if "lr_scheduler" in callbacks_list: - pass - - return cbks.CallbackList(callbacks) - - def train_epoch(self): - ''' - The purpose of the method is to perform distributed mini-batch SGD for one epoch. - It takes the batch iterator function and a NN model from MPIModel object, fetches mini-batches - in a while-loop until number of samples seen by the ensemble of workers (num_so_far) exceeds the - training dataset size (num_total). - - During each iteration, the gradient updates (deltas) and the loss are calculated for each model replica - in the ensemble, weights are averaged over ensemble, and the new weights are set. - - It performs calls to: MPIModel.get_deltas, MPIModel.set_new_weights methods - - Argument list: Empty - - Returns: - - step: epoch number - - ave_loss: training loss averaged over replicas - - curr_loss: - - num_so_far: the number of samples seen by ensemble of replicas to a current epoch (step) - - Intermediate outputs and logging: debug printout of task_index (MPI), epoch number, number of samples seen to - a current epoch, average training loss - ''' - - verbose = False - first_run = True - step = 0 - loss_averager = Averager() - t_start = time.time() - - batch_iterator_func = self.batch_iterator_func - num_total = 1 - ave_loss = -1 - curr_loss = -1 - t0 = 0 - t1 = 0 - t2 = 0 - - while (self.num_so_far-self.epoch*num_total) < num_total or step < self.num_batches_minimum: - - try: - batch_xs,batch_ys,batches_to_reset,num_so_far_curr,num_total,is_warmup_period = next(batch_iterator_func) - except StopIteration: - print("Resetting batch iterator.") - self.num_so_far_accum = self.num_so_far_indiv - self.set_batch_iterator_func() - batch_iterator_func = self.batch_iterator_func - batch_xs,batch_ys,batches_to_reset,num_so_far_curr,num_total,is_warmup_period = next(batch_iterator_func) - self.num_so_far_indiv = self.num_so_far_accum+num_so_far_curr - - # if batches_to_reset: - # self.model.reset_states(batches_to_reset) - - warmup_phase = (step < self.warmup_steps and self.epoch == 0) - num_replicas = 1 if warmup_phase else self.num_replicas - - self.num_so_far = self.mpi_sum_scalars(self.num_so_far_indiv,num_replicas) - - #run the model once to force compilation. Don't actually use these values. - if first_run: - first_run = False - t0_comp = time.time() - _,_ = self.train_on_batch_and_get_deltas(batch_xs,batch_ys,verbose) - self.comm.Barrier() - sys.stdout.flush() - print_unique('Compilation finished in {:.2f}s'.format(time.time()-t0_comp)) + def __init__( + self, + model, + optimizer, + comm, + batch_iterator, + batch_size, + num_replicas=None, + warmup_steps=1000, + lr=0.01, + num_batches_minimum=100): + random.seed(task_index) + np.random.seed(task_index) + self.start_time = time.time() + self.epoch = 0 + self.num_so_far = 0 + self.num_so_far_accum = 0 + self.num_so_far_indiv = 0 + self.model = model + self.optimizer = optimizer + self.max_lr = 0.1 + self.DUMMY_LR = 0.001 + self.comm = comm + self.batch_size = batch_size + self.batch_iterator = batch_iterator + self.set_batch_iterator_func() + self.warmup_steps = warmup_steps + self.num_batches_minimum = num_batches_minimum + self.num_workers = comm.Get_size() + self.task_index = comm.Get_rank() + self.history = cbks.History() + self.model.stop_training = False + if (num_replicas is None or num_replicas < 1 + or num_replicas > self.num_workers): + self.num_replicas = self.num_workers + else: + self.num_replicas = num_replicas + self.lr = ( + lr/(1.0+self.num_replicas/100.0) if (lr < self.max_lr) + else self.max_lr/(1.0+self.num_replicas/100.0) + ) + + def set_batch_iterator_func(self): + self.batch_iterator_func = ProcessGenerator(self.batch_iterator()) + + def close(self): + self.batch_iterator_func.__exit__() + + def set_lr(self, lr): + self.lr = lr + + def save_weights(self, path, overwrite=False): + self.model.save_weights(path, overwrite=overwrite) + + def load_weights(self, path): + self.model.load_weights(path) + + def compile(self, optimizer, clipnorm, loss='mse'): + # TODO(KGF): check the following import taken from runner.py + # Was not in this file, originally. + from keras.optimizers import SGD, Adam, RMSprop, Nadam, TFOptimizer + if optimizer == 'sgd': + optimizer_class = SGD(lr=self.DUMMY_LR, clipnorm=clipnorm) + elif optimizer == 'momentum_sgd': + optimizer_class = SGD( + lr=self.DUMMY_LR, + clipnorm=clipnorm, + decay=1e-6, + momentum=0.9) + elif optimizer == 'tf_momentum_sgd': + optimizer_class = TFOptimizer( + tf.train.MomentumOptimizer( + learning_rate=self.DUMMY_LR, + momentum=0.9)) + elif optimizer == 'adam': + optimizer_class = Adam(lr=self.DUMMY_LR, clipnorm=clipnorm) + elif optimizer == 'tf_adam': + optimizer_class = TFOptimizer( + tf.train.AdamOptimizer( + learning_rate=self.DUMMY_LR)) + elif optimizer == 'rmsprop': + optimizer_class = RMSprop(lr=self.DUMMY_LR, clipnorm=clipnorm) + elif optimizer == 'nadam': + optimizer_class = Nadam(lr=self.DUMMY_LR, clipnorm=clipnorm) + else: + print("Optimizer not implemented yet") + exit(1) + self.model.compile(optimizer=optimizer_class, loss=loss) + self.ensure_equal_weights() + + def ensure_equal_weights(self): + if task_index == 0: + new_weights = self.model.get_weights() + else: + new_weights = None + nw = comm.bcast(new_weights, root=0) + self.model.set_weights(nw) + + def train_on_batch_and_get_deltas(self, X_batch, Y_batch, verbose=False): + ''' + The purpose of the method is to perform a single gradient update over + one mini-batch for one model replica. Given a mini-batch, it first + accesses the current model weights, performs single gradient update + over one mini-batch, gets new model weights, calculates weight updates + (deltas) by subtracting weight scalars, applies the learning rate. + + It performs calls to: subtract_params, multiply_params + + Argument list: + - X_batch: input data for one mini-batch as a Numpy array + - Y_batch: labels for one mini-batch as a Numpy array + - verbose: set verbosity level (currently unused) + + Returns: + - deltas: a list of model weight updates + - loss: scalar training loss + + ''' + weights_before_update = self.model.get_weights() + + loss = self.model.train_on_batch(X_batch, Y_batch) + + weights_after_update = self.model.get_weights() + self.model.set_weights(weights_before_update) + + # unscale before subtracting + weights_before_update = multiply_params( + weights_before_update, 1.0/self.DUMMY_LR) + weights_after_update = multiply_params( + weights_after_update, 1.0/self.DUMMY_LR) + + deltas = subtract_params(weights_after_update, weights_before_update) + + # unscale loss + if conf['model']['loss_scale_factor'] != 1.0: + deltas = multiply_params( + deltas, 1.0/conf['model']['loss_scale_factor']) + + return deltas, loss + + def get_new_weights(self, deltas): + return add_params(self.model.get_weights(), deltas) + + def mpi_average_gradients(self, arr, num_replicas=None): + if num_replicas is None: + num_replicas = self.num_workers + if self.task_index >= num_replicas: + arr *= 0.0 + arr_global = np.empty_like(arr) + if K.floatx() == 'float16': + self.comm.Allreduce(arr, arr_global, op=mpi_sum_f16) + else: + self.comm.Allreduce(arr, arr_global, op=MPI.SUM) + arr_global /= num_replicas + return arr_global + + def mpi_average_scalars(self, val, num_replicas=None): + ''' + The purpose of the method is to calculate a simple scalar arithmetic + mean over num_replicas. + + It performs calls to: MPIModel.mpi_sum_scalars + + Argument list: + - val: value averaged, scalar + - num_replicas: the size of the ensemble an average is perfromed over + + Returns: + - val_global: scalar arithmetic mean over num_replicas + ''' + val_global = self.mpi_sum_scalars(val, num_replicas) + val_global /= num_replicas + return val_global + + def mpi_sum_scalars(self, val, num_replicas=None): + ''' + The purpose of the method is to calculate a simple scalar arithmetic + mean over num_replicas using MPI allreduce action with fixed op=MPI.SIM + + Argument list: + - val: value averaged, scalar + - num_replicas: the size of the ensemble an average is perfromed over + + Returns: + - val_global: scalar arithmetic mean over num_replicas + ''' + if num_replicas is None: + num_replicas = self.num_workers + if self.task_index >= num_replicas: + val *= 0.0 + val_global = 0.0 + val_global = self.comm.allreduce(val, op=MPI.SUM) + return val_global + + def sync_deltas(self, deltas, num_replicas=None): + global_deltas = [] + # default is to reduce the deltas from all workers + for delta in deltas: + global_deltas.append( + self.mpi_average_gradients( + delta, num_replicas)) + return global_deltas + + def set_new_weights(self, deltas, num_replicas=None): + global_deltas = self.sync_deltas(deltas, num_replicas) + effective_lr = self.get_effective_lr(num_replicas) + + self.optimizer.set_lr(effective_lr) + global_deltas = self.optimizer.get_deltas(global_deltas) + + new_weights = self.get_new_weights(global_deltas) + self.model.set_weights(new_weights) + + def build_callbacks(self, conf, callbacks_list): + ''' + The purpose of the method is to set up logging and history. It is based + on Keras Callbacks + https://github.com/fchollet/keras/blob/fbc9a18f0abc5784607cd4a2a3886558efa3f794/keras/callbacks.py + + Currently used callbacks include: BaseLogger, CSVLogger, EarlyStopping. + Other possible callbacks to add in future: RemoteMonitor, + LearningRateScheduler + + Argument list: + - conf: There is a "callbacks" section in conf.yaml file. + + Relevant parameters are: + - list: Parameter specifying additional callbacks, read + in the driver script and passed as an argument of type list (see next + arg) + + - metrics: List of quantities monitored during training and validation + + - mode: one of {auto, min, max}. The decision to overwrite the current + save file is made based on either the maximization or the minimization + of the monitored quantity. For val_acc, this should be max, for + val_loss this should be min, etc. In auto mode, the direction is + automatically inferred from the name of the monitored quantity. + + -monitor: Quantity used for early stopping, has to + be from the list of metrics + + - patience: Number of epochs used to decide on whether to apply early + stopping or continue training + + - callbacks_list: uses callbacks.list configuration parameter, + specifies the list of additional callbacks Returns: modified list of + callbacks + + ''' + + mode = conf['callbacks']['mode'] + monitor = conf['callbacks']['monitor'] + patience = conf['callbacks']['patience'] + csvlog_save_path = conf['paths']['csvlog_save_path'] + # CSV callback is on by default + if not os.path.exists(csvlog_save_path): + os.makedirs(csvlog_save_path) + + callbacks_list = conf['callbacks']['list'] + + callbacks = [cbks.BaseLogger()] + callbacks += [self.history] + callbacks += [cbks.CSVLogger("{}callbacks-{}.log".format( + csvlog_save_path, + datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")))] + + if "earlystop" in callbacks_list: + callbacks += [ + cbks.EarlyStopping( + patience=patience, + monitor=monitor, + mode=mode)] + if "lr_scheduler" in callbacks_list: + pass + + return cbks.CallbackList(callbacks) + + def train_epoch(self): + ''' + The purpose of the method is to perform distributed mini-batch SGD for + one epoch. It takes the batch iterator function and a NN model from + MPIModel object, fetches mini-batches in a while-loop until number of + samples seen by the ensemble of workers (num_so_far) exceeds the + training dataset size (num_total). + + During each iteration, the gradient updates (deltas) and the loss are + calculated for each model replica in the ensemble, weights are averaged + over ensemble, and the new weights are set. + + It performs calls to: MPIModel.get_deltas, MPIModel.set_new_weights + methods + + Argument list: Empty + + Returns: + - step: epoch number + - ave_loss: training loss averaged over replicas + - curr_loss: + - num_so_far: the number of samples seen by ensemble of replicas to a + current epoch (step) + + + Intermediate outputs and logging: debug printout of task_index (MPI), + epoch number, number of samples seen to a current epoch, average + training loss + + ''' + + verbose = False + first_run = True + step = 0 + loss_averager = Averager() t_start = time.time() - sys.stdout.flush() - - if np.any(batches_to_reset): - reset_states(self.model,batches_to_reset) - - t0 = time.time() - deltas,loss = self.train_on_batch_and_get_deltas(batch_xs,batch_ys,verbose) - t1 = time.time() - if not is_warmup_period: - self.set_new_weights(deltas,num_replicas) - t2 = time.time() - write_str_0 = self.calculate_speed(t0,t1,t2,num_replicas) - - curr_loss = self.mpi_average_scalars(1.0*loss,num_replicas) - #if self.task_index == 0: - #print(self.model.get_weights()[0][0][:4]) - loss_averager.add_val(curr_loss) - ave_loss = loss_averager.get_val() - eta = self.estimate_remaining_time(t0 - t_start,self.num_so_far-self.epoch*num_total,num_total) - write_str = '\r[{}] step: {} [ETA: {:.2f}s] [{:.2f}/{}], loss: {:.5f} [{:.5f}] | walltime: {:.4f} | '.format(self.task_index,step,eta,1.0*self.num_so_far,num_total,ave_loss,curr_loss,time.time()-self.start_time) - print_unique(write_str + write_str_0) - step += 1 - else: - print_unique('\r[{}] warmup phase, num so far: {}'.format(self.task_index,self.num_so_far)) - - - - - effective_epochs = 1.0*self.num_so_far/num_total - epoch_previous = self.epoch - self.epoch = effective_epochs - print_unique('\nEpoch {:.2f} finished ({:.2f} epochs passed) in {:.2f} seconds.\n'.format(1.0*self.epoch,self.epoch-epoch_previous,t2 - t_start)) - return (step,ave_loss,curr_loss,self.num_so_far,effective_epochs) - - - def estimate_remaining_time(self,time_so_far,work_so_far,work_total): - eps = 1e-6 - total_time = 1.0*time_so_far*work_total/(work_so_far + eps) - return total_time - time_so_far - - def get_effective_lr(self,num_replicas): - effective_lr = self.lr * num_replicas - if effective_lr > self.max_lr: - print_unique('Warning: effective learning rate set to {}, larger than maximum {}. Clipping.'.format(effective_lr,self.max_lr)) - effective_lr = self.max_lr - return effective_lr - - def get_effective_batch_size(self,num_replicas): - return self.batch_size*num_replicas - - def calculate_speed(self,t0,t_after_deltas,t_after_update,num_replicas,verbose=False): - effective_batch_size = self.get_effective_batch_size(num_replicas) - t_calculate = t_after_deltas - t0 - t_sync = t_after_update - t_after_deltas - t_tot = t_after_update - t0 - - examples_per_sec = effective_batch_size/t_tot - frac_calculate = t_calculate/t_tot - frac_sync = t_sync/t_tot - - print_str = '{:.2E} Examples/sec | {:.2E} sec/batch [{:.1%} calc., {:.1%} synch.]'.format(examples_per_sec,t_tot,frac_calculate,frac_sync) - print_str += '[batch = {} = {}*{}] [lr = {:.2E} = {:.2E}*{}]'.format(effective_batch_size,self.batch_size,num_replicas,self.get_effective_lr(num_replicas),self.lr,num_replicas) - if verbose: - print_unique(print_str) - return print_str + batch_iterator_func = self.batch_iterator_func + num_total = 1 + ave_loss = -1 + curr_loss = -1 + t0 = 0 + t1 = 0 + t2 = 0 + + while ( + self.num_so_far + - self.epoch + * num_total) < num_total or step < self.num_batches_minimum: + + try: + (batch_xs, batch_ys, batches_to_reset, num_so_far_curr, + num_total, is_warmup_period) = next(batch_iterator_func) + except StopIteration: + print("Resetting batch iterator.") + self.num_so_far_accum = self.num_so_far_indiv + self.set_batch_iterator_func() + batch_iterator_func = self.batch_iterator_func + (batch_xs, batch_ys, batches_to_reset, num_so_far_curr, + num_total, is_warmup_period) = next(batch_iterator_func) + self.num_so_far_indiv = self.num_so_far_accum + num_so_far_curr + + # if batches_to_reset: + # self.model.reset_states(batches_to_reset) + + warmup_phase = (step < self.warmup_steps and self.epoch == 0) + num_replicas = 1 if warmup_phase else self.num_replicas + + self.num_so_far = self.mpi_sum_scalars( + self.num_so_far_indiv, num_replicas) + + # run the model once to force compilation. Don't actually use these + # values. + if first_run: + first_run = False + t0_comp = time.time() + _, _ = self.train_on_batch_and_get_deltas( + batch_xs, batch_ys, verbose) + self.comm.Barrier() + sys.stdout.flush() + print_unique( + 'Compilation finished in {:.2f}s'.format( + time.time()-t0_comp)) + t_start = time.time() + sys.stdout.flush() + + if np.any(batches_to_reset): + reset_states(self.model, batches_to_reset) + + t0 = time.time() + deltas, loss = self.train_on_batch_and_get_deltas( + batch_xs, batch_ys, verbose) + t1 = time.time() + if not is_warmup_period: + self.set_new_weights(deltas, num_replicas) + t2 = time.time() + write_str_0 = self.calculate_speed(t0, t1, t2, num_replicas) + + curr_loss = self.mpi_average_scalars(1.0*loss, num_replicas) + # if self.task_index == 0: + # print(self.model.get_weights()[0][0][:4]) + loss_averager.add_val(curr_loss) + ave_loss = loss_averager.get_val() + eta = self.estimate_remaining_time( + t0 - t_start, + self.num_so_far - self.epoch*num_total, num_total) + write_str = ( + '\r[{}] step: {} [ETA: {:.2f}s] [{:.2f}/{}], '.format( + self.task_index, step, eta, + 1.0*self.num_so_far, num_total) + + 'loss: {:.5f} [{:.5f}] | '.format(ave_loss, curr_loss) + + 'walltime: {:.4f} | '.format( + time.time() - self.start_time)) + print_unique(write_str + write_str_0) + step += 1 + else: + print_unique('\r[{}] warmup phase, num so far: {}'.format( + self.task_index, self.num_so_far)) + + effective_epochs = 1.0*self.num_so_far/num_total + epoch_previous = self.epoch + self.epoch = effective_epochs + print_unique('\nEpoch {:.2f} finished ({:.2f} epochs passed)'.format( + 1.0 * self.epoch, self.epoch - epoch_previous) + + ' in {:.2f} seconds.\n'.format(t2 - t_start)) + return (step, ave_loss, curr_loss, self.num_so_far, effective_epochs) + + def estimate_remaining_time(self, time_so_far, work_so_far, work_total): + eps = 1e-6 + total_time = 1.0*time_so_far*work_total/(work_so_far + eps) + return total_time - time_so_far + + def get_effective_lr(self, num_replicas): + effective_lr = self.lr * num_replicas + if effective_lr > self.max_lr: + print_unique('Warning: effective learning rate set to {}, '.format( + effective_lr) + + 'larger than maximum {}. Clipping.'.format( + self.max_lr)) + effective_lr = self.max_lr + return effective_lr + + def get_effective_batch_size(self, num_replicas): + return self.batch_size*num_replicas + + def calculate_speed( + self, + t0, + t_after_deltas, + t_after_update, + num_replicas, + verbose=False): + effective_batch_size = self.get_effective_batch_size(num_replicas) + t_calculate = t_after_deltas - t0 + t_sync = t_after_update - t_after_deltas + t_tot = t_after_update - t0 + + examples_per_sec = effective_batch_size/t_tot + frac_calculate = t_calculate/t_tot + frac_sync = t_sync/t_tot + + print_str = ( + '{:.2E} Examples/sec | {:.2E} sec/batch '.format(examples_per_sec, + t_tot) + + '[{:.1%} calc., {:.1%} synch.]'.format(frac_calculate, + frac_sync)) + print_str += '[batch = {} = {}*{}] [lr = {:.2E} = {:.2E}*{}]'.format( + effective_batch_size, + self.batch_size, + num_replicas, + self.get_effective_lr(num_replicas), + self.lr, + num_replicas) + if verbose: + print_unique(print_str) + return print_str def print_unique(print_str): - if task_index == 0: - sys.stdout.write(print_str) - sys.stdout.flush() + if task_index == 0: + sys.stdout.write(print_str) + sys.stdout.flush() + def print_all(print_str): - sys.stdout.write('[{}] '.format(task_index) + print_str) - sys.stdout.flush() + sys.stdout.write('[{}] '.format(task_index) + print_str) + sys.stdout.flush() + +def multiply_params(params, eps): + return [el*eps for el in params] -def multiply_params(params,eps): - return [el*eps for el in params] -def subtract_params(params1,params2): - return [p1 - p2 for p1,p2 in zip(params1,params2)] +def subtract_params(params1, params2): + return [p1 - p2 for p1, p2 in zip(params1, params2)] -def add_params(params1,params2): - return [p1 + p2 for p1,p2 in zip(params1,params2)] + +def add_params(params1, params2): + return [p1 + p2 for p1, p2 in zip(params1, params2)] def get_shot_list_path(conf): - return conf['paths']['base_path'] + '/normalization/shot_lists.npz' #kyle: not compatible with flexible conf.py hierarchy + # KGF: not compatible with flexible conf.py hierarchy + return conf['paths']['base_path'] + '/normalization/shot_lists.npz' + -def save_shotlists(conf,shot_list_train,shot_list_validate,shot_list_test): +def save_shotlists(conf, shot_list_train, shot_list_validate, shot_list_test): path = get_shot_list_path(conf) - np.savez(path,shot_list_train=shot_list_train,shot_list_validate=shot_list_validate,shot_list_test=shot_list_test) + np.savez( + path, + shot_list_train=shot_list_train, + shot_list_validate=shot_list_validate, + shot_list_test=shot_list_test) + def load_shotlists(conf): path = get_shot_list_path(conf) @@ -558,36 +674,40 @@ def load_shotlists(conf): shot_list_train = data['shot_list_train'][()] shot_list_validate = data['shot_list_validate'][()] shot_list_test = data['shot_list_test'][()] - return shot_list_train,shot_list_validate,shot_list_test + return shot_list_train, shot_list_validate, shot_list_test + +# shot_list_train, shot_list_validate, shot_list_test = load_shotlists(conf) -#shot_list_train,shot_list_validate,shot_list_test = load_shotlists(conf) -def mpi_make_predictions(conf,shot_list,loader,custom_path=None): +def mpi_make_predictions(conf, shot_list, loader, custom_path=None): loader.set_inference_mode(True) np.random.seed(task_index) - shot_list.sort()#make sure all replicas have the same list - specific_builder = builder.ModelBuilder(conf) + shot_list.sort() # make sure all replicas have the same list + specific_builder = builder.ModelBuilder(conf) y_prime = [] y_gold = [] disruptive = [] model = specific_builder.build_model(True) - specific_builder.load_model_weights(model,custom_path) + specific_builder.load_model_weights(model, custom_path) - #broadcast model weights then set it explicitely: fix for Py3.6 + # broadcast model weights then set it explicitely: fix for Py3.6 if sys.version_info[0] > 2: if task_index == 0: new_weights = model.get_weights() else: new_weights = None - nw = comm.bcast(new_weights,root=0) + nw = comm.bcast(new_weights, root=0) model.set_weights(nw) model.reset_states() if task_index == 0: - pbar = Progbar(len(shot_list)) - shot_sublists = shot_list.sublists(conf['model']['pred_batch_size'],do_shuffle=False,equal_size=True) + pbar = Progbar(len(shot_list)) + shot_sublists = shot_list.sublists( + conf['model']['pred_batch_size'], + do_shuffle=False, + equal_size=True) y_prime_global = [] y_gold_global = [] @@ -595,39 +715,36 @@ def mpi_make_predictions(conf,shot_list,loader,custom_path=None): if task_index != 0: loader.verbose = False - for (i,shot_sublist) in enumerate(shot_sublists): + for (i, shot_sublist) in enumerate(shot_sublists): if i % num_workers == task_index: - X,y,shot_lengths,disr = loader.load_as_X_y_pred(shot_sublist) - + X, y, shot_lengths, disr = loader.load_as_X_y_pred(shot_sublist) - - #load data and fit on data - y_p = model.predict(X,batch_size=conf['model']['pred_batch_size']) + # load data and fit on data + y_p = model.predict(X, batch_size=conf['model']['pred_batch_size']) model.reset_states() y_p = loader.batch_output_to_array(y_p) y = loader.batch_output_to_array(y) - #cut arrays back - y_p = [arr[:shot_lengths[j]] for (j,arr) in enumerate(y_p)] - y = [arr[:shot_lengths[j]] for (j,arr) in enumerate(y)] + # cut arrays back + y_p = [arr[:shot_lengths[j]] for (j, arr) in enumerate(y_p)] + y = [arr[:shot_lengths[j]] for (j, arr) in enumerate(y)] - # print('Shots {}/{}'.format(i*num_at_once + j*1.0*len(shot_sublist)/len(X_list),len(shot_list_train))) y_prime += y_p y_gold += y disruptive += disr # print_all('\nFinished with i = {}'.format(i)) - if i % num_workers == num_workers -1 or i == len(shot_sublists) - 1: + if i % num_workers == num_workers - 1 or i == len(shot_sublists) - 1: comm.Barrier() y_prime_global += concatenate_sublists(comm.allgather(y_prime)) y_gold_global += concatenate_sublists(comm.allgather(y_gold)) - disruptive_global += concatenate_sublists(comm.allgather(disruptive)) + disruptive_global += concatenate_sublists( + comm.allgather(disruptive)) comm.Barrier() y_prime = [] y_gold = [] disruptive = [] - # print_all('\nFinished subepoch with lists len(y_prime_global), gold, disruptive = {},{},{}'.format(len(y_prime_global),len(y_gold_global),len(disruptive_global))) if task_index == 0: pbar.add(1.0*len(shot_sublist)) @@ -637,19 +754,28 @@ def mpi_make_predictions(conf,shot_list,loader,custom_path=None): disruptive_global = disruptive_global[:len(shot_list)] loader.set_inference_mode(False) - return y_prime_global,y_gold_global,disruptive_global + return y_prime_global, y_gold_global, disruptive_global -def mpi_make_predictions_and_evaluate(conf,shot_list,loader,custom_path=None): - y_prime,y_gold,disruptive = mpi_make_predictions(conf,shot_list,loader,custom_path) +def mpi_make_predictions_and_evaluate( + conf, shot_list, loader, custom_path=None): + y_prime, y_gold, disruptive = mpi_make_predictions( + conf, shot_list, loader, custom_path) analyzer = PerformanceAnalyzer(conf=conf) - roc_area = analyzer.get_roc_area(y_prime,y_gold,disruptive) - shot_list.set_weights(analyzer.get_shot_difficulty(y_prime,y_gold,disruptive)) - loss = get_loss_from_list(y_prime,y_gold,conf['data']['target']) - return y_prime,y_gold,disruptive,roc_area,loss + roc_area = analyzer.get_roc_area(y_prime, y_gold, disruptive) + shot_list.set_weights( + analyzer.get_shot_difficulty( + y_prime, y_gold, disruptive)) + loss = get_loss_from_list(y_prime, y_gold, conf['data']['target']) + return y_prime, y_gold, disruptive, roc_area, loss -def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=None): +def mpi_train( + conf, + shot_list_train, + shot_list_validate, + loader, + callbacks_list=None): loader.set_inference_mode(False) conf['num_workers'] = comm.Get_size() @@ -657,7 +783,7 @@ def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=Non specific_builder = builder.ModelBuilder(conf) train_model = specific_builder.build_model(False) - #load the latest epoch we did. Returns -1 if none exist yet + # load the latest epoch we did. Returns -1 if none exist yet e = specific_builder.load_model_weights(train_model) e_old = e @@ -671,7 +797,8 @@ def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=Non if 'adam' in conf['model']['optimizer']: optimizer = MPIAdam(lr=lr) - elif conf['model']['optimizer'] == 'sgd' or conf['model']['optimizer'] == 'tf_sgd': + elif (conf['model']['optimizer'] == 'sgd' + or conf['model']['optimizer'] == 'tf_sgd'): optimizer = MPISGD(lr=lr) elif 'momentum_sgd' in conf['model']['optimizer']: optimizer = MPIMomentumSGD(lr=lr) @@ -681,30 +808,45 @@ def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=Non print('{} epochs left to go'.format(num_epochs - 1 - e)) - # batch_generator = partial(loader.training_batch_generator,shot_list=shot_list_train) - batch_generator = partial(loader.training_batch_generator_partial_reset,shot_list=shot_list_train) - #{}batch_generator = partial(loader.training_batch_generator_process,shot_list=shot_list_train) + batch_generator = partial( + loader.training_batch_generator_partial_reset, + shot_list=shot_list_train) print("warmup {}".format(warmup_steps)) - mpi_model = MPIModel(train_model,optimizer,comm,batch_generator,batch_size,lr=lr,warmup_steps = warmup_steps,num_batches_minimum=num_batches_minimum) - mpi_model.compile(conf['model']['optimizer'],clipnorm,conf['data']['target'].loss) + mpi_model = MPIModel( + train_model, + optimizer, + comm, + batch_generator, + batch_size, + lr=lr, + warmup_steps=warmup_steps, + num_batches_minimum=num_batches_minimum) + mpi_model.compile( + conf['model']['optimizer'], + clipnorm, + conf['data']['target'].loss) tensorboard = None if backend != "theano" and task_index == 0: tensorboard_save_path = conf['paths']['tensorboard_save_path'] write_grads = conf['callbacks']['write_grads'] - tensorboard = TensorBoard(log_dir=tensorboard_save_path,histogram_freq=1,write_graph=True,write_grads=write_grads) + tensorboard = TensorBoard( + log_dir=tensorboard_save_path, + histogram_freq=1, + write_graph=True, + write_grads=write_grads) tensorboard.set_model(mpi_model.model) mpi_model.model.summary() if task_index == 0: - callbacks = mpi_model.build_callbacks(conf,callbacks_list) + callbacks = mpi_model.build_callbacks(conf, callbacks_list) callbacks.set_model(mpi_model.model) callback_metrics = conf['callbacks']['metrics'] callbacks.set_params({ - 'epochs': num_epochs, - 'metrics': callback_metrics, - 'batch_size': batch_size, + 'epochs': num_epochs, + 'metrics': callback_metrics, + 'batch_size': batch_size, }) callbacks.on_train_begin() if conf['callbacks']['mode'] == 'max': @@ -718,29 +860,36 @@ def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=Non if task_index == 0: callbacks.on_epoch_begin(int(round(e))) mpi_model.set_lr(lr*lr_decay**e) - print_unique('\nEpoch {}/{}'.format(e,num_epochs)) + print_unique('\nEpoch {}/{}'.format(e, num_epochs)) - (step,ave_loss,curr_loss,num_so_far,effective_epochs) = mpi_model.train_epoch() + (step, ave_loss, curr_loss, num_so_far, + effective_epochs) = mpi_model.train_epoch() e = e_old + effective_epochs - loader.verbose=False #True during the first iteration - if task_index == 0: - specific_builder.save_model_weights(train_model,int(round(e))) + loader.verbose = False # True during the first iteration + if task_index == 0: + specific_builder.save_model_weights(train_model, int(round(e))) epoch_logs = {} - - _,_,_,roc_area,loss = mpi_make_predictions_and_evaluate(conf,shot_list_validate,loader) + + _, _, _, roc_area, loss = mpi_make_predictions_and_evaluate( + conf, shot_list_validate, loader) if conf['training']['ranking_difficulty_fac'] != 1.0: - _,_,_,roc_area_train,loss_train = mpi_make_predictions_and_evaluate(conf,shot_list_train,loader) - batch_generator = partial(loader.training_batch_generator_partial_reset,shot_list=shot_list_train) + (_, _, _, roc_area_train, + loss_train) = mpi_make_predictions_and_evaluate( + conf, shot_list_train, loader) + batch_generator = partial( + loader.training_batch_generator_partial_reset, + shot_list=shot_list_train) mpi_model.batch_iterator = batch_generator mpi_model.batch_iterator_func.__exit__() mpi_model.num_so_far_accum = mpi_model.num_so_far_indiv mpi_model.set_batch_iterator_func() - epoch_logs['val_roc'] = roc_area + epoch_logs['val_roc'] = roc_area epoch_logs['val_loss'] = loss epoch_logs['train_loss'] = ave_loss - best_so_far = cmp_fn(epoch_logs[conf['callbacks']['monitor']],best_so_far) + best_so_far = cmp_fn( + epoch_logs[conf['callbacks']['monitor']], best_so_far) if task_index == 0: print('=========Summary======== for epoch{}'.format(step)) @@ -752,17 +901,23 @@ def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=Non print('Training ROC: {:.4f}'.format(roc_area_train)) callbacks.on_epoch_end(int(round(e)), epoch_logs) - if best_so_far != epoch_logs[conf['callbacks']['monitor']]: #only save model weights if quantity we are tracking is improving + # only save model weights if quantity we are tracking is improving + if best_so_far != epoch_logs[conf['callbacks']['monitor']]: print("Not saving model weights") - specific_builder.delete_model_weights(train_model,int(round(e))) + specific_builder.delete_model_weights( + train_model, int(round(e))) - #tensorboard + # tensorboard if backend != 'theano': - val_generator = partial(loader.training_batch_generator,shot_list=shot_list_validate)() + val_generator = partial( + loader.training_batch_generator, + shot_list=shot_list_validate)() val_steps = 1 - tensorboard.on_epoch_end(val_generator,val_steps,int(round(e)),epoch_logs) + tensorboard.on_epoch_end( + val_generator, val_steps, int( + round(e)), epoch_logs) - stop_training = comm.bcast(mpi_model.model.stop_training,root=0) + stop_training = comm.bcast(mpi_model.model.stop_training, root=0) if stop_training: print("Stopping training due to early stopping") break @@ -776,12 +931,13 @@ def mpi_train(conf,shot_list_train,shot_list_validate,loader, callbacks_list=Non def get_stop_training(callbacks): for cb in callbacks.callbacks: - if isinstance(cb,cbks.EarlyStopping): + if isinstance(cb, cbks.EarlyStopping): print("Checking for early stopping") return cb.model.stop_training print("No early stopping callback found.") return False + class TensorBoard(object): def __init__(self, log_dir='./logs', histogram_freq=0, @@ -812,28 +968,29 @@ def set_model(self, model): mapped_weight_name = weight.name.replace(':', '_') tf.summary.histogram(mapped_weight_name, weight) if self.write_grads: - grads = self.model.optimizer.get_gradients(self.model.total_loss, - weight) + grads = self.model.optimizer.get_gradients( + self.model.total_loss, weight) + def is_indexed_slices(grad): return type(grad).__name__ == 'IndexedSlices' grads = [ grad.values if is_indexed_slices(grad) else grad for grad in grads] - for grad in grads: - tf.summary.histogram('{}_grad'.format(mapped_weight_name), grad) + for grad in grads: + tf.summary.histogram( + '{}_grad'.format(mapped_weight_name), grad) if hasattr(layer, 'output'): tf.summary.histogram('{}_out'.format(layer.name), - layer.output) + layer.output) self.merged = tf.summary.merge_all() if self.write_graph: self.writer = tf.summary.FileWriter(self.log_dir, - self.sess.graph) + self.sess.graph) else: self.writer = tf.summary.FileWriter(self.log_dir) - def on_epoch_end(self, val_generator, val_steps, epoch, logs=None): logs = logs or {} @@ -847,9 +1004,9 @@ def on_epoch_end(self, val_generator, val_steps, epoch, logs=None): self.writer.add_summary(summary, epoch) self.writer.flush() - tensors = (self.model.inputs + - self.model.targets + - self.model.sample_weights) + tensors = (self.model.inputs + + self.model.targets + + self.model.sample_weights) if self.model.uses_learning_phase: tensors += [K.learning_phase()] @@ -870,8 +1027,8 @@ def on_epoch_end(self, val_generator, val_steps, epoch, logs=None): summary_str = result[0] self.writer.add_summary(summary_str, int(round(epoch))) val_steps -= 1 - if val_steps <= 0: break - + if val_steps <= 0: + break def on_train_end(self): self.writer.close() diff --git a/plasma/models/runner.py b/plasma/models/runner.py index 11f3ff3d..2eb0b58b 100644 --- a/plasma/models/runner.py +++ b/plasma/models/runner.py @@ -1,71 +1,78 @@ -from __future__ import print_function -import matplotlib -matplotlib.use('Agg') -import matplotlib.pyplot as plt - -import numpy as np - -from hyperopt import hp, STATUS_OK - +from plasma.utils.state_reset import reset_states +from plasma.utils.evaluation import lr, tf, get_loss_from_list +from plasma.utils.performance import PerformanceAnalyzer +from plasma.models.loader import Loader, ProcessGenerator +from plasma.conf import conf +import pathos.multiprocessing as mp +from functools import partial +import os import time +from hyperopt import hp, STATUS_OK +import numpy as np import sys -import os -from functools import partial -import pathos.multiprocessing as mp +import matplotlib.pyplot as plt +import matplotlib +matplotlib.use('Agg') -if sys.version_info[0] < 3: - from itertools import imap +# if sys.version_info[0] < 3: +# from itertools import imap -from plasma.conf import conf -from plasma.models.loader import Loader, ProcessGenerator -from plasma.utils.performance import PerformanceAnalyzer -from plasma.utils.evaluation import * -from plasma.utils.state_reset import reset_states backend = conf['model']['backend'] -def train(conf,shot_list_train,shot_list_validate,loader): + +def train(conf, shot_list_train, shot_list_validate, loader): loader.set_inference_mode(False) np.random.seed(1) validation_losses = [] validation_roc = [] training_losses = [] - print('validate: {} shots, {} disruptive'.format(len(shot_list_validate),shot_list_validate.num_disruptive())) - print('training: {} shots, {} disruptive'.format(len(shot_list_train),shot_list_train.num_disruptive())) + print( + 'validate: {} shots, {} disruptive'.format( + len(shot_list_validate), + shot_list_validate.num_disruptive())) + print( + 'training: {} shots, {} disruptive'.format( + len(shot_list_train), + shot_list_train.num_disruptive())) if backend == 'tf' or backend == 'tensorflow': first_time = "tensorflow" not in sys.modules if first_time: - import tensorflow as tf - os.environ['KERAS_BACKEND'] = 'tensorflow' - from keras.backend.tensorflow_backend import set_session - config = tf.ConfigProto(device_count={"GPU":1}) - set_session(tf.Session(config=config)) + import tensorflow as tf + os.environ['KERAS_BACKEND'] = 'tensorflow' + from keras.backend.tensorflow_backend import set_session + config = tf.ConfigProto(device_count={"GPU": 1}) + set_session(tf.Session(config=config)) else: os.environ['KERAS_BACKEND'] = 'theano' os.environ['THEANO_FLAGS'] = 'device=gpu,floatX=float32' - import theano + # import theano - from keras.utils.generic_utils import Progbar + from keras.utils.generic_utils import Progbar from keras import backend as K from plasma.models import builder - print('Build model...',end='') + print('Build model...', end='') specific_builder = builder.ModelBuilder(conf) - train_model = specific_builder.build_model(False) - print('Compile model',end='') - train_model.compile(optimizer=optimizer_class(),loss=conf['data']['target'].loss) + train_model = specific_builder.build_model(False) + print('Compile model', end='') + train_model.compile( + optimizer=optimizer_class(), + loss=conf['data']['target'].loss) print('...done') - #load the latest epoch we did. Returns -1 if none exist yet + # load the latest epoch we did. Returns -1 if none exist yet e = specific_builder.load_model_weights(train_model) e_start = e - batch_generator = partial(loader.training_batch_generator_partial_reset,shot_list=shot_list_train) + batch_generator = partial( + loader.training_batch_generator_partial_reset, + shot_list=shot_list_train) batch_iterator = ProcessGenerator(batch_generator()) num_epochs = conf['training']['num_epochs'] - num_at_once = conf['training']['num_shots_at_once'] + # num_at_once = conf['training']['num_shots_at_once'] lr_decay = conf['model']['lr_decay'] print('{} epochs left to go'.format(num_epochs - 1 - e)) num_so_far_accum = 0 @@ -81,67 +88,77 @@ def train(conf,shot_list_train,shot_list_validate,loader): while e < num_epochs-1: e += 1 - print('\nEpoch {}/{}'.format(e+1,num_epochs)) - pbar = Progbar(len(shot_list_train)) + print('\nEpoch {}/{}'.format(e+1, num_epochs)) + pbar = Progbar(len(shot_list_train)) - #decay learning rate each epoch: + # decay learning rate each epoch: K.set_value(train_model.optimizer.lr, lr*lr_decay**(e)) - - #print('Learning rate: {}'.format(train_model.optimizer.lr.get_value())) + num_batches_minimum = 100 num_batches_current = 0 training_losses_tmp = [] - while num_so_far < (e - e_start)*num_total or num_batches_current < num_batches_minimum: + while (num_so_far < (e - e_start)*num_total + or num_batches_current < num_batches_minimum): num_so_far_old = num_so_far try: - batch_xs,batch_ys,batches_to_reset,num_so_far_curr,num_total,is_warmup_period = next(batch_iterator) + (batch_xs, batch_ys, batches_to_reset, num_so_far_curr, + num_total, is_warmup_period) = next(batch_iterator) except StopIteration: print("Resetting batch iterator.") num_so_far_accum = num_so_far batch_iterator = ProcessGenerator(batch_generator()) - batch_xs,batch_ys,batches_to_reset,num_so_far_curr,num_total,is_warmup_period = next(batch_iterator) + (batch_xs, batch_ys, batches_to_reset, num_so_far_curr, + num_total, is_warmup_period) = next(batch_iterator) if np.any(batches_to_reset): - reset_states(train_model,batches_to_reset) + reset_states(train_model, batches_to_reset) if not is_warmup_period: num_so_far = num_so_far_accum+num_so_far_curr - num_batches_current +=1 + num_batches_current += 1 - - loss = train_model.train_on_batch(batch_xs,batch_ys) + loss = train_model.train_on_batch(batch_xs, batch_ys) training_losses_tmp.append(loss) - pbar.add(num_so_far - num_so_far_old, values=[("train loss", loss)]) - loader.verbose=False#True during the first iteration + pbar.add(num_so_far - num_so_far_old, + values=[("train loss", loss)]) + loader.verbose = False # True during the first iteration else: - _ = train_model.predict(batch_xs,batch_size=conf['training']['batch_size']) - + _ = train_model.predict( + batch_xs, batch_size=conf['training']['batch_size']) e = e_start+1.0*num_so_far/num_total sys.stdout.flush() ave_loss = np.mean(training_losses_tmp) training_losses.append(ave_loss) - specific_builder.save_model_weights(train_model,int(round(e))) + specific_builder.save_model_weights(train_model, int(round(e))) if conf['training']['validation_frac'] > 0.0: print("prediction on GPU...") - _,_,_,roc_area,loss = make_predictions_and_evaluate_gpu(conf,shot_list_validate,loader) + _, _, _, roc_area, loss = make_predictions_and_evaluate_gpu( + conf, shot_list_validate, loader) validation_losses.append(loss) validation_roc.append(roc_area) epoch_logs = {} - epoch_logs['val_roc'] = roc_area + epoch_logs['val_roc'] = roc_area epoch_logs['val_loss'] = loss epoch_logs['train_loss'] = ave_loss - best_so_far = cmp_fn(epoch_logs[conf['callbacks']['monitor']],best_so_far) - if best_so_far != epoch_logs[conf['callbacks']['monitor']]: #only save model weights if quantity we are tracking is improving + best_so_far = cmp_fn( + epoch_logs[conf['callbacks']['monitor']], best_so_far) + # only save model weights if quantity we are tracking is improving + if best_so_far != epoch_logs[conf['callbacks']['monitor']]: print("Not saving model weights") - specific_builder.delete_model_weights(train_model,int(round(e))) + specific_builder.delete_model_weights( + train_model, int(round(e))) if conf['training']['ranking_difficulty_fac'] != 1.0: - _,_,_,roc_area_train,loss_train = make_predictions_and_evaluate_gpu(conf,shot_list_train,loader) + (_, _, _, roc_area_train, + loss_train) = make_predictions_and_evaluate_gpu( + conf, shot_list_train, loader) batch_iterator.__exit__() - batch_generator = partial(loader.training_batch_generator_partial_reset,shot_list=shot_list_train) + batch_generator = partial( + loader.training_batch_generator_partial_reset, + shot_list=shot_list_train) batch_iterator = ProcessGenerator(batch_generator()) num_so_far_accum = num_so_far @@ -153,58 +170,78 @@ def train(conf,shot_list_train,shot_list_validate,loader): if conf['training']['ranking_difficulty_fac'] != 1.0: print('Train Loss: {:.3e}'.format(loss_train)) print('Train ROC: {:.4f}'.format(roc_area_train)) - - # plot_losses(conf,[training_losses],specific_builder,name='training') if conf['training']['validation_frac'] > 0.0: - plot_losses(conf,[training_losses,validation_losses,validation_roc],specific_builder,name='training_validation_roc') + plot_losses(conf, + [training_losses, + validation_losses, + validation_roc], + specific_builder, + name='training_validation_roc') batch_iterator.__exit__() print('...done') + def optimizer_class(): - from keras.optimizers import SGD,Adam,RMSprop,Nadam,TFOptimizer + from keras.optimizers import SGD, Adam, RMSprop, Nadam, TFOptimizer if conf['model']['optimizer'] == 'sgd': - return SGD(lr=conf['model']['lr'],clipnorm=conf['model']['clipnorm']) + return SGD(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm']) elif conf['model']['optimizer'] == 'momentum_sgd': - return SGD(lr=conf['model']['lr'],clipnorm=conf['model']['clipnorm'], decay=1e-6, momentum=0.9) + return SGD( + lr=conf['model']['lr'], + clipnorm=conf['model']['clipnorm'], + decay=1e-6, + momentum=0.9) elif conf['model']['optimizer'] == 'tf_momentum_sgd': - return TFOptimizer(tf.train.MomentumOptimizer(learning_rate=conf['model']['lr'],momentum=0.9)) + return TFOptimizer( + tf.train.MomentumOptimizer( + learning_rate=conf['model']['lr'], + momentum=0.9)) elif conf['model']['optimizer'] == 'adam': - return Adam(lr=conf['model']['lr'],clipnorm=conf['model']['clipnorm']) + return Adam(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm']) elif conf['model']['optimizer'] == 'tf_adam': - return TFOptimizer(tf.train.AdamOptimizer(learning_rate=conf['model']['lr'])) + return TFOptimizer( + tf.train.AdamOptimizer( + learning_rate=conf['model']['lr'])) elif conf['model']['optimizer'] == 'rmsprop': - return RMSprop(lr=conf['model']['lr'],clipnorm=conf['model']['clipnorm']) + return RMSprop( + lr=conf['model']['lr'], + clipnorm=conf['model']['clipnorm']) elif conf['model']['optimizer'] == 'nadam': - return Nadam(lr=conf['model']['lr'],clipnorm=conf['model']['clipnorm']) + return Nadam( + lr=conf['model']['lr'], + clipnorm=conf['model']['clipnorm']) else: print("Optimizer not implemented yet") exit(1) class HyperRunner(object): - def __init__(self,conf,loader,shot_list): + def __init__(self, conf, loader, shot_list): self.loader = loader self.shot_list = shot_list self.conf = conf - #FIXME setup for hyperas search - def keras_fmin_fnct(self,space): + # FIXME setup for hyperas search + def keras_fmin_fnct(self, space): from plasma.models import builder specific_builder = builder.ModelBuilder(self.conf) - train_model = specific_builder.hyper_build_model(space,False) - train_model.compile(optimizer=optimizer_class(),loss=conf['data']['target'].loss) + train_model = specific_builder.hyper_build_model(space, False) + train_model.compile( + optimizer=optimizer_class(), + loss=conf['data']['target'].loss) np.random.seed(1) validation_losses = [] validation_roc = [] training_losses = [] - shot_list_train,shot_list_validate = self.shot_list.split_direct(1.0-conf['training']['validation_frac'],do_shuffle=True) - + shot_list_train, shot_list_validate = self.shot_list.split_direct( + 1.0-conf['training']['validation_frac'], do_shuffle=True) + from keras.utils.generic_utils import Progbar from keras import backend as K @@ -212,46 +249,59 @@ def keras_fmin_fnct(self,space): num_at_once = self.conf['training']['num_shots_at_once'] lr_decay = self.conf['model']['lr_decay'] - resulting_dict = {'loss':None,'status':STATUS_OK,'model':None} + resulting_dict = {'loss': None, 'status': STATUS_OK, 'model': None} e = -1 - #print("Current num_epochs {}".format(e)) + # print("Current num_epochs {}".format(e)) while e < num_epochs-1: e += 1 - pbar = Progbar(len(shot_list_train)) + pbar = Progbar(len(shot_list_train)) shot_list_train.shuffle() shot_sublists = shot_list_train.sublists(num_at_once)[:1] training_losses_tmp = [] K.set_value(train_model.optimizer.lr, lr*lr_decay**(e)) - for (i,shot_sublist) in enumerate(shot_sublists): - X_list,y_list = self.loader.load_as_X_y_list(shot_sublist) - for j,(X,y) in enumerate(zip(X_list,y_list)): + for (i, shot_sublist) in enumerate(shot_sublists): + X_list, y_list = self.loader.load_as_X_y_list(shot_sublist) + for j, (X, y) in enumerate(zip(X_list, y_list)): history = builder.LossHistory() - train_model.fit(X,y, - batch_size=Loader.get_batch_size(self.conf['training']['batch_size'],prediction_mode=False), - epochs=1,shuffle=False,verbose=0, - validation_split=0.0,callbacks=[history]) + train_model.fit( + X, + y, + batch_size=Loader.get_batch_size( + self.conf['training']['batch_size'], + prediction_mode=False), + epochs=1, + shuffle=False, + verbose=0, + validation_split=0.0, + callbacks=[history]) train_model.reset_states() train_loss = np.mean(history.losses) training_losses_tmp.append(train_loss) - pbar.add(1.0*len(shot_sublist)/len(X_list), values=[("train loss", train_loss)]) - self.loader.verbose=False + pbar.add(1.0*len(shot_sublist)/len(X_list), + values=[("train loss", train_loss)]) + self.loader.verbose = False sys.stdout.flush() training_losses.append(np.mean(training_losses_tmp)) - specific_builder.save_model_weights(train_model,e) + specific_builder.save_model_weights(train_model, e) - _,_,_,roc_area,loss = make_predictions_and_evaluate_gpu(self.conf,shot_list_validate,self.loader) - print("Epoch: {}, loss: {}, validation_losses_size: {}".format(e,loss,len(validation_losses))) + _, _, _, roc_area, loss = make_predictions_and_evaluate_gpu( + self.conf, shot_list_validate, self.loader) + print( + "Epoch: {}, loss: {}, validation_losses_size: {}".format( + e, loss, len(validation_losses))) validation_losses.append(loss) validation_roc.append(roc_area) resulting_dict['loss'] = loss resulting_dict['model'] = train_model - #print("Results {}, before {}".format(resulting_dict,id(resulting_dict))) + # print("Results {}, before + # {}".format(resulting_dict,id(resulting_dict))) - #print("Results {}, after {}".format(resulting_dict,id(resulting_dict))) + # print("Results {}, after + # {}".format(resulting_dict,id(resulting_dict))) return resulting_dict def get_space(self): @@ -263,29 +313,31 @@ def frnn_minimize(self, algo, max_evals, trials, rseed=1337): from hyperopt import fmin best_run = fmin(self.keras_fmin_fnct, - space=self.get_space(), - algo=algo, - max_evals=max_evals, - trials=trials, - rstate=np.random.RandomState(rseed)) + space=self.get_space(), + algo=algo, + max_evals=max_evals, + trials=trials, + rstate=np.random.RandomState(rseed)) best_model = None for trial in trials: vals = trial.get('misc').get('vals') for key in vals.keys(): vals[key] = vals[key][0] - if trial.get('misc').get('vals') == best_run and 'model' in trial.get('result').keys(): + if (trial.get('misc').get('vals') == best_run + and 'model' in trial.get('result').keys()): best_model = trial.get('result').get('model') return best_run, best_model -def plot_losses(conf,losses_list,specific_builder,name=''): + +def plot_losses(conf, losses_list, specific_builder, name=''): unique_id = specific_builder.get_unique_id() savedir = 'losses' if not os.path.exists(savedir): os.makedirs(savedir) - save_path = os.path.join(savedir,'{}_loss_{}.png'.format(name,unique_id)) + save_path = os.path.join(savedir, '{}_loss_{}.png'.format(name, unique_id)) plt.figure() for losses in losses_list: plt.semilogy(losses) @@ -295,43 +347,49 @@ def plot_losses(conf,losses_list,specific_builder,name=''): plt.savefig(save_path) -def make_predictions(conf,shot_list,loader): +def make_predictions(conf, shot_list, loader): loader.set_inference_mode(True) - use_cores = max(1,mp.cpu_count()-2) + use_cores = max(1, mp.cpu_count()-2) if backend == 'tf' or backend == 'tensorflow': first_time = "tensorflow" not in sys.modules if first_time: - import tensorflow as tf - os.environ['KERAS_BACKEND'] = 'tensorflow' - from keras.backend.tensorflow_backend import set_session - config = tf.ConfigProto(device_count={"CPU":use_cores}) - set_session(tf.Session(config=config)) + import tensorflow as tf + os.environ['KERAS_BACKEND'] = 'tensorflow' + from keras.backend.tensorflow_backend import set_session + config = tf.ConfigProto(device_count={"CPU": use_cores}) + set_session(tf.Session(config=config)) else: os.environ['THEANO_FLAGS'] = 'device=cpu' - import theano + # import theano from plasma.models.builder import ModelBuilder - specific_builder = ModelBuilder(conf) + specific_builder = ModelBuilder(conf) y_prime = [] y_gold = [] disruptive = [] model = specific_builder.build_model(True) - model.compile(optimizer=optimizer_class(),loss=conf['data']['target'].loss) + model.compile( + optimizer=optimizer_class(), + loss=conf['data']['target'].loss) specific_builder.load_model_weights(model) model_save_path = specific_builder.get_latest_save_path() start_time = time.time() pool = mp.Pool(use_cores) - fn = partial(make_single_prediction,builder=specific_builder,loader=loader,model_save_path=model_save_path) + fn = partial( + make_single_prediction, + builder=specific_builder, + loader=loader, + model_save_path=model_save_path) print('running in parallel on {} processes'.format(pool._processes)) - for (i,(y_p,y,is_disruptive)) in enumerate(pool.imap(fn,shot_list)): - print('Shot {}/{}'.format(i,len(shot_list))) + for (i, (y_p, y, is_disruptive)) in enumerate(pool.imap(fn, shot_list)): + print('Shot {}/{}'.format(i, len(shot_list))) sys.stdout.flush() y_prime.append(y_p) y_gold.append(y) @@ -340,77 +398,87 @@ def make_predictions(conf,shot_list,loader): pool.join() print('Finished Predictions in {} seconds'.format(time.time()-start_time)) loader.set_inference_mode(False) - return y_prime,y_gold,disruptive + return y_prime, y_gold, disruptive -def make_single_prediction(shot,specific_builder,loader,model_save_path): +def make_single_prediction(shot, specific_builder, loader, model_save_path): loader.set_inference_mode(True) model = specific_builder.build_model(True) - model.compile(optimizer=optimizer_class(),loss=conf['data']['target'].loss) + model.compile( + optimizer=optimizer_class(), + loss=conf['data']['target'].loss) model.load_weights(model_save_path) model.reset_states() - X,y = loader.load_as_X_y(shot,prediction_mode=True) + X, y = loader.load_as_X_y(shot, prediction_mode=True) assert(X.shape[0] == y.shape[0]) - y_p = model.predict(X,batch_size=Loader.get_batch_size(conf['training']['batch_size'],prediction_mode=True),verbose=0) + y_p = model.predict( + X, + batch_size=Loader.get_batch_size(conf['training']['batch_size'], + prediction_mode=True), + verbose=0) answer_dims = y_p.shape[-1] if conf['model']['return_sequences']: shot_length = y_p.shape[0]*y_p.shape[1] else: shot_length = y_p.shape[0] - y_p = np.reshape(y_p,(shot_length,answer_dims)) - y = np.reshape(y,(shot_length,answer_dims)) + y_p = np.reshape(y_p, (shot_length, answer_dims)) + y = np.reshape(y, (shot_length, answer_dims)) is_disruptive = shot.is_disruptive_shot() model.reset_states() loader.set_inference_mode(False) - return y_p,y,is_disruptive + return y_p, y, is_disruptive -def make_predictions_gpu(conf,shot_list,loader,custom_path=None): +def make_predictions_gpu(conf, shot_list, loader, custom_path=None): loader.set_inference_mode(True) if backend == 'tf' or backend == 'tensorflow': first_time = "tensorflow" not in sys.modules if first_time: - import tensorflow as tf - os.environ['KERAS_BACKEND'] = 'tensorflow' - from keras.backend.tensorflow_backend import set_session - config = tf.ConfigProto(device_count={"GPU":1}) - set_session(tf.Session(config=config)) + import tensorflow as tf + os.environ['KERAS_BACKEND'] = 'tensorflow' + from keras.backend.tensorflow_backend import set_session + config = tf.ConfigProto(device_count={"GPU": 1}) + set_session(tf.Session(config=config)) else: os.environ['THEANO_FLAGS'] = 'device=gpu,floatX=float32' - import theano + # import theano - from keras.utils.generic_utils import Progbar + from keras.utils.generic_utils import Progbar from plasma.models.builder import ModelBuilder - specific_builder = ModelBuilder(conf) + specific_builder = ModelBuilder(conf) y_prime = [] y_gold = [] disruptive = [] model = specific_builder.build_model(True) - model.compile(optimizer=optimizer_class(),loss=conf['data']['target'].loss) + model.compile(optimizer=optimizer_class(), + loss=conf['data']['target'].loss) - specific_builder.load_model_weights(model,custom_path) + specific_builder.load_model_weights(model, custom_path) model.reset_states() - pbar = Progbar(len(shot_list)) - shot_sublists = shot_list.sublists(conf['model']['pred_batch_size'],do_shuffle=False,equal_size=True) - for (i,shot_sublist) in enumerate(shot_sublists): - X,y,shot_lengths,disr = loader.load_as_X_y_pred(shot_sublist) - #load data and fit on data + pbar = Progbar(len(shot_list)) + shot_sublists = shot_list.sublists( + conf['model']['pred_batch_size'], + do_shuffle=False, + equal_size=True) + for (i, shot_sublist) in enumerate(shot_sublists): + X, y, shot_lengths, disr = loader.load_as_X_y_pred(shot_sublist) + # load data and fit on data y_p = model.predict(X, - batch_size=conf['model']['pred_batch_size']) + batch_size=conf['model']['pred_batch_size']) model.reset_states() y_p = loader.batch_output_to_array(y_p) y = loader.batch_output_to_array(y) - #cut arrays back - y_p = [arr[:shot_lengths[j]] for (j,arr) in enumerate(y_p)] - y = [arr[:shot_lengths[j]] for (j,arr) in enumerate(y)] + # cut arrays back + y_p = [arr[:shot_lengths[j]] for (j, arr) in enumerate(y_p)] + y = [arr[:shot_lengths[j]] for (j, arr) in enumerate(y)] pbar.add(1.0*len(shot_sublist)) - loader.verbose=False#True during the first iteration + loader.verbose = False # True during the first iteration y_prime += y_p y_gold += y disruptive += disr @@ -418,67 +486,83 @@ def make_predictions_gpu(conf,shot_list,loader,custom_path=None): y_gold = y_gold[:len(shot_list)] disruptive = disruptive[:len(shot_list)] loader.set_inference_mode(False) - return y_prime,y_gold,disruptive - + return y_prime, y_gold, disruptive -def make_predictions_and_evaluate_gpu(conf,shot_list,loader,custom_path=None): - y_prime,y_gold,disruptive = make_predictions_gpu(conf,shot_list,loader,custom_path) +def make_predictions_and_evaluate_gpu( + conf, shot_list, loader, custom_path=None): + y_prime, y_gold, disruptive = make_predictions_gpu( + conf, shot_list, loader, custom_path) analyzer = PerformanceAnalyzer(conf=conf) - roc_area = analyzer.get_roc_area(y_prime,y_gold,disruptive) - shot_list.set_weights(analyzer.get_shot_difficulty(y_prime,y_gold,disruptive)) - loss = get_loss_from_list(y_prime,y_gold,conf['data']['target']) - return y_prime,y_gold,disruptive,roc_area,loss + roc_area = analyzer.get_roc_area(y_prime, y_gold, disruptive) + shot_list.set_weights( + analyzer.get_shot_difficulty( + y_prime, y_gold, disruptive)) + loss = get_loss_from_list(y_prime, y_gold, conf['data']['target']) + return y_prime, y_gold, disruptive, roc_area, loss -def make_evaluations_gpu(conf,shot_list,loader): + +def make_evaluations_gpu(conf, shot_list, loader): loader.set_inference_mode(True) if backend == 'tf' or backend == 'tensorflow': first_time = "tensorflow" not in sys.modules if first_time: - import tensorflow as tf - os.environ['KERAS_BACKEND'] = 'tensorflow' - from keras.backend.tensorflow_backend import set_session - config = tf.ConfigProto(device_count={"GPU":1}) - set_session(tf.Session(config=config)) + import tensorflow as tf + os.environ['KERAS_BACKEND'] = 'tensorflow' + from keras.backend.tensorflow_backend import set_session + config = tf.ConfigProto(device_count={"GPU": 1}) + set_session(tf.Session(config=config)) else: os.environ['THEANO_FLAGS'] = 'device=gpu,floatX=float32' - import theano - - from keras.utils.generic_utils import Progbar + # import theano + + from keras.utils.generic_utils import Progbar from plasma.models.builder import ModelBuilder - specific_builder = ModelBuilder(conf) + specific_builder = ModelBuilder(conf) - y_prime = [] - y_gold = [] - disruptive = [] - batch_size = min(len(shot_list),conf['model']['pred_batch_size']) + # y_prime = [] + # y_gold = [] + # disruptive = [] + batch_size = min(len(shot_list), conf['model']['pred_batch_size']) - pbar = Progbar(len(shot_list)) - print('evaluating {} shots using batchsize {}'.format(len(shot_list),batch_size)) + pbar = Progbar(len(shot_list)) + print( + 'evaluating {} shots using batchsize {}'.format( + len(shot_list), + batch_size)) - shot_sublists = shot_list.sublists(batch_size,equal_size=False) + shot_sublists = shot_list.sublists(batch_size, equal_size=False) all_metrics = [] all_weights = [] - for (i,shot_sublist) in enumerate(shot_sublists): + for (i, shot_sublist) in enumerate(shot_sublists): batch_size = len(shot_sublist) - model = specific_builder.build_model(True,custom_batch_size=batch_size) - model.compile(optimizer=optimizer_class(),loss=conf['data']['target'].loss) + model = specific_builder.build_model( + True, custom_batch_size=batch_size) + model.compile( + optimizer=optimizer_class(), + loss=conf['data']['target'].loss) specific_builder.load_model_weights(model) model.reset_states() - X,y,shot_lengths,disr = loader.load_as_X_y_pred(shot_sublist,custom_batch_size=batch_size) - #load data and fit on data - all_metrics.append(model.evaluate(X,y,batch_size=batch_size,verbose=False)) + X, y, shot_lengths, disr = loader.load_as_X_y_pred( + shot_sublist, custom_batch_size=batch_size) + # load data and fit on data + all_metrics.append( + model.evaluate( + X, + y, + batch_size=batch_size, + verbose=False)) all_weights.append(batch_size) model.reset_states() pbar.add(1.0*len(shot_sublist)) - loader.verbose=False#True during the first iteration + loader.verbose = False # True during the first iteration if len(all_metrics) > 1: print('evaluations all: {}'.format(all_metrics)) - loss = np.average(all_metrics,weights = all_weights) + loss = np.average(all_metrics, weights=all_weights) print('Evaluation Loss: {}'.format(loss)) loader.set_inference_mode(False) - return loss + return loss diff --git a/plasma/models/shallow_runner.py b/plasma/models/shallow_runner.py index b9ccf883..c4fa1552 100644 --- a/plasma/models/shallow_runner.py +++ b/plasma/models/shallow_runner.py @@ -1,43 +1,46 @@ -from __future__ import print_function +from sklearn import svm +from keras.utils.generic_utils import Progbar +import keras.callbacks as cbks +from sklearn.metrics import classification_report +# accuracy_score, auc, confusion_matrix +from sklearn.externals import joblib +from sklearn.ensemble import RandomForestClassifier +import hashlib +from plasma.utils.downloading import makedirs_process_safe +# from plasma.utils.state_reset import reset_states +from plasma.utils.evaluation import ttd, get_loss_from_list +from plasma.utils.performance import PerformanceAnalyzer +# from plasma.models.loader import Loader, ProcessGenerator +# from plasma.conf import conf +from sklearn.neural_network import MLPClassifier +from xgboost import XGBClassifier +import pathos.multiprocessing as mp +from functools import partial +import os +import datetime +import time +import numpy as np +# import matplotlib.pyplot as plt import matplotlib matplotlib.use('Agg') -import matplotlib.pyplot as plt -import numpy as np -import sys -if sys.version_info[0] < 3: - from itertools import imap +# import sys +# if sys.version_info[0] < 3: +# from itertools import imap -#leading to import errors: -#from hyperopt import hp, STATUS_OK -#from hyperas.distributions import conditional +# leading to import errors: +# from hyperopt import hp, STATUS_OK +# from hyperas.distributions import conditional -import time -import datetime -import os -from functools import partial -import pathos.multiprocessing as mp -from xgboost import XGBClassifier -from sklearn.neural_network import MLPClassifier - -from plasma.conf import conf -from plasma.models.loader import Loader, ProcessGenerator -from plasma.utils.performance import PerformanceAnalyzer -from plasma.utils.evaluation import * -from plasma.utils.state_reset import reset_states -from plasma.utils.downloading import makedirs_process_safe - -from keras.utils.generic_utils import Progbar - -import hashlib debug_use_shots = 100000 model_filename = "saved_model.pkl" dataset_path = "dataset.npz" dataset_test_path = "dataset_test.npz" + class FeatureExtractor(object): - def __init__(self,loader,timesteps = 32): + def __init__(self, loader, timesteps=32): self.loader = loader self.timesteps = timesteps self.positional_fit_order = 4 @@ -45,53 +48,72 @@ def __init__(self,loader,timesteps = 32): self.temporal_fit_order = 3 self.num_temporal_features = self.temporal_fit_order + 1 + 3 - def get_sample_probs(self,shot_list,num_samples): + def get_sample_probs(self, shot_list, num_samples): print("Calculating number of timesteps") - timesteps_total,timesteps_d,timesteps_nd = shot_list.num_timesteps(self.loader.conf['paths']['processed_prepath']) - print("Total data: {} time samples, {} disruptive".format(timesteps_total,1.0*timesteps_d/timesteps_total)) + timesteps_total, timesteps_d, timesteps_nd = shot_list.num_timesteps( + self.loader.conf['paths']['processed_prepath']) + print("Total data: {} time samples, {} disruptive".format( + timesteps_total, 1.0*timesteps_d/timesteps_total)) if self.loader.conf['data']['equalize_classes']: - sample_prob_d = np.minimum(1.0,1.0*timesteps_nd/timesteps_d) - sample_prob_nd = np.minimum(1.0,1.0*timesteps_d/timesteps_nd) - timesteps_total = 1.0*sample_prob_d*timesteps_d+sample_prob_nd*timesteps_nd - sample_prob = np.minimum(1.0,1.0*num_samples/timesteps_total) + sample_prob_d = np.minimum(1.0, 1.0*timesteps_nd/timesteps_d) + sample_prob_nd = np.minimum(1.0, 1.0*timesteps_d/timesteps_nd) + timesteps_total = (1.0*sample_prob_d*timesteps_d + + sample_prob_nd*timesteps_nd) + sample_prob = np.minimum(1.0, 1.0*num_samples/timesteps_total) sample_prob_d *= sample_prob sample_prob_nd *= sample_prob else: - sample_prob_d = np.minimum(1.0,1.0*num_samples/timesteps_total) + sample_prob_d = np.minimum(1.0, 1.0*num_samples/timesteps_total) sample_prob_nd = sample_prob_d if sample_prob_nd <= 0.0 or sample_prob_d <= 0.0: - val = np.minimum(1.0,num_samples/timesteps_total) - return val,val - return sample_prob_d,sample_prob_nd - - def load_shots(self,shot_list,is_inference=False,as_list=False,num_samples=np.Inf): + val = np.minimum(1.0, num_samples/timesteps_total) + return val, val + return sample_prob_d, sample_prob_nd + + def load_shots( + self, + shot_list, + is_inference=False, + as_list=False, + num_samples=np.Inf): X = [] Y = [] Disr = [] print("loading...") - pbar = Progbar(len(shot_list)) - - sample_prob_d,sample_prob_nd = self.get_sample_probs(shot_list,num_samples) - fn = partial(self.load_shot,is_inference=is_inference,sample_prob_d=sample_prob_d,sample_prob_nd=sample_prob_nd) + pbar = Progbar(len(shot_list)) + + sample_prob_d, sample_prob_nd = self.get_sample_probs( + shot_list, num_samples) + fn = partial( + self.load_shot, + is_inference=is_inference, + sample_prob_d=sample_prob_d, + sample_prob_nd=sample_prob_nd) pool = mp.Pool() - print('loading data in parallel on {} processes'.format(pool._processes)) - for x,y,disr in pool.imap(fn,shot_list): + print('loading data in parallel on {} processes'.format( + pool._processes)) + for x, y, disr in pool.imap(fn, shot_list): X.append(x) Y.append(y) Disr.append(disr) pbar.add(1.0) pool.close() pool.join() - return X,Y,np.array(Disr) + return X, Y, np.array(Disr) def get_save_prepath(self): prepath = self.loader.conf['paths']['processed_prepath'] use_signals = self.loader.conf['paths']['use_signals'] - identifying_tuple = ''.join(tuple(map(lambda x: x.description, sorted(use_signals)))).encode('utf-8') - save_prepath = prepath + "shallow/use_signals_{}/".format(int(hashlib.md5(identifying_tuple).hexdigest(),16)) + identifying_tuple = ''.join( + tuple(map(lambda x: x.description, + sorted(use_signals)))).encode('utf-8') + save_prepath = ( + prepath + "shallow/use_signals_{}/".format( + int(hashlib.md5(identifying_tuple).hexdigest(), 16)) + ) return save_prepath - def process(self,shot): + def process(self, shot): save_prepath = self.get_save_prepath() save_path = shot.get_save_path(save_prepath) if not os.path.exists(save_prepath): @@ -99,45 +121,45 @@ def process(self,shot): prepath = self.loader.conf['paths']['processed_prepath'] assert(shot.valid) shot.restore(prepath) - self.loader.set_inference_mode(True)#make sure shots aren't cut + self.loader.set_inference_mode(True) # make sure shots aren't cut if self.loader.normalizer is not None: self.loader.normalizer.apply(shot) else: - print('Warning, no normalization. Training data may be poorly conditioned') + print('Warning, no normalization. ', + 'Training data may be poorly conditioned') self.loader.set_inference_mode(False) - # sig,res = self.get_signal_result_from_shot(shot) + # sig, res = self.get_signal_result_from_shot(shot) disr = 1 if shot.is_disruptive else 0 - if not os.path.isfile(save_path): X = self.get_X(shot) - np.savez(save_path,X=X)#,Y=Y,disr=disr - #print(X.shape,Y.shape) + np.savez(save_path, X=X) # , Y=Y, disr=disr + # print(X.shape, Y.shape) else: try: dat = np.load(save_path) - # X,Y,disr = dat["X"],dat["Y"],dat["disr"][()] + # X, Y, disr = dat["X"], dat["Y"], dat["disr"][()] X = dat["X"] - except: #data was there but corrupted, save it again + except BaseException: + # data was there but corrupted, save it again X = self.get_X(shot) - np.savez(save_path,X=X) - + np.savez(save_path, X=X) Y = self.get_Y(shot) shot.make_light() - return X,Y,disr + return X, Y, disr - def get_X(self,shot): + def get_X(self, shot): use_signals = self.loader.conf['paths']['use_signals'] - sig_sample = shot.signals_dict[use_signals[0]] + sig_sample = shot.signals_dict[use_signals[0]] if len(shot.ttd.shape) == 1: - shot.ttd = np.expand_dims(shot.ttd,axis=1) + shot.ttd = np.expand_dims(shot.ttd, axis=1) length = sig_sample.shape[0] if length < self.timesteps: - print(ttd,shot,shot.number) + print(ttd, shot, shot.number) print("Shot must be at least as long as the RNN length.") exit(1) assert(len(sig_sample.shape) == len(shot.ttd.shape) == 2) @@ -146,24 +168,29 @@ def get_X(self,shot): X = [] while(len(X) == 0): for i in range(length-self.timesteps+1): - #if np.random.rand() < sample_prob: - x = self.get_x(i,shot) + # if np.random.rand() < sample_prob: + x = self.get_x(i, shot) X.append(x) X = np.stack(X) return X - def get_Y(self,shot): + def get_Y(self, shot): if len(shot.ttd.shape) == 1: - shot.ttd = np.expand_dims(shot.ttd,axis=1) + shot.ttd = np.expand_dims(shot.ttd, axis=1) offset = self.timesteps - 1 - return np.round(shot.ttd[offset:,0]).astype(np.int) + return np.round(shot.ttd[offset:, 0]).astype(np.int) - def load_shot(self,shot,is_inference=False,sample_prob_d=1.0,sample_prob_nd=1.0): + def load_shot( + self, + shot, + is_inference=False, + sample_prob_d=1.0, + sample_prob_nd=1.0): - X,Y,disr = self.process(shot) + X, Y, disr = self.process(shot) - #cut shot ends if we are supposed to - if self.loader.conf['data']['cut_shot_ends'] and not is_inference: + # cut shot ends if we are supposed to + if self.loader.conf['data']['cut_shot_ends'] and not is_inference: T_min_warn = self.loader.conf['data']['T_min_warn'] X = X[:-T_min_warn] Y = Y[:-T_min_warn] @@ -171,23 +198,24 @@ def load_shot(self,shot,is_inference=False,sample_prob_d=1.0,sample_prob_nd=1.0) sample_prob = sample_prob_nd if disr: sample_prob = sample_prob_d - if sample_prob < 1.0: - indices = np.sort(np.random.choice(np.array(range(len(Y))),int(round(sample_prob*len(Y))),replace=False)) + if sample_prob < 1.0: + indices = np.sort( + np.random.choice(np.array(range(len(Y))), + int(round(sample_prob*len(Y))), + replace=False)) X = X[indices] Y = Y[indices] - return X,Y,disr + return X, Y, disr - - def get_x(self,timestep,shot): + def get_x(self, timestep, shot): x = [] use_signals = self.loader.conf['paths']['use_signals'] for sig in use_signals: - x += [self.extract_features(timestep,shot,sig)] - # x = sig[timestep:timestep+timesteps,:] - x = np.concatenate(x,axis=0) + x += [self.extract_features(timestep, shot, sig)] + # x = sig[timestep:timestep + timesteps,:] + x = np.concatenate(x, axis=0) return x - # def get_x_y(self,timestep,shot): # x = [] # use_signals = self.loader.conf['paths']['use_signals'] @@ -198,24 +226,28 @@ def get_x(self,timestep,shot): # y = np.round(shot.ttd[timestep+self.timesteps-1,0]).astype(np.int) # return x,y - - def extract_features(self,timestep,shot,signal): - raw_sig = shot.signals_dict[signal][timestep:timestep+self.timesteps] - num_positional_features = self.num_positional_features if signal.num_channels > 1 else 1 - output_arr = np.empty((self.timesteps,num_positional_features)) - final_output_arr = np.empty((num_positional_features*self.num_temporal_features)) + def extract_features(self, timestep, shot, signal): + raw_sig = shot.signals_dict[signal][timestep:timestep + self.timesteps] + num_positional_features = ( + self.num_positional_features if signal.num_channels > 1 else 1) + output_arr = np.empty((self.timesteps, num_positional_features)) + final_output_arr = np.empty( + (num_positional_features*self.num_temporal_features)) for t in range(self.timesteps): - output_arr[t,:] = self.extract_positional_features(raw_sig[t,:]) + output_arr[t, :] = self.extract_positional_features(raw_sig[t, :]) for i in range(num_positional_features): idx = i*self.num_temporal_features - final_output_arr[idx:idx+self.num_temporal_features] = self.extract_temporal_features(output_arr[:,i]) + final_output_arr[idx:idx + self.num_temporal_features] = ( + self.extract_temporal_features(output_arr[:, i])) return final_output_arr - def extract_positional_features(self,arr): + def extract_positional_features(self, arr): num_channels = len(arr) if num_channels > 1: ret_arr = np.empty(self.num_positional_features) - coefficients = np.polynomial.polynomial.polyfit(np.linspace(0,1,num_channels),arr,self.positional_fit_order) + coefficients = np.polynomial.polynomial.polyfit( + np.linspace(0, 1, num_channels), arr, + self.positional_fit_order) mu = np.mean(arr) std = np.std(arr) max_val = np.max(arr) @@ -227,9 +259,10 @@ def extract_positional_features(self,arr): else: return arr - def extract_temporal_features(self,arr): + def extract_temporal_features(self, arr): ret_arr = np.empty(self.num_temporal_features) - coefficients = np.polynomial.polynomial.polyfit(np.linspace(0,1,self.timesteps),arr,self.temporal_fit_order) + coefficients = np.polynomial.polynomial.polyfit( + np.linspace(0, 1, self.timesteps), arr, self.temporal_fit_order) mu = np.mean(arr) std = np.std(arr) max_val = np.max(arr) @@ -239,75 +272,95 @@ def extract_temporal_features(self,arr): ret_arr[self.temporal_fit_order+3] = max_val return ret_arr - def prepend_timesteps(self,arr): + def prepend_timesteps(self, arr): prepend = arr[0]*np.ones(self.timesteps-1) - return np.concatenate((prepend,arr)) + return np.concatenate((prepend, arr)) -from sklearn import svm -from sklearn.ensemble import RandomForestClassifier -from sklearn.externals import joblib -from sklearn.metrics import accuracy_score,auc,classification_report,confusion_matrix -import keras.callbacks as cbks def build_callbacks(conf): ''' - The purpose of the method is to set up logging and history. It is based on Keras Callbacks + The purpose of the method is to set up logging and history. It is based on + Keras Callbacks https://github.com/fchollet/keras/blob/fbc9a18f0abc5784607cd4a2a3886558efa3f794/keras/callbacks.py - Currently used callbacks include: BaseLogger, CSVLogger, EarlyStopping. - Other possible callbacks to add in future: RemoteMonitor, LearningRateScheduler - - Argument list: - - conf: There is a "callbacks" section in conf.yaml file. Relevant parameters are: - list: Parameter specifying additional callbacks, read in the driver script and passed as an argument of type list (see next arg) - metrics: List of quantities monitored during training and validation - mode: one of {auto, min, max}. The decision to overwrite the current save file is made based on either the maximization or the minimization of the monitored quantity. For val_acc, this should be max, for val_loss this should be min, etc. In auto mode, the direction is automatically inferred from the name of the monitored quantity. - monitor: Quantity used for early stopping, has to be from the list of metrics - patience: Number of epochs used to decide on whether to apply early stopping or continue training - - callbacks_list: uses callbacks.list configuration parameter, specifies the list of additional callbacks - Returns: modified list of callbacks + Currently used callbacks include: BaseLogger, CSVLogger, EarlyStopping. + Other possible callbacks to add in future: + RemoteMonitor, LearningRateScheduler + + Argument list: + - conf: There is a "callbacks" section in conf.yaml file. + + Relevant parameters are: + list: Parameter specifying additional callbacks, read in the driver + script and passed as an argument of type list (see next arg) + metrics: List of quantities monitored during training and + validation + mode: one of {auto, min, max}. The decision to overwrite the + current save file is made based on either the maximization or the + minimization of the monitored quantity. For val_acc, this should be max, + for val_loss this should be min, etc. In auto mode, the direction is + automatically inferred from the name of the monitored quantity. + monitor: Quantity used for early stopping, has to be from the list + of metrics + patience: Number of epochs used to decide on whether to apply early + stopping or continue training + + - callbacks_list: uses callbacks.list configuration parameter, + specifies the list of additional callbacks + + Returns: + modified list of callbacks ''' - mode = conf['callbacks']['mode'] - monitor = conf['callbacks']['monitor'] - patience = conf['callbacks']['patience'] + # mode = conf['callbacks']['mode'] + # monitor = conf['callbacks']['monitor'] + # patience = conf['callbacks']['patience'] csvlog_save_path = conf['paths']['csvlog_save_path'] - #CSV callback is on by default + # CSV callback is on by default if not os.path.exists(csvlog_save_path): - os.makedirs(csvlog_save_path) + os.makedirs(csvlog_save_path) - callbacks_list = conf['callbacks']['list'] + # callbacks_list = conf['callbacks']['list'] callbacks = [cbks.BaseLogger()] - callbacks += [cbks.CSVLogger("{}callbacks-{}.log".format(csvlog_save_path,datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")))] - + callbacks += [cbks.CSVLogger("{}callbacks-{}.log".format( + csvlog_save_path, + datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S"))) + ] return cbks.CallbackList(callbacks) -def train(conf,shot_list_train,shot_list_validate,loader): +def train(conf, shot_list_train, shot_list_validate, loader): np.random.seed(1) - print('validate: {} shots, {} disruptive'.format(len(shot_list_validate),shot_list_validate.num_disruptive())) - print('training: {} shots, {} disruptive'.format(len(shot_list_train),shot_list_train.num_disruptive())) + print('validate: {} shots, {} disruptive'.format( + len(shot_list_validate), + shot_list_validate.num_disruptive())) + print('training: {} shots, {} disruptive'.format( + len(shot_list_train), + shot_list_train.num_disruptive())) num_samples = conf['model']['shallow_model']['num_samples'] feature_extractor = FeatureExtractor(loader) shot_list_train = shot_list_train.random_sublist(debug_use_shots) - X,Y,_ = feature_extractor.load_shots(shot_list_train,num_samples = num_samples) - Xv,Yv,_ = feature_extractor.load_shots(shot_list_validate,num_samples = num_samples) - X = np.concatenate(X,axis=0) - Y = np.concatenate(Y,axis=0) - Xv = np.concatenate(Xv,axis=0) - Yv = np.concatenate(Yv,axis=0) - - #max_samples = 100000 - #num_samples = min(max_samples,len(Y)) - #indices = np.random.choice(np.array(range(len(Y))),num_samples,replace=False) - #X = X[indices] - #Y = Y[indices] - - print("fitting on {} samples, {} positive".format(len(X),np.sum(Y > 0))) + X, Y, _ = feature_extractor.load_shots( + shot_list_train, num_samples=num_samples) + Xv, Yv, _ = feature_extractor.load_shots( + shot_list_validate, num_samples=num_samples) + X = np.concatenate(X, axis=0) + Y = np.concatenate(Y, axis=0) + Xv = np.concatenate(Xv, axis=0) + Yv = np.concatenate(Yv, axis=0) + + # max_samples = 100000 + # num_samples = min(max_samples, len(Y)) + # indices = np.random.choice(np.array(range(len(Y))), num_samples, + # replace=False) + # X = X[indices] + # Y = Y[indices] + + print("fitting on {} samples, {} positive".format(len(X), np.sum(Y > 0))) callbacks = build_callbacks(conf) callback_metrics = conf['callbacks']['metrics'] callbacks.set_params({ @@ -316,119 +369,131 @@ def train(conf,shot_list_train,shot_list_validate,loader): callbacks.on_train_begin() callbacks.on_epoch_begin(0) - save_prepath = feature_extractor.get_save_prepath() - model_path = conf['paths']['model_save_path'] + model_filename #save_prepath + model_filename + # save_prepath = feature_extractor.get_save_prepath() + model_path = (conf['paths']['model_save_path'] + + model_filename) # save_prepath + model_filename makedirs_process_safe(conf['paths']['model_save_path']) model_conf = conf['model']['shallow_model'] if not model_conf['skip_train'] or not os.path.isfile(model_path): - + start_time = time.time() if model_conf["scale_pos_weight"] != 1: - scale_pos_weight_dict = {np.min(Y) : 1, np.max(Y):model_conf["scale_pos_weight"]} + scale_pos_weight_dict = { + np.min(Y): 1, np.max(Y): model_conf["scale_pos_weight"]} else: scale_pos_weight_dict = None if model_conf['type'] == "svm": model = svm.SVC(probability=True, - C=model_conf["C"], - kernel=model_conf["kernel"], - class_weight=scale_pos_weight_dict) + C=model_conf["C"], + kernel=model_conf["kernel"], + class_weight=scale_pos_weight_dict) elif model_conf['type'] == "random_forest": - model = RandomForestClassifier(n_estimators=model_conf["n_estimators"], + model = RandomForestClassifier( + n_estimators=model_conf["n_estimators"], max_depth=model_conf["max_depth"], class_weight=scale_pos_weight_dict, n_jobs=-1) elif model_conf['type'] == "xgboost": max_depth = model_conf["max_depth"] - if max_depth == None: + if max_depth is None: max_depth = 0 - model = XGBClassifier(max_depth=max_depth, + model = XGBClassifier( + max_depth=max_depth, learning_rate=model_conf['learning_rate'], n_estimators=model_conf["n_estimators"], scale_pos_weight=model_conf["scale_pos_weight"]) elif model_conf['type'] == 'mlp': - hidden_layer_sizes = tuple(reversed([model_conf['final_hidden_layer_size']*2**x for x in range(model_conf['num_hidden_layers'])])) - model = MLPClassifier(hidden_layer_sizes = hidden_layer_sizes, - learning_rate_init = model_conf['learning_rate_mlp'], - alpha = model_conf['mlp_regularization']) + hidden_layer_sizes = tuple(reversed( + [model_conf['final_hidden_layer_size']*2**x + for x in range(model_conf['num_hidden_layers'])])) + model = MLPClassifier( + hidden_layer_sizes=hidden_layer_sizes, + learning_rate_init=model_conf['learning_rate_mlp'], + alpha=model_conf['mlp_regularization']) else: print("Unkown model type, exiting.") exit(1) - model.fit(X,Y) - joblib.dump(model,model_path) + model.fit(X, Y) + joblib.dump(model, model_path) print("Fit model in {} seconds".format(time.time()-start_time)) else: model = joblib.load(model_path) print("model exists.") - Y_pred = model.predict(X) print("Train") - print(classification_report(Y,Y_pred)) + print(classification_report(Y, Y_pred)) Y_predv = model.predict(Xv) print("Validate") - print(classification_report(Yv,Y_predv)) - #print(confusion_matrix(Y,Y_pred)) - _,_,_,roc_area,loss = make_predictions_and_evaluate_gpu(conf,shot_list_validate,loader) - # _,_,_,roc_area_train,loss_train = make_predictions_and_evaluate_gpu(conf,shot_list_train,loader) + print(classification_report(Yv, Y_predv)) + # print(confusion_matrix(Y,Y_pred)) + _, _, _, roc_area, loss = make_predictions_and_evaluate_gpu( + conf, shot_list_validate, loader) + # _, _, _, roc_area_train, loss_train = make_predictions_and_evaluate_gpu( + # conf, shot_list_train, loader) print('Validation Loss: {:.3e}'.format(loss)) print('Validation ROC: {:.4f}'.format(roc_area)) epoch_logs = {} - epoch_logs['val_roc'] = roc_area + epoch_logs['val_roc'] = roc_area epoch_logs['val_loss'] = loss # epoch_logs['train_roc'] = roc_area_train - # epoch_logs['train_loss'] = loss_train + # epoch_logs['train_loss'] = loss_train callbacks.on_epoch_end(0, epoch_logs) - print('...done') -def make_predictions(conf,shot_list,loader,custom_path=None): +def make_predictions(conf, shot_list, loader, custom_path=None): feature_extractor = FeatureExtractor(loader) - save_prepath = feature_extractor.get_save_prepath() - if custom_path == None: - model_path = conf['paths']['model_save_path'] + model_filename#save_prepath + model_filename + # save_prepath = feature_extractor.get_save_prepath() + if custom_path is None: + model_path = conf['paths']['model_save_path'] + \ + model_filename # save_prepath + model_filename else: model_path = custom_path model = joblib.load(model_path) - #shot_list = shot_list.random_sublist(10) + # shot_list = shot_list.random_sublist(10) y_prime = [] y_gold = [] disruptive = [] - pbar = Progbar(len(shot_list)) - fn = partial(predict_single_shot,model=model,feature_extractor=feature_extractor) + pbar = Progbar(len(shot_list)) + fn = partial( + predict_single_shot, + model=model, + feature_extractor=feature_extractor) pool = mp.Pool() print('predicting in parallel on {} processes'.format(pool._processes)) - #for (y_p,y,disr) in map(fn,shot_list): - for (y_p,y,disr) in pool.imap(fn,shot_list): - #y_p,y,disr = predict_single_shot(model,feature_extractor,shot) - y_prime += [np.expand_dims(y_p,axis=1)] - y_gold += [np.expand_dims(y,axis=1)] + # for (y_p, y, disr) in map(fn, shot_list): + for (y_p, y, disr) in pool.imap(fn, shot_list): + # y_p, y, disr = predict_single_shot(model, feature_extractor,shot) + y_prime += [np.expand_dims(y_p, axis=1)] + y_gold += [np.expand_dims(y, axis=1)] disruptive += [disr] pbar.add(1.0) pool.close() pool.join() - return y_prime,y_gold,disruptive + return y_prime, y_gold, disruptive + -def predict_single_shot(shot,model,feature_extractor): - X,y,disr = feature_extractor.load_shot(shot,is_inference=True) - y_p = model.predict_proba(X)[:,1] - #print(y) - #print(y_p) +def predict_single_shot(shot, model, feature_extractor): + X, y, disr = feature_extractor.load_shot(shot, is_inference=True) + y_p = model.predict_proba(X)[:, 1] + # print(y) + # print(y_p) y = feature_extractor.prepend_timesteps(y) y_p = feature_extractor.prepend_timesteps(y_p) - return y_p,y,disr + return y_p, y, disr - -def make_predictions_and_evaluate_gpu(conf,shot_list,loader,custom_path = None): - y_prime,y_gold,disruptive = make_predictions(conf,shot_list,loader,custom_path) +def make_predictions_and_evaluate_gpu( + conf, shot_list, loader, custom_path=None): + y_prime, y_gold, disruptive = make_predictions( + conf, shot_list, loader, custom_path) analyzer = PerformanceAnalyzer(conf=conf) - roc_area = analyzer.get_roc_area(y_prime,y_gold,disruptive) - loss = get_loss_from_list(y_prime,y_gold,conf['data']['target']) - return y_prime,y_gold,disruptive,roc_area,loss - + roc_area = analyzer.get_roc_area(y_prime, y_gold, disruptive) + loss = get_loss_from_list(y_prime, y_gold, conf['data']['target']) + return y_prime, y_gold, disruptive, roc_area, loss diff --git a/plasma/models/targets.py b/plasma/models/targets.py index 7aa3df6c..558d644f 100644 --- a/plasma/models/targets.py +++ b/plasma/models/targets.py @@ -1,27 +1,32 @@ import numpy as np import abc -from keras.losses import hinge, squared_hinge, mean_absolute_percentage_error -from plasma.utils.evaluation import mae_np,mse_np,binary_crossentropy_np,hinge_np,squared_hinge_np +from keras.losses import hinge # squared_hinge, mean_absolute_percentage_error +from plasma.utils.evaluation import ( + mse_np, binary_crossentropy_np, hinge_np, + # mae_np, squared_hinge_np, + ) import keras.backend as K -#Requirement: larger value must mean disruption more likely. +# Requirement: larger value must mean disruption more likely. + + class Target(object): activation = 'linear' loss = 'mse' @abc.abstractmethod - def loss_np(y_true,y_pred): + def loss_np(y_true, y_pred): from plasma.conf import conf - return conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) + return conf['model']['loss_scale_factor']*mse_np(y_true, y_pred) @abc.abstractmethod - def remapper(ttd,T_warning): + def remapper(ttd, T_warning): return -ttd @abc.abstractmethod def threshold_range(T_warning): - return np.logspace(-1,4,100) + return np.logspace(-1, 4, 100) class BinaryTarget(Target): @@ -29,12 +34,13 @@ class BinaryTarget(Target): loss = 'binary_crossentropy' @staticmethod - def loss_np(y_true,y_pred): - from plasma.conf import conf - return conf['model']['loss_scale_factor']*binary_crossentropy_np(y_true,y_pred) + def loss_np(y_true, y_pred): + from plasma.conf import conf + return (conf['model']['loss_scale_factor'] + * binary_crossentropy_np(y_true, y_pred)) @staticmethod - def remapper(ttd,T_warning,as_array_of_shots=True): + def remapper(ttd, T_warning, as_array_of_shots=True): binary_ttd = 0*ttd mask = ttd < np.log10(T_warning) binary_ttd[mask] = 1.0 @@ -43,28 +49,27 @@ def remapper(ttd,T_warning,as_array_of_shots=True): @staticmethod def threshold_range(T_warning): - return np.logspace(-6,0,100) + return np.logspace(-6, 0, 100) class TTDTarget(Target): activation = 'linear' loss = 'mse' - @staticmethod - def loss_np(y_true,y_pred): - from plasma.conf import conf - return conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) + def loss_np(y_true, y_pred): + from plasma.conf import conf + return conf['model']['loss_scale_factor']*mse_np(y_true, y_pred) @staticmethod - def remapper(ttd,T_warning): + def remapper(ttd, T_warning): mask = ttd < np.log10(T_warning) ttd[~mask] = np.log10(T_warning) return -ttd @staticmethod def threshold_range(T_warning): - return np.linspace(-np.log10(T_warning),6,100) + return np.linspace(-np.log10(T_warning), 6, 100) class TTDInvTarget(Target): @@ -72,21 +77,21 @@ class TTDInvTarget(Target): loss = 'mse' @staticmethod - def loss_np(y_true,y_pred): - return mse_np(y_true,y_pred) + def loss_np(y_true, y_pred): + return mse_np(y_true, y_pred) @staticmethod - def remapper(ttd,T_warning): + def remapper(ttd, T_warning): eps = 1e-4 ttd = 10**(ttd) mask = ttd < T_warning ttd[~mask] = T_warning - ttd = (1.0)/(ttd+eps)#T_warning + ttd = (1.0)/(ttd+eps) # T_warning return ttd @staticmethod def threshold_range(T_warning): - return np.logspace(-6,np.log10(T_warning),100) + return np.logspace(-6, np.log10(T_warning), 100) class TTDLinearTarget(Target): @@ -94,26 +99,25 @@ class TTDLinearTarget(Target): loss = 'mse' @staticmethod - def loss_np(y_true,y_pred): + def loss_np(y_true, y_pred): from plasma.conf import conf - return conf['model']['loss_scale_factor']*mse_np(y_true,y_pred) - + return conf['model']['loss_scale_factor']*mse_np(y_true, y_pred) @staticmethod - def remapper(ttd,T_warning): + def remapper(ttd, T_warning): ttd = 10**(ttd) mask = ttd < T_warning - ttd[~mask] = 0#T_warning - ttd[mask] = T_warning - ttd[mask]#T_warning + ttd[~mask] = 0 # T_warning + ttd[mask] = T_warning - ttd[mask] # T_warning return ttd @staticmethod def threshold_range(T_warning): - return np.logspace(-6,np.log10(T_warning),100) + return np.logspace(-6, np.log10(T_warning), 100) -#implements a "maximum" driven loss function. Only the maximal value in the time sequence is punished. -#Also implements class weighting +# implements a "maximum" driven loss function. Only the maximal value in the +# time sequence is punished. Also implements class weighting class MaxHingeTarget(Target): activation = 'linear' fac = 1.0 @@ -122,35 +126,46 @@ class MaxHingeTarget(Target): def loss(y_true, y_pred): from plasma.conf import conf fac = MaxHingeTarget.fac - #overall_fac = np.prod(np.array(K.shape(y_pred)[1:]).astype(np.float32)) - overall_fac = K.prod(K.cast(K.shape(y_pred)[1:],K.floatx())) - max_val = K.max(y_pred,axis=-2) #temporal axis! - max_val1 = K.repeat(max_val,K.shape(y_pred)[-2]) - mask = K.cast(K.equal(max_val1,y_pred),K.floatx()) + # overall_fac = + # np.prod(np.array(K.shape(y_pred)[1:]).astype(np.float32)) + overall_fac = K.prod(K.cast(K.shape(y_pred)[1:], K.floatx())) + max_val = K.max(y_pred, axis=-2) # temporal axis! + max_val1 = K.repeat(max_val, K.shape(y_pred)[-2]) + mask = K.cast(K.equal(max_val1, y_pred), K.floatx()) y_pred1 = mask * y_pred + (1-mask) * y_true - weight_mask = K.mean(y_true,axis=-1) - weight_mask = K.cast(K.greater(weight_mask,0.0),K.floatx()) #positive label! + weight_mask = K.mean(y_true, axis=-1) + weight_mask = K.cast(K.greater(weight_mask, 0.0), + K.floatx()) # positive label! weight_mask = fac*weight_mask + (1 - weight_mask) - #return weight_mask*squared_hinge(y_true,y_pred1) - return conf['model']['loss_scale_factor']*overall_fac*weight_mask*hinge(y_true,y_pred1) + # return weight_mask*squared_hinge(y_true, y_pred1) + return conf['model']['loss_scale_factor'] * \ + overall_fac*weight_mask*hinge(y_true, y_pred1) @staticmethod def loss_np(y_true, y_pred): from plasma.conf import conf fac = MaxHingeTarget.fac - #print(y_pred.shape) + # print(y_pred.shape) overall_fac = np.prod(np.array(y_pred.shape).astype(np.float32)) - max_val = np.max(y_pred,axis=-2) #temporal axis! - max_val = np.reshape(max_val,max_val.shape[:-1] + (1,) + (max_val.shape[-1],)) - max_val = np.tile(max_val,(1,y_pred.shape[-2],1)) - mask = np.equal(max_val,y_pred) + max_val = np.max(y_pred, axis=-2) # temporal axis! + max_val = np.reshape( + max_val, max_val.shape[:-1] + (1,) + (max_val.shape[-1],)) + max_val = np.tile(max_val, (1, y_pred.shape[-2], 1)) + mask = np.equal(max_val, y_pred) mask = mask.astype(np.float32) y_pred = mask * y_pred + (1-mask) * y_true - weight_mask = np.greater(y_true,0.0).astype(np.float32) #positive label! + weight_mask = np.greater( + y_true, 0.0).astype( + np.float32) # positive label! weight_mask = fac*weight_mask + (1 - weight_mask) - #return np.mean(weight_mask*np.square(np.maximum(1. - y_true * y_pred, 0.)))#, axis=-1) only during training, here we want to completely sum up over all instances - return conf['model']['loss_scale_factor']*np.mean(overall_fac*weight_mask*np.maximum(1. - y_true * y_pred, 0.))#, axis=-1) only during training, here we want to completely sum up over all instances - + # return np.mean( + # weight_mask*np.square(np.maximum(1. - y_true * y_pred, 0.))) + # , axis=-1) + # only during training, here we want to completely sum up over all + # instances + return (conf['model']['loss_scale_factor'] + * np.mean(overall_fac * weight_mask + * np.maximum(1. - y_true * y_pred, 0.))) # def _loss_tensor_old(y_true, y_pred): # max_val = K.max(y_pred) #temporal axis! @@ -158,9 +173,8 @@ def loss_np(y_true, y_pred): # y_pred = mask * y_pred + (1-mask) * y_true # return squared_hinge(y_true,y_pred) - @staticmethod - def remapper(ttd,T_warning,as_array_of_shots=True): + def remapper(ttd, T_warning, as_array_of_shots=True): binary_ttd = 0*ttd mask = ttd < np.log10(T_warning) binary_ttd[mask] = 1.0 @@ -169,22 +183,24 @@ def remapper(ttd,T_warning,as_array_of_shots=True): @staticmethod def threshold_range(T_warning): - return np.concatenate((np.linspace(-2,-1.06,100),np.linspace(-1.06,-0.96,100),np.linspace(-0.96,2,50))) + return np.concatenate( + (np.linspace(-2, -1.06, 100), np.linspace(-1.06, -0.96, 100), + np.linspace(-0.96, 2, 50))) class HingeTarget(Target): activation = 'linear' - loss = 'hinge' #hinge - + loss = 'hinge' # hinge + @staticmethod def loss_np(y_true, y_pred): from plasma.conf import conf - return conf['model']['loss_scale_factor']*hinge_np(y_true,y_pred) - #return squared_hinge_np(y_true,y_pred) - + return conf['model']['loss_scale_factor']*hinge_np(y_true, y_pred) + # return squared_hinge_np(y_true, y_pred) + @staticmethod - def remapper(ttd,T_warning,as_array_of_shots=True): + def remapper(ttd, T_warning, as_array_of_shots=True): binary_ttd = 0*ttd mask = ttd < np.log10(T_warning) binary_ttd[mask] = 1.0 @@ -193,4 +209,6 @@ def remapper(ttd,T_warning,as_array_of_shots=True): @staticmethod def threshold_range(T_warning): - return np.concatenate((np.linspace(-2,-1.06,100),np.linspace(-1.06,-0.96,100),np.linspace(-0.96,2,50))) + return np.concatenate( + (np.linspace(-2, -1.06, 100), np.linspace(-1.06, -0.96, 100), + np.linspace(-0.96, 2, 50))) From 040407aeee557a057f30852df24e3718a28b1982 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 24 Sep 2019 14:14:25 -0500 Subject: [PATCH 089/272] Clean up files in plasma/preprocessor --- plasma/preprocessor/augment.py | 103 +++++----- plasma/preprocessor/normalize.py | 319 +++++++++++++++++------------- plasma/preprocessor/preprocess.py | 205 +++++++++++-------- 3 files changed, 365 insertions(+), 262 deletions(-) diff --git a/plasma/preprocessor/augment.py b/plasma/preprocessor/augment.py index 38903f1f..9ecb4d43 100644 --- a/plasma/preprocessor/augment.py +++ b/plasma/preprocessor/augment.py @@ -1,13 +1,11 @@ from __future__ import print_function -import os -import time,sys import abc import numpy as np import random class ByShotAugmentator(object): - def __init__(self,normalizer): + def __init__(self, normalizer): self.normalizer = normalizer def __str__(self): @@ -15,35 +13,39 @@ def __str__(self): s += "\n including by shot augmentation" return s - def apply(self,shot): + def apply(self, shot): ''' - The purpose of the method is to apply normalization to a shot and then optionally apply augmentation with a function that is individual to every shot. + The purpose of the method is to apply normalization to a shot and then + optionally apply augmentation with a function that is individual to + every shot. - Argument list: + Argument list: - shot: plasma shot. Should contain an augment function Config parameters list: - - conf['data']['augment_during_training']: boolean flag, yes or no to augment during training + - conf['data']['augment_during_training']: boolean flag, yes or no to + augment during training + ''' - #first just apply normalization as usual. + # first just apply normalization as usual. self.normalizer.apply(shot) if shot.augmentation_fn is not None: shot.augmentation_fn(shot) - def set_inference_mode(self,is_inference): + def set_inference_mode(self, is_inference): self.normalizer.set_inference_mode(is_inference) class AbstractAugmentator(object): - def __init__(self,normalizer,is_inference,conf): + def __init__(self, normalizer, is_inference, conf): self.conf = conf self.to_augment_str = self.conf['data']['signal_to_augment'] self.normalizer = normalizer self.is_inference = is_inference - #set whether we are training or testing - def set_inference(self,is_inference): + # set whether we are training or testing + def set_inference(self, is_inference): self.is_inference = is_inference def __str__(self): @@ -52,78 +54,89 @@ def __str__(self): s += "Signal to augmented: {}\n".format(self.to_augment_str) s += "Is inference: {}\n".format(self.is_inference) return s - - #for compatibility with code that changes the mode of the normalizer - def set_inference_mode(self,is_inference): + + # for compatibility with code that changes the mode of the normalizer + def set_inference_mode(self, is_inference): self.normalizer.set_inference_mode(is_inference) @abc.abstractmethod - def apply(self,shot): + def apply(self, shot): pass @abc.abstractmethod - def augment(self,sig): + def augment(self, sig): pass - + class Augmentator(AbstractAugmentator): - def apply(self,shot): + def apply(self, shot): ''' - The purpose of the method is to apply normalization to a shot and then optionally apply augmentation. - During inference, a specific signal (one at a time) is augmented based on the string supplied in the config file. - During training, augment a random signal (again, one at a time) or do not augment at all. + The purpose of the method is to apply normalization to a shot and then + optionally apply augmentation. During inference, a specific signal + (one at a time) is augmented based on the string supplied in the config + file. During training, augment a random signal (again, one at a time) + or do not augment at all. It performs calls to: Augmentator.augment(), random.random.choice - Argument list: + Argument list: - shot: plasma shot Config parameters list: - - conf['data']['augment_during_training']: boolean flag, yes or no to augment during training + - conf['data']['augment_during_training']: boolean flag, yes or no to + augment during training + ''' - #first just apply normalization as usual. + # first just apply normalization as usual. self.normalizer.apply(shot) if self.is_inference: - #during inference, augment a specific signal (one at a time) + # during inference, augment a specific signal (one at a time) to_augment_str = self.to_augment_str - else: - #during training augment a random signal, one at a time + else: + # during training augment a random signal, one at a time if self.conf['data']['augment_during_training']: - to_augment_str = random.choice([x.description for x in shot.signals]) + to_augment_str = random.choice( + [x.description for x in shot.signals]) else: - to_augment_str = None + to_augment_str = None if to_augment_str is not None: - #FIXME might be better to use search. are we always going to augment 1 signal at a time? - for (i,sig) in enumerate(shot.signals): + # FIXME might be better to use search. are we always going to + # augment 1 signal at a time? + for (i, sig) in enumerate(shot.signals): if sig.description == to_augment_str: - print ('Augmenting {} signal'.format(sig.description)) - shot.signals_dict[sig] = self.augment(shot.signals_dict[sig]) + print('Augmenting {} signal'.format(sig.description)) + shot.signals_dict[sig] = self.augment( + shot.signals_dict[sig]) - def augment(self,signal,strength=10): + def augment(self, signal, strength=10): ''' - The purpose of the method is to modify a signal specified by a configuration parameter or at random according to - a specific mode. Modes include: noise, zeroing and no augmentation. + The purpose of the method is to modify a signal specified by a + configuration parameter or at random according to a specific + mode. Modes include: noise, zeroing and no augmentation. It performs calls to: numpy random number generator - Argument list: + Argument list: - signal: signal - - strength: strength of the noise, measured in standard deviations. Integer, default value: 10 + - strength: strength of the noise, measured in standard + deviations. Integer, default value: 10 Config parameters list: - - conf['data']['augmentation_mode']: categorical config parameter specifying how to augment. Possible values - include "noise", "zero" and "none" (strings) + - conf['data']['augmentation_mode']: categorical config parameter + specifying how to augment. Possible values include "noise", "zero" and + "none" (strings) - Returns: + Returns: - signal: augmented signal ... numpy array of numeric types? ''' if self.conf['data']['augmentation_mode'] == "noise": - return np.random.normal(0,strength,signal.shape) + return np.random.normal(0, strength, signal.shape) elif self.conf['data']['augmentation_mode'] == "zero": - return signal*0.0 #if "set to zero" augmentation. Can control in conf. + # if "set to zero" augmentation. Can control in conf. + return signal*0.0 elif self.conf['data']['augmentation_mode'] == "none": - return signal #if no augmentation. Should be the default in conf. + return signal # if no augmentation. Should be the default in conf. else: print("Unknown augmentation mode. Exiting") exit(-1) diff --git a/plasma/preprocessor/normalize.py b/plasma/preprocessor/normalize.py index f9f23ac0..3f181d0d 100644 --- a/plasma/preprocessor/normalize.py +++ b/plasma/preprocessor/normalize.py @@ -10,32 +10,34 @@ from __future__ import print_function import os -import time,sys +import time +import sys import abc import numpy as np -from scipy.signal import exponential,correlate +from scipy.signal import exponential, correlate import pathos.multiprocessing as mp from plasma.primitives.shots import ShotList, Shot -from plasma.utils.processing import get_signal_slices '''TODO - incorporate stats, pass machine (perhaps save machine in stats object!) - incorporate stats, have a dictionary of aggregate stats for every machine. -- check "is_previously_saved" by making sure there is a normalizer for every machine -''' - +- check "is_previously_saved" by making sure there is a normalizer for every +machine +''' -#######NORMALIZATION########## +################# +# NORMALIZATION # +################# class Stats(object): pass class Normalizer(object): - def __init__(self,conf): + def __init__(self, conf): self.num_processed = dict() self.num_disruptive = dict() self.conf = conf @@ -50,15 +52,15 @@ def __str__(self): pass @abc.abstractmethod - def extract_stats(self,shot): + def extract_stats(self, shot): pass @abc.abstractmethod - def incorporate_stats(self,stats): + def incorporate_stats(self, stats): pass @abc.abstractmethod - def apply(self,shot): + def apply(self, shot): pass @abc.abstractmethod @@ -69,19 +71,20 @@ def save_stats(self): def load_stats(self): pass - def set_inference_mode(self,val): + def set_inference_mode(self, val): self.inference_mode = val - def ensure_machine(self,machine): + def ensure_machine(self, machine): if machine not in self.means: - self.num_processed[machine] = 0 - self.num_disruptive[machine] = 0 - ######Modify the above to change the specifics of the normalization scheme####### + self.num_processed[machine] = 0 + self.num_disruptive[machine] = 0 + # Modify the above to change the specifics of the normalization scheme def train(self): conf = self.conf - #only use training shots here!! "Don't touch testing shots" - shot_files = conf['paths']['shot_files']# + conf['paths']['shot_files_test'] + # only use training shots here!! "Don't touch testing shots" + # + conf['paths']['shot_files_test'] + shot_files = conf['paths']['shot_files'] shot_files_all = conf['paths']['shot_files_all'] all_machines = set([file.machine for file in shot_files_all]) train_machines = set([file.machine for file in shot_files]) @@ -89,22 +92,22 @@ def train(self): if train_machines >= all_machines: shot_files_use = shot_files else: - print('Testing set contains new machine, using testing set to train normalizer for that machine.') + print('Testing set contains new machine, using testing set ', + 'to train normalizer for that machine.') shot_files_use = shot_files_all # shot_list_dir = conf['paths']['shot_list_dir'] - use_shots = max(400,conf['data']['use_shots']) - return self.train_on_files(shot_files_use,use_shots,all_machines) + use_shots = max(400, conf['data']['use_shots']) + return self.train_on_files(shot_files_use, use_shots, all_machines) - - def train_on_files(self,shot_files,use_shots,all_machines): + def train_on_files(self, shot_files, use_shots, all_machines): conf = self.conf - all_signals = conf['paths']['all_signals'] + all_signals = conf['paths']['all_signals'] shot_list = ShotList() - shot_list.load_from_shot_list_files_objects(shot_files,all_signals) - shot_list_picked = shot_list.random_sublist(use_shots) + shot_list.load_from_shot_list_files_objects(shot_files, all_signals) + shot_list_picked = shot_list.random_sublist(use_shots) - previously_saved,machines_saved = self.previously_saved_stats() + previously_saved, machines_saved = self.previously_saved_stats() machines_to_compute = all_machines - machines_saved recompute = conf['data']['recompute_normalization'] if recompute: @@ -114,34 +117,43 @@ def train_on_files(self,shot_files,use_shots,all_machines): if not previously_saved or len(machines_to_compute) > 0: if previously_saved: self.load_stats() - print('computing normalization for machines {}'.format(machines_to_compute)) - use_cores = max(1,mp.cpu_count()-2) + print('computing normalization for machines {}'.format( + machines_to_compute)) + use_cores = max(1, mp.cpu_count()-2) pool = mp.Pool(use_cores) - print('running in parallel on {} processes'.format(pool._processes)) + print('running in parallel on {} processes'.format( + pool._processes)) start_time = time.time() - for (i,stats) in enumerate(pool.imap_unordered(self.train_on_single_shot,shot_list_picked)): - #for (i,stats) in enumerate(map(self.train_on_single_shot,shot_list_picked)): + for (i, stats) in enumerate(pool.imap_unordered( + self.train_on_single_shot, + shot_list_picked)): + # for (i,stats) in + # enumerate(map(self.train_on_single_shot,shot_list_picked)): if stats.machine in machines_to_compute: self.incorporate_stats(stats) self.machines.add(stats.machine) - sys.stdout.write('\r' + '{}/{}'.format(i,len(shot_list_picked))) + sys.stdout.write('\r' + + '{}/{}'.format(i, len(shot_list_picked))) pool.close() pool.join() - print('Finished Training Normalizer on {} files in {} seconds'.format(len(shot_list_picked),time.time()-start_time)) + print('Finished Training Normalizer on ', + '{} files in {} seconds'.format(len(shot_list_picked), + time.time()-start_time)) self.save_stats() else: self.load_stats() print(self) - - def cut_end_of_shot(self,shot): + def cut_end_of_shot(self, shot): cut_shot_ends = self.conf['data']['cut_shot_ends'] - if not self.inference_mode and cut_shot_ends: #only cut shots during training + # only cut shots during training + if not self.inference_mode and cut_shot_ends: T_min_warn = self.conf['data']['T_min_warn'] for key in shot.signals_dict: - shot.signals_dict[key] = shot.signals_dict[key][:-T_min_warn,:] + shot.signals_dict[key] = shot.signals_dict[key][:-T_min_warn, + :] shot.ttd = shot.ttd[:-T_min_warn] # def apply_mask(self,shot): @@ -151,18 +163,19 @@ def cut_end_of_shot(self,shot): # def apply_positivity_mask(self,shot): # mask = self.conf['paths']['positivity_mask'] # mask = [np.array(subl) for subl in mask] - # indices = np.concatenate([indices_sublist[mask[i]] for i,indices_sublist in enumerate(self.get_indices_list())]) + # indices = np.concatenate([indices_sublist[mask[i]] for + # i,indices_sublist in enumerate(self.get_indices_list())]) # shot.signals[:,indices] = np.clip(shot.signals[:,indices],0,np.Inf) - def train_on_single_shot(self,shot): - assert isinstance(shot,Shot), 'should be instance of shot' + def train_on_single_shot(self, shot): + assert isinstance(shot, Shot), 'should be instance of shot' processed_prepath = self.conf['paths']['processed_prepath'] shot.restore(processed_prepath) - #print(shot) - stats = self.extract_stats(shot) + # print(shot) + stats = self.extract_stats(shot) shot.make_light() return stats - + def ensure_save_directory(self): prepath = os.path.dirname(self.path) if not os.path.exists(prepath): @@ -170,27 +183,25 @@ def ensure_save_directory(self): def previously_saved_stats(self): if not os.path.isfile(self.path): - return False,set([]) + return False, set([]) else: - dat = np.load(self.path,encoding="latin1") + dat = np.load(self.path, encoding="latin1") machines = dat['machines'][()] - ret = all([m in machines for m in self.conf['paths']['all_machines']]) + ret = all( + [m in machines for m in self.conf['paths']['all_machines']]) if not ret: print(machines) print(self.conf['paths']['all_machines']) print('Not all machines present. Recomputing normalizer.') - return True,set(machines) + return True, set(machines) # def get_indices_list(self): # return get_signal_slices(self.conf['paths']['signals_dirs']) - - - class MeanVarNormalizer(Normalizer): - def __init__(self,conf): - Normalizer.__init__(self,conf) + def __init__(self, conf): + Normalizer.__init__(self, conf) self.means = dict() self.stds = dict() self.bound = self.conf['data']['norm_stat_range'] @@ -198,19 +209,23 @@ def __init__(self,conf): def __str__(self): s = '' for machine in self.means: - means = np.median(self.means[machine],axis=0) - stds = np.median(self.stds[machine],axis=0) - s += 'Machine: {}:\nMean Var Normalizer.\nmeans: {}\nstds: {}'.format(machine,means,stds) - return s + means = np.median(self.means[machine], axis=0) + stds = np.median(self.stds[machine], axis=0) + s += 'Machine: {}:\nMean Var Normalizer.\n'.format(machine) + s += 'means: {}\nstds: {}'.format(means, stds) + return s - def extract_stats(self,shot): + def extract_stats(self, shot): stats = Stats() if shot.valid: list_of_signals = shot.get_individual_signal_arrays() num_signals = len(list_of_signals) - stats.means = np.reshape(np.array([np.mean(sig) for sig in list_of_signals]),(1,num_signals)) - stats.stds = np.reshape(np.array([np.std(sig,dtype=np.float64) for sig in list_of_signals]),(1,num_signals)) - + stats.means = np.reshape(np.array([np.mean(sig) for + sig in list_of_signals]), + (1, num_signals)) + stats.stds = np.reshape(np.array([np.std(sig, dtype=np.float64) for + sig in list_of_signals]), + (1, num_signals)) stats.is_disruptive = shot.is_disruptive else: print('Warning: shot {} not valid, omitting'.format(shot.number)) @@ -218,9 +233,7 @@ def extract_stats(self,shot): stats.machine = shot.machine return stats - - - def incorporate_stats(self,stats): + def incorporate_stats(self, stats): machine = stats.machine self.ensure_machine(stats.machine) if stats.valid: @@ -228,95 +241,113 @@ def incorporate_stats(self,stats): stds = stats.stds if self.num_processed[machine] == 0: self.means[machine] = means - self.stds[machine] = stds + self.stds[machine] = stds else: - self.means[machine] = np.concatenate((self.means[machine],means),axis=0) - self.stds[machine] = np.concatenate((self.stds[machine],stds),axis=0) + self.means[machine] = np.concatenate( + (self.means[machine], means), axis=0) + self.stds[machine] = np.concatenate( + (self.stds[machine], stds), axis=0) self.num_processed[machine] = self.num_processed[machine] + 1 - self.num_disruptive[machine] = self.num_disruptive[machine] + (1 if stats.is_disruptive else 0) + self.num_disruptive[machine] = self.num_disruptive[machine] + \ + (1 if stats.is_disruptive else 0) - - def apply(self,shot): + def apply(self, shot): apply_positivity(shot) m = shot.machine - assert self.means[m] is not None and self.stds[m] is not None, "self.means or self.stds not initialized" - means = np.median(self.means[m],axis=0) - stds = np.median(self.stds[m],axis=0) - for (i,sig) in enumerate(shot.signals): + assert self.means[m] is not None and self.stds[m] is not None, ( + "self.means or self.stds not initialized") + means = np.median(self.means[m], axis=0) + stds = np.median(self.stds[m], axis=0) + for (i, sig) in enumerate(shot.signals): if sig.normalize: stds_curr = stds[i] if stds_curr == 0.0: stds_curr = 1.0 - shot.signals_dict[sig] = (shot.signals_dict[sig] - means[i])/stds_curr - shot.signals_dict[sig] = np.clip(shot.signals_dict[sig],-self.bound,self.bound) + shot.signals_dict[sig] = ( + shot.signals_dict[sig] - means[i])/stds_curr + shot.signals_dict[sig] = np.clip( + shot.signals_dict[sig], -self.bound, self.bound) - shot.ttd = self.remapper(shot.ttd,self.conf['data']['T_warning']) + shot.ttd = self.remapper(shot.ttd, self.conf['data']['T_warning']) self.cut_end_of_shot(shot) # self.apply_positivity_mask(shot) # self.apply_mask(shot) - def save_stats(self): # standard_deviations = dat['standard_deviations'] # num_processed = dat['num_processed'] # num_disruptive = dat['num_disruptive'] self.ensure_save_directory() - np.savez(self.path,means = self.means,stds = self.stds, - num_processed=self.num_processed,num_disruptive=self.num_disruptive,machines=self.machines) - print('saved normalization data from {} shots ( {} disruptive )'.format(self.num_processed,self.num_disruptive)) + np.savez( + self.path, + means=self.means, + stds=self.stds, + num_processed=self.num_processed, + num_disruptive=self.num_disruptive, + machines=self.machines) + print( + 'saved normalization data from {} shots ( {} disruptive )'.format( + self.num_processed, + self.num_disruptive)) def load_stats(self): assert self.previously_saved_stats()[0], "stats not saved before" - dat = np.load(self.path,encoding="latin1") + dat = np.load(self.path, encoding="latin1") self.means = dat['means'][()] self.stds = dat['stds'][()] self.num_processed = dat['num_processed'][()] self.num_disruptive = dat['num_disruptive'][()] self.machines = dat['machines'][()] for machine in self.means: - print('Machine {}:'.format(machine)) - print('loaded normalization data from {} shots ( {} disruptive )'.format(self.num_processed,self.num_disruptive)) - #print('loading normalization data from {} shots, {} disruptive'.format(num_processed,num_disruptive)) + print('Machine {}:'.format(machine)) + print('loaded normalization data from ', + '{} shots ( {} disruptive )'.format(self.num_processed, + self.num_disruptive)) class VarNormalizer(MeanVarNormalizer): - def apply(self,shot): + def apply(self, shot): apply_positivity(shot) - assert self.means is not None and self.stds is not None, "self.means or self.stds not initialized" + assert self.means is not None and self.stds is not None, ( + "self.means or self.stds not initialized") m = shot.machine - stds = np.median(self.stds[m],axis=0) - for (i,sig) in enumerate(shot.signals): + stds = np.median(self.stds[m], axis=0) + for (i, sig) in enumerate(shot.signals): if sig.normalize: stds_curr = stds[i] if stds_curr == 0.0: stds_curr = 1.0 shot.signals_dict[sig] = (shot.signals_dict[sig])/stds_curr - shot.signals_dict[sig] = np.clip(shot.signals_dict[sig],-self.bound,self.bound) - shot.ttd = self.remapper(shot.ttd,self.conf['data']['T_warning']) + shot.signals_dict[sig] = np.clip( + shot.signals_dict[sig], -self.bound, self.bound) + shot.ttd = self.remapper(shot.ttd, self.conf['data']['T_warning']) self.cut_end_of_shot(shot) def __str__(self): s = '' for m in self.stds: - stds = np.median(self.stds[m],axis=0) - s += 'Machine: {}:\n'.format(m) - s += 'Var Normalizer.\nstds: {}\n'.format(stds) + stds = np.median(self.stds[m], axis=0) + s += 'Machine: {}:\n'.format(m) + s += 'Var Normalizer.\nstds: {}\n'.format(stds) return s class AveragingVarNormalizer(VarNormalizer): - def apply(self,shot): + def apply(self, shot): apply_positivity(shot) - super(AveragingVarNormalizer,self).apply(shot) + super(AveragingVarNormalizer, self).apply(shot) window_decay = self.conf['data']['window_decay'] window_size = self.conf['data']['window_size'] - window = exponential(window_size,0,window_decay,False) + window = exponential(window_size, 0, window_decay, False) window /= np.sum(window) - for (i,sig) in enumerate(shot.signals): + for (i, sig) in enumerate(shot.signals): if sig.normalize: - shot.signals_dict[sig] = apply_along_axis(lambda m : correlate(m,window,'valid'),axis=0,arr=shot.signals_dict[sig]) - shot.signals_dict[sig] = np.clip(shot.signals_dict[sig],-self.bound,self.bound) + shot.signals_dict[sig] = np.apply_along_axis( + lambda m: correlate(m, window, 'valid'), + axis=0, arr=shot.signals_dict[sig]) + shot.signals_dict[sig] = np.clip( + shot.signals_dict[sig], -self.bound, self.bound) shot.ttd = shot.ttd[-shot.signals.shape[0]:] def __str__(self): @@ -324,27 +355,30 @@ def __str__(self): window_size = self.conf['data']['window_size'] s = '' for m in self.stds: - stds = np.median(self.stds[m],axis=0) - s += 'Machine: {}:\n'.format(m) - s += 'Averaging Var Normalizer.\nstds: {}\nWindow size: {}, Window decay: {}'.format(stds,window_size,window_decay) + stds = np.median(self.stds[m], axis=0) + s += 'Machine: {}:\n'.format(m) + s += 'Averaging Var Normalizer.\nstds: ' + s += ' {}\nWindow size: {}, Window decay: {}'.format( + stds, window_size, window_decay) return s class MinMaxNormalizer(Normalizer): - def __init__(self,conf): - Normalizer.__init__(self,conf) + def __init__(self, conf): + Normalizer.__init__(self, conf) self.minimums = None self.maximums = None self.bound = self.conf['data']['norm_stat_range'] - def __str__(self): s = '' for m in self.minimums: - s += 'Machine {}:\n.Min Max Normalizer.\nminimums: {}\nmaximums: {}'.format(m,self.minimums[m],self.maximums[m]) - return s + s += 'Machine {}:\n.Min Max Normalizer.\n'.format(m, + self.minimums[m]) + s += 'minimums: {}\nmaximums: {}'.format(self.maximums[m]) + return s - def extract_stats(self,shot): + def extract_stats(self, shot): stats = Stats() if shot.valid: list_of_signals = shot.get_individual_signal_arrays() @@ -357,8 +391,7 @@ def extract_stats(self,shot): stats.machine = shot.machine return stats - - def incorporate_stats(self,stats): + def incorporate_stats(self, stats): self.ensure_machine(stats.machine) if stats.valid: m = stats.machine @@ -368,25 +401,30 @@ def incorporate_stats(self,stats): self.minimums[m] = minimums self.maximums[m] = maximums else: - self.minimums[m] = (self.num_processed[m]*self.minimums + minimums)/(self.num_processed[m] + 1.0)#snp.min(vstack((self.minimums,minimums)),0) - self.maximums[m] = (self.num_processed[m]*self.maximums + maximums)/(self.num_processed[m] + 1.0)#snp.max(vstack((self.maximums,maximums)),0) + self.minimums[m] = (self.num_processed[m]*self.minimums + + minimums)/(self.num_processed[m] + 1.0) + self.maximums[m] = (self.num_processed[m]*self.maximums + + maximums)/(self.num_processed[m] + 1.0) self.num_processed[m] = self.num_processed[m] + 1 - self.num_disruptive[m] = self.num_disruptive[m] + (1 if stats.is_disruptive else 0) - + self.num_disruptive[m] = self.num_disruptive[m] + \ + (1 if stats.is_disruptive else 0) - def apply(self,shot): + def apply(self, shot): apply_positivity(shot) - assert(self.minimums is not None and self.maximums is not None) + assert(self.minimums is not None and self.maximums is not None) m = shot.machine curr_range = (self.maximums[m] - self.minimums[m]) if curr_range == 0.0: - curr_range = 1.0 + curr_range = 1.0 shot.signals = (shot.signals - self.minimums[m])/curr_range - for (i,sig) in enumerate(shot.signals): + for (i, sig) in enumerate(shot.signals): if sig.normalize: - shot.signals_dict[sig] = (shot.signals_dict[sig] - self.minimums[m])/(self.maximums[m] - self.minimums[m]) - shot.signals_dict[sig] = np.clip(shot.signals_dict[sig],-self.bound,self.bound) - shot.ttd = self.remapper(shot.ttd,self.conf['data']['T_warning']) + shot.signals_dict[sig] = ( + shot.signals_dict[sig] - self.minimums[m])/( + self.maximums[m] - self.minimums[m]) + shot.signals_dict[sig] = np.clip( + shot.signals_dict[sig], -self.bound, self.bound) + shot.ttd = self.remapper(shot.ttd, self.conf['data']['T_warning']) self.cut_end_of_shot(shot) # self.apply_positivity_mask(shot) # self.apply_mask(shot) @@ -396,29 +434,42 @@ def save_stats(self): # num_processed = dat['num_processed'] # num_disruptive = dat['num_disruptive'] self.ensure_save_directory() - np.savez(self.path,minimums = self.minimums,maximums = self.maximums, - num_processed=self.num_processed,num_disruptive=self.num_disruptive,machines=self.machines) - print('saved normalization data from {} shots ( {} disruptive )'.format(self.num_processed,self.num_disruptive)) + np.savez( + self.path, + minimums=self.minimums, + maximums=self.maximums, + num_processed=self.num_processed, + num_disruptive=self.num_disruptive, + machines=self.machines) + print( + 'saved normalization data from {} shots ( {} disruptive )'.format( + self.num_processed, + self.num_disruptive)) def load_stats(self): assert(self.previously_saved_stats()[0]) - dat = np.load(self.path,encoding="latin1") + dat = np.load(self.path, encoding="latin1") self.minimums = dat['minimums'][()] self.maximums = dat['maximums'][()] self.num_processed = dat['num_processed'][()] self.num_disruptive = dat['num_disruptive'][()] self.machines = dat['machines'][()] - print('loaded normalization data from {} shots ( {} disruptive )'.format(self.num_processed,self.num_disruptive)) - #print('loading normalization data from {} shots, {} disruptive'.format(num_processed,num_disruptive)) + print( + 'loaded normalization data from {} shots ( {} disruptive )'.format( + self.num_processed, + self.num_disruptive)) -def get_individual_shot_file(prepath,shot_num,ext='.txt'): - return prepath + str(shot_num) + ext +def get_individual_shot_file(prepath, shot_num, ext='.txt'): + return prepath + str(shot_num) + ext def apply_positivity(shot): - for (i,sig) in enumerate(shot.signals): - if hasattr(sig,"is_strictly_positive"): #backwards compatibility when this attribute didn't exist + for (i, sig) in enumerate(shot.signals): + if hasattr(sig, "is_strictly_positive"): + # backwards compatibility when this attribute didn't exist if sig.is_strictly_positive: - #print ('Applying positivity constraint to {} signal'.format(sig.description)) - shot.signals_dict[sig]=np.clip(shot.signals_dict[sig],0,np.inf) + # print ('Applying positivity constraint to {} + # signal'.format(sig.description)) + shot.signals_dict[sig] = np.clip( + shot.signals_dict[sig], 0, np.inf) diff --git a/plasma/preprocessor/preprocess.py b/plasma/preprocessor/preprocess.py index 1d12f6c3..17a5699f 100644 --- a/plasma/preprocessor/preprocess.py +++ b/plasma/preprocessor/preprocess.py @@ -9,7 +9,7 @@ ''' from __future__ import print_function -from os import listdir,remove +from os import listdir # , remove import time import sys import os @@ -17,42 +17,44 @@ import numpy as np import pathos.multiprocessing as mp -from plasma.utils.processing import * +from plasma.utils.processing import append_to_filename from plasma.primitives.shots import ShotList from plasma.utils.downloading import mkdirdepth + class Preprocessor(object): - def __init__(self,conf): + def __init__(self, conf): self.conf = conf - def clean_shot_lists(self): shot_list_dir = self.conf['paths']['shot_list_dir'] - paths = [os.path.join(shot_list_dir, f) for f in listdir(shot_list_dir) if os.path.isfile(os.path.join(shot_list_dir, f))] + paths = [ + os.path.join( + shot_list_dir, + f) for f in listdir(shot_list_dir) if os.path.isfile( + os.path.join( + shot_list_dir, + f))] for path in paths: self.clean_shot_list(path) - - def clean_shot_list(self,path): + def clean_shot_list(self, path): data = np.loadtxt(path) - ending_idx = path.rfind('.') - new_path = append_to_filename(path,'_clear') + # ending_idx = path.rfind('.') + new_path = append_to_filename(path, '_clear') if len(np.shape(data)) < 2: - #nondisruptive + # nondisruptive nd_times = -1.0*np.ones_like(data) - data_two_column = np.vstack((data,nd_times)).transpose() - np.savetxt(new_path,data_two_column,fmt = '%d %f') + data_two_column = np.vstack((data, nd_times)).transpose() + np.savetxt(new_path, data_two_column, fmt='%d %f') print('created new file: {}'.format(new_path)) print('deleting old file: {}'.format(path)) os.remove(path) - def all_are_preprocessed(self): return os.path.isfile(self.get_shot_list_path()) - - def preprocess_all(self): conf = self.conf shot_files_all = conf['paths']['shot_files_all'] @@ -60,47 +62,65 @@ def preprocess_all(self): # shot_files_test = conf['paths']['shot_files_test'] # shot_list_dir = conf['paths']['shot_list_dir'] use_shots = conf['data']['use_shots'] - train_frac = conf['training']['train_frac'] - use_shots_train = int(round(train_frac*use_shots)) - use_shots_test = int(round((1-train_frac)*use_shots)) -# print(use_shots_train) -# print(use_shots_test) #each print out 100,000 + # train_frac = conf['training']['train_frac'] + # use_shots_train = int(round(train_frac*use_shots)) + # use_shots_test = int(round((1-train_frac)*use_shots)) + # print(use_shots_train) + # print(use_shots_test) #each print out 100,000 + # if len(shot_files_test) > 0: - # return self.preprocess_from_files(shot_list_dir,shot_files_train,machines_train,use_shots_train) + \ - # self.preprocess_from_files(shot_list_dir,shot_files_test,machines_train,use_shots_test) + # return + # self.preprocess_from_files(shot_list_dir,shot_files_train, + # machines_train,use_shots_train) + # + self.preprocess_from_files(shot_list_dir, + # shot_files_test,machines_train,use_shots_test) # else: - return self.preprocess_from_files(shot_files_all,use_shots) + return self.preprocess_from_files(shot_files_all, use_shots) - - def preprocess_from_files(self,shot_files,use_shots): - #all shots, including invalid ones - all_signals = self.conf['paths']['all_signals'] + def preprocess_from_files(self, shot_files, use_shots): + # all shots, including invalid ones + all_signals = self.conf['paths']['all_signals'] shot_list = ShotList() - shot_list.load_from_shot_list_files_objects(shot_files,all_signals) + shot_list.load_from_shot_list_files_objects(shot_files, all_signals) shot_list_picked = shot_list.random_sublist(use_shots) - #empty + # empty used_shots = ShotList() - use_cores = max(1,mp.cpu_count()-2) + use_cores = max(1, mp.cpu_count()-2) pool = mp.Pool(use_cores) print('running in parallel on {} processes'.format(pool._processes)) start_time = time.time() - for (i,shot) in enumerate(pool.imap_unordered(self.preprocess_single_file,shot_list_picked)): - #for (i,shot) in enumerate(map(self.preprocess_single_file,shot_list_picked)): - sys.stdout.write('\r{}/{}'.format(i,len(shot_list_picked))) + for ( + i, + shot) in enumerate( + pool.imap_unordered( + self.preprocess_single_file, + shot_list_picked)): + # for (i,shot) in + # enumerate(map(self.preprocess_single_file,shot_list_picked)): + sys.stdout.write('\r{}/{}'.format(i, len(shot_list_picked))) used_shots.append_if_valid(shot) pool.close() pool.join() - print('Finished Preprocessing {} files in {} seconds'.format(len(shot_list_picked),time.time()-start_time)) - print('Omitted {} shots of {} total.'.format(len(shot_list_picked) - len(used_shots),len(shot_list_picked))) - print('{}/{} disruptive shots'.format(used_shots.num_disruptive(),len(used_shots))) + print('Finished Preprocessing {} files in {} seconds'.format( + len(shot_list_picked), time.time()-start_time)) + print( + 'Omitted {} shots of {} total.'.format( + len(shot_list_picked) + - len(used_shots), + len(shot_list_picked))) + print('{}/{} disruptive shots'.format(used_shots.num_disruptive(), + len(used_shots))) if len(used_shots) == 0: - print("WARNING: All shots were omitted, please ensure raw data is complete and available at {}.".format(self.conf['paths']['signal_prepath'])) - return used_shots + print( + "WARNING: All shots were omitted, please ensure raw data " + " is complete and available at {}.".format( + self.conf['paths']['signal_prepath'])) + return used_shots - def preprocess_single_file(self,shot): + def preprocess_single_file(self, shot): processed_prepath = self.conf['paths']['processed_prepath'] recompute = self.conf['data']['recompute'] # print('({}/{}): '.format(num_processed,use_shots)) @@ -110,52 +130,62 @@ def preprocess_single_file(self,shot): else: try: - shot.restore(processed_prepath,light=True) + shot.restore(processed_prepath, light=True) sys.stdout.write('\r{} exists.'.format(shot.number)) - except: + except BaseException: shot.preprocess(self.conf) shot.save(processed_prepath) - sys.stdout.write('\r{} exists but corrupted, resaved.'.format(shot.number)) + sys.stdout.write( + '\r{} exists but corrupted, resaved.'.format( + shot.number)) shot.make_light() - return shot - + return shot def get_individual_channel_dirs(self): - signals_dirs = self.conf['paths']['signals_dirs'] + return self.conf['paths']['signals_dirs'] def get_shot_list_path(self): return self.conf['paths']['saved_shotlist_path'] def load_shotlists(self): path = self.get_shot_list_path() - data = np.load(path,encoding="latin1") + data = np.load(path, encoding="latin1") shot_list_train = data['shot_list_train'][()] shot_list_validate = data['shot_list_validate'][()] shot_list_test = data['shot_list_test'][()] - if isinstance(shot_list_train,ShotList): - return shot_list_train,shot_list_validate,shot_list_test + if isinstance(shot_list_train, ShotList): + return shot_list_train, shot_list_validate, shot_list_test else: - return ShotList(shot_list_train),ShotList(shot_list_validate),ShotList(shot_list_test) - - - def save_shotlists(self,shot_list_train,shot_list_validate,shot_list_test): + return ShotList(shot_list_train), ShotList( + shot_list_validate), ShotList(shot_list_test) + + def save_shotlists( + self, + shot_list_train, + shot_list_validate, + shot_list_test): path = self.get_shot_list_path() mkdirdepth(path) - np.savez(path,shot_list_train=shot_list_train,shot_list_validate=shot_list_validate,shot_list_test=shot_list_test) + np.savez( + path, + shot_list_train=shot_list_train, + shot_list_validate=shot_list_validate, + shot_list_test=shot_list_test) - -def apply_bleed_in(conf,shot_list_train,shot_list_validate,shot_list_test): +def apply_bleed_in(conf, shot_list_train, shot_list_validate, shot_list_test): np.random.seed(2) num = conf['data']['bleed_in'] - new_shots = [] + # new_shots = [] if num > 0: shot_list_bleed = ShotList() print('applying bleed in with {} disruptive shots\n'.format(num)) - num_total = len(shot_list_test) + # num_total = len(shot_list_test) num_d = shot_list_test.num_disruptive() - num_nd = num_total - num_d - assert(num_d >= num), "Not enough disruptive shots {} to cover bleed in {}".format(num_d,num) + # num_nd = num_total - num_d + assert num_d >= num, ( + "Not enough disruptive shots {} to cover bleed in {}".format( + num_d, num)) num_sampled_d = 0 num_sampled_nd = 0 while num_sampled_d < num: @@ -167,12 +197,15 @@ def apply_bleed_in(conf,shot_list_train,shot_list_validate,shot_list_test): num_sampled_d += 1 else: num_sampled_nd += 1 - print("Sampled {} shots, {} disruptive, {} nondisruptive".format(num_sampled_nd+num_sampled_d,num_sampled_d,num_sampled_nd)) - print("Before adding: training shots: {} validation shots: {}".format(len(shot_list_train),len(shot_list_validate))) + print("Sampled {} shots, {} disruptive, {} nondisruptive".format( + num_sampled_nd+num_sampled_d, num_sampled_d, num_sampled_nd)) + print("Before adding: training shots: {} validation shots: {}".format( + len(shot_list_train), len(shot_list_validate))) assert(num_sampled_d == num) - if conf['data']['bleed_in_equalize_sets']:#add bleed-in shots to training and validation set repeatedly + # add bleed-in shots to training and validation set repeatedly + if conf['data']['bleed_in_equalize_sets']: print("Applying equalized bleed in") - for shot_list_curr in [shot_list_train,shot_list_validate]: + for shot_list_curr in [shot_list_train, shot_list_validate]: for i in range(len(shot_list_curr)): s = shot_list_bleed.sample_shot() shot_list_curr.append(s) @@ -184,12 +217,13 @@ def apply_bleed_in(conf,shot_list_train,shot_list_validate,shot_list_test): s = shot_list_bleed.sample_shot() shot_list_train.append(s) shot_list_validate.append(s) - else: #add each shot only once + else: # add each shot only once print("Applying bleed in without repetition") for s in shot_list_bleed: shot_list_train.append(s) shot_list_validate.append(s) - print("After adding: training shots: {} validation shots: {}".format(len(shot_list_train),len(shot_list_validate))) + print("After adding: training shots: {} validation shots: {}".format( + len(shot_list_train), len(shot_list_validate))) print("Added bleed in shots to training and validation sets") # if num_d > 0: # for i in range(num): @@ -209,34 +243,39 @@ def apply_bleed_in(conf,shot_list_train,shot_list_validate,shot_list_test): # shot_list_test.remove(s) # else: # print('No nondisruptive shots in test set, omitting bleed in') - return shot_list_train,shot_list_validate,shot_list_test - - - + return shot_list_train, shot_list_validate, shot_list_test def guarantee_preprocessed(conf): pp = Preprocessor(conf) if pp.all_are_preprocessed(): print("shots already processed.") - shot_list_train,shot_list_validate,shot_list_test = pp.load_shotlists() + (shot_list_train, shot_list_validate, + shot_list_test) = pp.load_shotlists() else: - print("preprocessing all shots",end='') + print("preprocessing all shots", end='') pp.clean_shot_lists() - shot_list = pp.preprocess_all() - shot_list.sort() - shot_list_train,shot_list_test = shot_list.split_train_test(conf) - num_shots = len(shot_list_train) + len(shot_list_test) + shot_list = sorted(pp.preprocess_all()) + shot_list_train, shot_list_test = shot_list.split_train_test(conf) + # num_shots = len(shot_list_train) + len(shot_list_test) validation_frac = conf['training']['validation_frac'] if validation_frac <= 0.05: print('Setting validation to a minimum of 0.05') validation_frac = 0.05 - shot_list_train,shot_list_validate = shot_list_train.split_direct(1.0-validation_frac,do_shuffle=True) - pp.save_shotlists(shot_list_train,shot_list_validate,shot_list_test) - shot_list_train,shot_list_validate,shot_list_test = apply_bleed_in(conf,shot_list_train,shot_list_validate,shot_list_test) - print('validate: {} shots, {} disruptive'.format(len(shot_list_validate),shot_list_validate.num_disruptive())) - print('training: {} shots, {} disruptive'.format(len(shot_list_train),shot_list_train.num_disruptive())) - print('testing: {} shots, {} disruptive'.format(len(shot_list_test),shot_list_test.num_disruptive())) + shot_list_train, shot_list_validate = shot_list_train.split_direct( + 1.0-validation_frac, do_shuffle=True) + pp.save_shotlists(shot_list_train, shot_list_validate, shot_list_test) + shot_list_train, shot_list_validate, shot_list_test = apply_bleed_in( + conf, shot_list_train, shot_list_validate, shot_list_test) + print( + 'validate: {} shots, {} disruptive'.format( + len(shot_list_validate), + shot_list_validate.num_disruptive())) + print( + 'training: {} shots, {} disruptive'.format( + len(shot_list_train), + shot_list_train.num_disruptive())) + print('testing: {} shots, {} disruptive'.format( + len(shot_list_test), shot_list_test.num_disruptive())) print("...done") - return shot_list_train,shot_list_validate,shot_list_test - + return shot_list_train, shot_list_validate, shot_list_test From dcbb00e91d47c6c06a5811b76e1b0696c66d5611 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 24 Sep 2019 14:38:53 -0500 Subject: [PATCH 090/272] Fix style for files in plasma/primitives --- plasma/primitives/data.py | 374 +++++++++++++++---------- plasma/primitives/hyperparameters.py | 77 +++--- plasma/primitives/ops.py | 6 +- plasma/primitives/shots.py | 396 ++++++++++++++------------- 4 files changed, 494 insertions(+), 359 deletions(-) diff --git a/plasma/primitives/data.py b/plasma/primitives/data.py index fb19f466..93663757 100644 --- a/plasma/primitives/data.py +++ b/plasma/primitives/data.py @@ -1,13 +1,13 @@ from __future__ import division import numpy as np -import time -import sys,os +import sys +import os import re from scipy.interpolate import UnivariateSpline from plasma.utils.processing import get_individual_shot_file -from plasma.utils.downloading import format_save_path,get_missing_value_array +from plasma.utils.downloading import get_missing_value_array # class SignalCollection: # """GA Data Obj""" @@ -16,25 +16,31 @@ # for i in range(len(signal_paths)) # self.signals.append(Signal(signal_descriptions[i],signal_paths[i])) +try: + from MDSplus import Connection +except ImportError: + pass + + class Signal(object): - def __init__(self,description,paths,machines,tex_label=None, - causal_shifts=None,is_ip=False,normalize=True, - data_avail_tolerances=None,is_strictly_positive=False, - mapping_paths=None): + def __init__(self, description, paths, machines, tex_label=None, + causal_shifts=None, is_ip=False, normalize=True, + data_avail_tolerances=None, is_strictly_positive=False, + mapping_paths=None): assert(len(paths) == len(machines)) self.description = description self.paths = paths - self.machines = machines #on which machines is the signal defined - if causal_shifts == None: + self.machines = machines # on which machines is the signal defined + if causal_shifts is None: causal_shifts = [0 for m in machines] - self.causal_shifts = causal_shifts #causal shift in ms + self.causal_shifts = causal_shifts # causal shift in ms self.is_ip = is_ip self.num_channels = 1 self.normalize = normalize - if data_avail_tolerances == None: + if data_avail_tolerances is None: data_avail_tolerances = [0 for m in machines] self.data_avail_tolerances = data_avail_tolerances - self.is_strictly_positive=is_strictly_positive + self.is_strictly_positive = is_strictly_positive self.mapping_paths = mapping_paths def is_strictly_positive_fn(self): @@ -43,231 +49,317 @@ def is_strictly_positive_fn(self): def is_ip(self): return self.is_ip - def get_file_path(self,prepath,machine,shot_number): + def get_file_path(self, prepath, machine, shot_number): dirname = self.get_path(machine) - return get_individual_shot_file(prepath + '/' + machine.name + '/' +dirname + '/',shot_number) + return get_individual_shot_file(prepath + '/' + machine.name + '/' + + dirname + '/', shot_number) - def is_valid(self,prepath,shot,dtype='float32'): - t,data,exists = self.load_data(prepath,shot,dtype) - return exists + def is_valid(self, prepath, shot, dtype='float32'): + t, data, exists = self.load_data(prepath, shot, dtype) + return exists - def is_saved(self,prepath,shot): - file_path = self.get_file_path(prepath,shot.machine,shot.number) + def is_saved(self, prepath, shot): + file_path = self.get_file_path(prepath, shot.machine, shot.number) return os.path.isfile(file_path) - def load_data_from_txt_safe(self,prepath,shot,dtype='float32'): - file_path = self.get_file_path(prepath,shot.machine,shot.number) - if not self.is_saved(prepath,shot): - print('Signal {}, shot {} was never downloaded'.format(self.description,shot.number)) - return None,False + def load_data_from_txt_safe(self, prepath, shot, dtype='float32'): + file_path = self.get_file_path(prepath, shot.machine, shot.number) + if not self.is_saved(prepath, shot): + print( + 'Signal {}, shot {} was never downloaded'.format( + self.description, + shot.number)) + return None, False if os.path.getsize(file_path) == 0: - print('Signal {}, shot {} was downloaded incorrectly (empty file). Removing.'.format(self.description,shot.number)) + print('Signal {}, shot {} '.format(self.description, shot.number), + 'was downloaded incorrectly (empty file). Removing.') os.remove(file_path) - return None,False + return None, False try: - data = np.loadtxt(file_path,dtype=dtype) + data = np.loadtxt(file_path, dtype=dtype) if np.all(data == get_missing_value_array()): - print('Signal {}, shot {} contains no data'.format(self.description,shot.number)) - return None,False + print( + 'Signal {}, shot {} contains no data'.format( + self.description, shot.number)) + return None, False except Exception as e: print(e) - print('Couldnt load signal {} shot {}. Removing.'.format(file_path,shot.number)) + print( + 'Couldnt load signal {} shot {}. Removing.'.format( + file_path, shot.number)) os.remove(file_path) return None, False - return data,True + return data, True - def load_data(self,prepath,shot,dtype='float32'): - data,succ = self.load_data_from_txt_safe(prepath,shot) + def load_data(self, prepath, shot, dtype='float32'): + data, succ = self.load_data_from_txt_safe(prepath, shot) if not succ: - return None,None,False - + return None, None, False + if np.ndim(data) == 1: - data = np.expand_dims(data,axis=0) + data = np.expand_dims(data, axis=0) - t = data[:,0] - sig = data[:,1:] + t = data[:, 0] + sig = data[:, 1:] - if self.is_ip: #restrict shot to current threshold + if self.is_ip: # restrict shot to current threshold region = np.where(np.abs(sig) >= shot.machine.current_threshold)[0] if len(region) == 0: print('shot {} has no current'.format(shot.number)) - return None,None,False + return None, None, False first_idx = region[0] last_idx = region[-1] - last_time = t[last_idx]+5e-2 #add 50 ms to cover possible disruption event + # add 50 ms to cover possible disruption event + last_time = t[last_idx]+5e-2 last_indices = np.where(t > last_time)[0] if len(last_indices) == 0: last_idx = -1 else: last_idx = last_indices[0] t = t[first_idx:last_idx] - sig = sig[first_idx:last_idx,:] + sig = sig[first_idx:last_idx, :] - #make sure shot is not garbage data + # make sure shot is not garbage data if len(t) <= 1 or (np.max(sig) == 0.0 and np.min(sig) == 0.0): if self.is_ip: print('shot {} has no current'.format(shot.number)) else: - print('Signal {}, shot {} contains no data'.format(self.description,shot.number)) - return None,None,False - - #make sure data doesn't contain nan - if np.any(np.isnan(t)) or np.any(np.isnan(sig)): - print('Signal {}, shot {} contains NAN'.format(self.description,shot.number)) - return None,None,False + print( + 'Signal {}, shot {} contains no data'.format( + self.description, shot.number)) + return None, None, False - return t,sig,True + # make sure data doesn't contain nan + if np.any(np.isnan(t)) or np.any(np.isnan(sig)): + print( + 'Signal {}, shot {} contains NAN'.format( + self.description, + shot.number)) + return None, None, False + return t, sig, True - def fetch_data_basic(self,machine,shot_num,c,path=None): + def fetch_data_basic(self, machine, shot_num, c, path=None): if path is None: path = self.get_path(machine) success = False mapping = None try: - time,data,mapping,success = machine.fetch_data_fn(path,shot_num,c) + time, data, mapping, success = machine.fetch_data_fn( + path, shot_num, c) except Exception as e: print(e) sys.stdout.flush() if not success: - return None,None,None,False + return None, None, None, False time = np.array(time) + 1e-3*self.get_causal_shift(machine) - return time,np.array(data),mapping,success + return time, np.array(data), mapping, success - def fetch_data(self,machine,shot_num,c): - return self.fetch_data_basic(machine,shot_num,c) + def fetch_data(self, machine, shot_num, c): + return self.fetch_data_basic(machine, shot_num, c) - def is_defined_on_machine(self,machine): + def is_defined_on_machine(self, machine): return machine in self.machines - def is_defined_on_machines(self,machines): + def is_defined_on_machines(self, machines): return all([m in self.machines for m in machines]) - def get_path(self,machine): + def get_path(self, machine): idx = self.get_idx(machine) return self.paths[idx] - def get_mapping_path(self,machine): + def get_mapping_path(self, machine): if self.mapping_paths is None: return None else: idx = self.get_idx(machine) - return self.mapping_paths[idx] + return self.mapping_paths[idx] - def get_causal_shift(self,machine): + def get_causal_shift(self, machine): idx = self.get_idx(machine) return self.causal_shifts[idx] - def get_data_avail_tolerance(self,machine): + def get_data_avail_tolerance(self, machine): idx = self.get_idx(machine) return self.data_avail_tolerances[idx] - def get_idx(self,machine): + def get_idx(self, machine): assert(machine in self.machines) - idx = self.machines.index(machine) + idx = self.machines.index(machine) return idx - def __eq__(self,other): + def __eq__(self, other): if other is None: return False return self.description.__eq__(other.description) - - def __ne__(self,other): + def __ne__(self, other): return self.description.__ne__(other.description) - def __lt__(self,other): + def __lt__(self, other): return self.description.__lt__(other.description) - + def __hash__(self): - import hashlib - return int(hashlib.md5(self.description.encode('utf-8')).hexdigest(),16) + import hashlib + return int( + hashlib.md5( + self.description.encode('utf-8')).hexdigest(), + 16) def __str__(self): return self.description - + def __repr__(self): return self.description + class ProfileSignal(Signal): - def __init__(self,description,paths,machines,tex_label=None,causal_shifts=None,mapping_range=(0,1),num_channels=32,data_avail_tolerances=None,is_strictly_positive=False,mapping_paths=None): - super(ProfileSignal, self).__init__(description,paths,machines,tex_label,causal_shifts,is_ip=False,data_avail_tolerances=data_avail_tolerances,is_strictly_positive=is_strictly_positive,mapping_paths=mapping_paths) + def __init__( + self, + description, + paths, + machines, + tex_label=None, + causal_shifts=None, + mapping_range=( + 0, + 1), + num_channels=32, + data_avail_tolerances=None, + is_strictly_positive=False, + mapping_paths=None): + super( + ProfileSignal, + self).__init__( + description, + paths, + machines, + tex_label, + causal_shifts, + is_ip=False, + data_avail_tolerances=data_avail_tolerances, + is_strictly_positive=is_strictly_positive, + mapping_paths=mapping_paths) self.mapping_range = mapping_range self.num_channels = num_channels - def load_data(self,prepath,shot,dtype='float32'): - data,succ = self.load_data_from_txt_safe(prepath,shot) + def load_data(self, prepath, shot, dtype='float32'): + data, succ = self.load_data_from_txt_safe(prepath, shot) if not succ: - return None,None,False + return None, None, False if np.ndim(data) == 1: - data = np.expand_dims(data,axis=0) - #_ = data[0,0] - T = data.shape[0]//2 #time is stored twice, once for mapping and once for signal - mapping = data[:T,1:] - remapping = np.linspace(self.mapping_range[0],self.mapping_range[1],self.num_channels) - t = data[:T,0] - sig = data[T:,1:] + data = np.expand_dims(data, axis=0) + # time is stored twice, once for mapping and once for signal + T = data.shape[0]//2 + mapping = data[:T, 1:] + remapping = np.linspace( + self.mapping_range[0], + self.mapping_range[1], + self.num_channels) + t = data[:T, 0] + sig = data[T:, 1:] if sig.shape[1] < 2: - print('Signal {}, shot {} should be profile but has only one channel. Possibly only one profile fit was run for the duration of the shot and was transposed during downloading. Need at least 2.'.format(self.description,shot.number)) - return None,None,False + print('Signal {}, shot {} '.format(self.description, shot.number), + 'should be profile but has only one channel. Possibly only ', + 'one profile fit was run for the duration of the shot and ', + 'was transposed during downloading. Need at least 2.') + return None, None, False if len(t) <= 1 or (np.max(sig) == 0.0 and np.min(sig) == 0.0): - print('Signal {}, shot {} contains no data'.format(self.description,shot.number)) - return None,None,False + print('Signal {}, shot {} '.format(self.description, shot.number), + 'contains no data.') + return None, None, False if np.any(np.isnan(t)) or np.any(np.isnan(sig)): - print('Signal {}, shot {} contains NAN'.format(self.description,shot.number)) - return None,None,False + print('Signal {}, shot {} '.format(self.description, shot.number), + 'contains NaN value(s).') + return None, None, False timesteps = len(t) - sig_interp = np.zeros((timesteps,self.num_channels)) + sig_interp = np.zeros((timesteps, self.num_channels)) for i in range(timesteps): - _,order = np.unique(mapping[i,:],return_index=True) #make sure the mapping is ordered and unique - if sig[i,order].shape[0] > 2: - f = UnivariateSpline(mapping[i,order],sig[i,order],s=0,k=1,ext=3) #ext = 0 is extrapolation, ext = 3 is boundary value. - sig_interp[i,:] = f(remapping) + # make sure the mapping is ordered and unique + _, order = np.unique(mapping[i, :], return_index=True) + if sig[i, order].shape[0] > 2: + # ext = 0 is extrapolation, ext = 3 is boundary value. + f = UnivariateSpline( + mapping[i, order], sig[i, order], s=0, k=1, ext=3) + sig_interp[i, :] = f(remapping) else: - print('Signal {}, shot {} has not enough points for linear interpolation. dfitpack.error: (m>k) failed for hidden m: fpcurf0:m=1'.format(self.description,shot.number)) - return None,None,False + print('Signal {}, shot {} '.format(self.description, + shot.number), + 'has insufficient points for linear interpolation. ', + 'dfitpack.error: (m>k) failed for hidden m: fpcurf0:m=1') + return None, None, False - return t,sig_interp,True + return t, sig_interp, True - def fetch_data(self,machine,shot_num,c): - time,data,mapping,success = self.fetch_data_basic(machine,shot_num,c) + def fetch_data(self, machine, shot_num, c): + time, data, mapping, success = self.fetch_data_basic( + machine, shot_num, c) path = self.get_path(machine) mapping_path = self.get_mapping_path(machine) - if mapping is not None and np.ndim(mapping) == 1:#make sure there is a mapping for every timestep + if mapping is not None and np.ndim(mapping) == 1: + # make sure there is a mapping for every timestep T = len(time) - mapping = np.tile(mapping,(T,1)).transpose() - assert(mapping.shape == data.shape), "shape of mapping and data is different" - if mapping_path is not None:#fetch the mapping separately - time_map,data_map,mapping_map,success_map = self.fetch_data_basic(machine,shot_num,c,path=mapping_path) + mapping = np.tile(mapping, (T, 1)).transpose() + assert mapping.shape == data.shape, ( + "mapping and data shapes are different") + if mapping_path is not None: + # fetch the mapping separately + (time_map, data_map, mapping_map, + success_map) = self.fetch_data_basic(machine, shot_num, c, + path=mapping_path) success = (success and success_map) if not success: - print("No success for signal {} and mapping {}".format(path,mapping_path)) + print("No success for signal {} and mapping {}".format( + path, mapping_path)) else: - assert(np.all(time == time_map)), "time for signal {} and mapping {} don't align: \n{}\n\n{}\n".format(path,mapping_path,time,time_map) + assert np.all(time == time_map), ( + "time for signal {} and mapping {} ".format(path, + mapping_path) + + "don't align: \n{}\n\n{}\n".format(time, time_map)) mapping = data_map if not success: - return None,None,None,False - return time,data,mapping,success + return None, None, None, False + return time, data, mapping, success class ChannelSignal(Signal): - def __init__(self,description,paths,machines,tex_label=None,causal_shifts=None,data_avail_tolerances=None,is_strictly_positive=False,mapping_paths=None): - super(ChannelSignal, self).__init__(description,paths,machines,tex_label,causal_shifts,is_ip=False,data_avail_tolerances=data_avail_tolerances,is_strictly_positive=is_strictly_positive,mapping_paths=mapping_paths) - nums,new_paths = self.get_channel_nums(paths) + def __init__( + self, + description, + paths, + machines, + tex_label=None, + causal_shifts=None, + data_avail_tolerances=None, + is_strictly_positive=False, + mapping_paths=None): + super( + ChannelSignal, + self).__init__( + description, + paths, + machines, + tex_label, + causal_shifts, + is_ip=False, + data_avail_tolerances=data_avail_tolerances, + is_strictly_positive=is_strictly_positive, + mapping_paths=mapping_paths) + nums, new_paths = self.get_channel_nums(paths) self.channel_nums = nums self.paths = new_paths - def get_channel_nums(self,paths): - regex = re.compile('channel\d+') - regex_int = re.compile('\d+') + def get_channel_nums(self, paths): + regex = re.compile(r'channel\d+') + regex_int = re.compile(r'\d+') nums = [] new_paths = [] for p in paths: @@ -281,35 +373,43 @@ def get_channel_nums(self,paths): else: nums.append(int(regex_int.findall(res[0])[0])) new_paths.append("/".join(elements[:-1])) - return nums,new_paths + return nums, new_paths - def get_channel_num(self,machine): + def get_channel_num(self, machine): idx = self.get_idx(machine) return self.channel_nums[idx] - def fetch_data(self,machine,shot_num,c): - time,data,mapping,success = self.fetch_data_basic(machine,shot_num,c) - mapping = None #we are not interested in the whole profile + def fetch_data(self, machine, shot_num, c): + time, data, mapping, success = self.fetch_data_basic( + machine, shot_num, c) + mapping = None # we are not interested in the whole profile channel_num = self.get_channel_num(machine) if channel_num is not None and success: if np.ndim(data) != 2: - print("Channel Signal {} expected 2D array for shot {}".format(self,shot)) + print("Channel Signal {} expected 2D array for shot {}".format( + self, self.shot_number)) success = False else: - data = data[channel_num,:] #extract channel of interest - return time,data,mapping,success + data = data[channel_num, :] # extract channel of interest + return time, data, mapping, success - def get_file_path(self,prepath,machine,shot_number): + def get_file_path(self, prepath, machine, shot_number): dirname = self.get_path(machine) num = self.get_channel_num(machine) if num is not None: dirname += "/channel{}".format(num) - return get_individual_shot_file(prepath + '/' + machine.name + '/' +dirname + '/',shot_number) - + return get_individual_shot_file(prepath + '/' + machine.name + '/' + + dirname + '/', shot_number) class Machine(object): - def __init__(self,name,server,fetch_data_fn,max_cores = 8,current_threshold=0): + def __init__( + self, + name, + server, + fetch_data_fn, + max_cores=8, + current_threshold=0): self.name = name self.server = server self.max_cores = max_cores @@ -317,24 +417,22 @@ def __init__(self,name,server,fetch_data_fn,max_cores = 8,current_threshold=0): self.current_threshold = current_threshold def get_connection(self): - return Connection(server) + return Connection(self.server) - def __eq__(self,other): + def __eq__(self, other): return self.name.__eq__(other.name) - def __lt__(self,other): + def __lt__(self, other): return self.name.__lt__(other.name) - - def __ne__(self,other): + + def __ne__(self, other): return self.name.__ne__(other.name) - + def __hash__(self): return self.name.__hash__() - + def __str__(self): return self.name def __repr__(self): return self.__str__() - - diff --git a/plasma/primitives/hyperparameters.py b/plasma/primitives/hyperparameters.py index d3e29e35..9bed2a78 100644 --- a/plasma/primitives/hyperparameters.py +++ b/plasma/primitives/hyperparameters.py @@ -1,7 +1,9 @@ import numpy as np import random import abc -import yaml,os +import yaml +import os + class Hyperparam(object): @@ -9,21 +11,22 @@ class Hyperparam(object): def choice(self): return 0 - def get_conf_entry(self,conf): + def get_conf_entry(self, conf): el = conf for sub_path in self.path: el = el[sub_path] return el - def assign_to_conf(self,conf,save_path): + def assign_to_conf(self, conf, save_path): val = self.choice() - print(" : ".join(self.path)+ ": {}".format(val)) + print(" : ".join(self.path) + ": {}".format(val)) el = conf for sub_path in self.path[:-1]: el = el[sub_path] el[self.path[-1]] = val - with open(os.path.join(save_path,"changed_params.out"), 'a+') as outfile: + with open(os.path.join(save_path, "changed_params.out"), + 'a+') as outfile: for el in self.path: outfile.write("{} : ".format(el)) outfile.write("{}\n".format(val)) @@ -31,56 +34,59 @@ def assign_to_conf(self,conf,save_path): class CategoricalHyperparam(Hyperparam): - def __init__(self,path,values): + def __init__(self, path, values): self.path = path self.values = values def choice(self): return random.choice(self.values) + class GridCategoricalHyperparam(Hyperparam): - def __init__(self,path,values): + def __init__(self, path, values): self.path = path self.values = iter(values) def choice(self): return next(self.values) + class ContinuousHyperparam(Hyperparam): - def __init__(self,path,lo,hi): + def __init__(self, path, lo, hi): self.path = path - self.lo =lo - self.hi =hi + self.lo = lo + self.hi = hi def choice(self): - return float(np.random.uniform(self.lo,self.hi)) + return float(np.random.uniform(self.lo, self.hi)) + class LogContinuousHyperparam(Hyperparam): - def __init__(self,path,lo,hi): + def __init__(self, path, lo, hi): self.path = path self.lo = self.to_log(lo) - self.hi = self.to_log(hi) + self.hi = self.to_log(hi) - def to_log(self,num_val): + def to_log(self, num_val): return np.log10(num_val) def choice(self): - return float(np.power(10,np.random.uniform(self.lo,self.hi))) + return float(np.power(10, np.random.uniform(self.lo, self.hi))) class IntegerHyperparam(Hyperparam): - def __init__(self,path,lo,hi): + def __init__(self, path, lo, hi): self.path = path - self.lo =lo - self.hi =hi + self.lo = lo + self.hi = hi def choice(self): - return int(np.random.random_integers(self.lo,self.hi)) + return int(np.random.random_integers(self.lo, self.hi)) class GenericHyperparam(Hyperparam): - def __init__(self,path,choice_fn): + def __init__(self, path, choice_fn): self.path = path self.choice_fn = choice_fn @@ -89,24 +95,24 @@ def choice(self): class HyperparamExperiment(object): - def __init__(self,path,conf_name = "conf.yaml"): + def __init__(self, path, conf_name="conf.yaml"): if not path.endswith('/'): path += '/' self.path = path self.finished = False self.success = False - self.logs_path = os.path.join(path,"csv_logs/") + self.logs_path = os.path.join(path, "csv_logs/") self.raw_logs_path = path[:-1] + ".out" - self.changed_path = os.path.join(path,"changed_params.out") - with open(os.path.join(self.path,conf_name), 'r') as yaml_file: - conf = yaml.load(yaml_file) + self.changed_path = os.path.join(path, "changed_params.out") + with open(os.path.join(self.path, conf_name), 'r') as yaml_file: + conf = yaml.load(yaml_file) self.name_to_monitor = conf['callbacks']['monitor'] self.load_data() self.get_changed() self.get_maximum() self.read_raw_logs() - def __lt__(self,other): + def __lt__(self, other): return self.path.__lt__(other.path) def get_number(self): @@ -125,8 +131,8 @@ def __str__(self): def summary(self): s = "Finished" if self.finished else "Running" - print("# {} [{}] maximum of {} at epoch {}".format(self.get_number(),s,*self.get_maximum(False))) - + print("# {} [{}] maximum of {} at epoch {}".format( + self.get_number(), s, *self.get_maximum(False))) def load_data(self): import pandas @@ -164,16 +170,17 @@ def read_raw_logs(self): if lines[-1].strip() == 'done.': self.finished = True if lines[-2].strip() == 'finished.': - self.success = True + self.success = True print('finished: {}, success: {}'.format(self.finished, self.success)) - def get_maximum(self,verbose=True): + def get_maximum(self, verbose=True): if len(self.epochs) > 0: - idx = np.argmax(self.values) + idx = np.argmax(self.values) s = "Finished" if self.finished else "Running" if verbose: - #print(self.path) - print("[{}] maximum of {} at epoch {}".format(s,self.values[idx],self.epochs[idx])) - return self.values[idx],self.epochs[idx] + # print(self.path) + print("[{}] maximum of {} at epoch {}".format( + s, self.values[idx], self.epochs[idx])) + return self.values[idx], self.epochs[idx] else: - return -1,-1 + return -1, -1 diff --git a/plasma/primitives/ops.py b/plasma/primitives/ops.py index 1152ad46..19e7ed8f 100644 --- a/plasma/primitives/ops.py +++ b/plasma/primitives/ops.py @@ -2,15 +2,17 @@ from mpi4py import MPI comm = MPI.COMM_WORLD -#define a float16 mpi datatype +# define a float16 mpi datatype mpi_float16 = MPI.BYTE.Create_contiguous(2).Commit() MPI._typedict['e'] = mpi_float16 + def sum_f16_cb(buffer_a, buffer_b, t): assert t == mpi_float16 array_a = np.frombuffer(buffer_a, dtype='float16') array_b = np.frombuffer(buffer_b, dtype='float16') array_b += array_a -#create new OP + +# create new OP mpi_sum_f16 = MPI.Op.Create(sum_f16_cb, commute=True) diff --git a/plasma/primitives/shots.py b/plasma/primitives/shots.py index e2865d72..3b96e9fc 100644 --- a/plasma/primitives/shots.py +++ b/plasma/primitives/shots.py @@ -16,11 +16,12 @@ import numpy as np -from plasma.utils.processing import train_test_split,cut_and_resample_signal +from plasma.utils.processing import train_test_split, cut_and_resample_signal from plasma.utils.downloading import makedirs_process_safe + class ShotListFiles(object): - def __init__(self,machine,prepath,paths,description=''): + def __init__(self, machine, prepath, paths, description=''): self.machine = machine self.prepath = prepath self.paths = paths @@ -34,121 +35,105 @@ def __str__(self): def __repr__(self): return self.__str__() - def get_single_shot_numbers_and_disruption_times(self,full_path): - data = np.loadtxt(full_path,ndmin=1,dtype={'names':('num','disrupt_times'), - 'formats':('i4','f4')}) + def get_single_shot_numbers_and_disruption_times(self, full_path): + data = np.loadtxt( + full_path, ndmin=1, dtype={ + 'names': ( + 'num', 'disrupt_times'), 'formats': ( + 'i4', 'f4')}) shots = np.array(list(zip(*data))[0]) disrupt_times = np.array(list(zip(*data))[1]) return shots, disrupt_times - def get_shot_numbers_and_disruption_times(self): all_shots = [] all_disruption_times = [] - all_machines_arr = [] + # all_machines_arr = [] for path in self.paths: full_path = self.prepath + path - shots,disruption_times = self.get_single_shot_numbers_and_disruption_times(full_path) + shots, disruption_times = ( + self.get_single_shot_numbers_and_disruption_times(full_path)) all_shots.append(shots) all_disruption_times.append(disruption_times) - return np.concatenate(all_shots),np.concatenate(all_disruption_times) - + return np.concatenate(all_shots), np.concatenate(all_disruption_times) class ShotList(object): ''' - A wrapper class around list of Shot objects, providing utilities to + A wrapper class around list of Shot objects, providing utilities to extract, load and transform Shots before passing them to an estimator. During distributed training, shot lists are split into sublists. - A sublist is a ShotList object having num_at_once shots. The ShotList contains an entire dataset - as specified in the configuration file. + A sublist is a ShotList object having num_at_once shots. The ShotList + contains an entire dataset as specified in the configuration file. ''' - def __init__(self,shots=None): + def __init__(self, shots=None): ''' A ShotList is a list of 2D Numpy arrays. ''' self.shots = [] if shots is not None: - assert(all([isinstance(shot,Shot) for shot in shots])) + assert(all([isinstance(shot, Shot) for shot in shots])) self.shots = [shot for shot in shots] - def load_from_shot_list_files_object(self,shot_list_files_object,signals): + def load_from_shot_list_files_object( + self, shot_list_files_object, signals): machine = shot_list_files_object.machine - shot_numbers,disruption_times = shot_list_files_object.get_shot_numbers_and_disruption_times() - for number,t in list(zip(shot_numbers,disruption_times)): - self.append(Shot(number=number,t_disrupt=t,machine=machine,signals=[s for s in signals if s.is_defined_on_machine(machine)])) - - - - def load_from_shot_list_files_objects(self,shot_list_files_objects,signals): + shot_numbers, disruption_times = ( + shot_list_files_object.get_shot_numbers_and_disruption_times()) + for number, t in list(zip(shot_numbers, disruption_times)): + self.append( + Shot(number=number, t_disrupt=t, machine=machine, + signals=[s for s in signals if + s.is_defined_on_machine(machine)] + ) + ) + + def load_from_shot_list_files_objects( + self, shot_list_files_objects, signals): for obj in shot_list_files_objects: - self.load_from_shot_list_files_object(obj,signals) - - # ######Generic Methods#### - - # @staticmethod - # def get_shots_and_disruption_times(shots_and_disruption_times_path,machine): - # data = np.loadtxt(shots_and_disruption_times_path,ndmin=1,dtype={'names':('num','disrupt_times'), - # 'formats':('i4','f4')}) - # shots = np.array(list(zip(*data))[0]) - # disrupt_times = np.array(list(zip(*data))[1]) - # machines = np.array([machine]*len(shots)) - # return shots, disrupt_times, machines - - # @staticmethod - # def get_multiple_shots_and_disruption_times(base_path,endings,machines): - # all_shots = [] - # all_disruption_times = [] - # all_machines_arr = [] - # for (ending,machine) in zip(endings,machines): - # path = base_path + ending - # shots,disruption_times,machines_arr = ShotList.get_shots_and_disruption_times(path,machine) - # all_shots.append(shots) - # all_disruption_times.append(disruption_times) - # all_machines_arr.append(machines_arr) - # return np.concatenate(all_shots),np.concatenate(all_disruption_times),np.concatenate(all_machines_arr) - - - def split_train_test(self,conf): - shot_list_dir = conf['paths']['shot_list_dir'] + self.load_from_shot_list_files_object(obj, signals) + + def split_train_test(self, conf): + # shot_list_dir = conf['paths']['shot_list_dir'] shot_files = conf['paths']['shot_files'] shot_files_test = conf['paths']['shot_files_test'] train_frac = conf['training']['train_frac'] shuffle_training = conf['training']['shuffle_training'] use_shots = conf['data']['use_shots'] all_signals = conf['paths']['all_signals'] - #split randomly + # split randomly use_shots_train = int(round(train_frac*use_shots)) use_shots_test = int(round((1-train_frac)*use_shots)) if len(shot_files_test) == 0: - shot_list_train,shot_list_test = train_test_split(self.shots,train_frac,shuffle_training) - #train and test list given + shot_list_train, shot_list_test = train_test_split( + self.shots, train_frac, shuffle_training) + # train and test list given else: shot_list_train = ShotList() - shot_list_train.load_from_shot_list_files_objects(shot_files,all_signals) - + shot_list_train.load_from_shot_list_files_objects( + shot_files, all_signals) + shot_list_test = ShotList() - shot_list_test.load_from_shot_list_files_objects(shot_files_test,all_signals) - - + shot_list_test.load_from_shot_list_files_objects( + shot_files_test, all_signals) + shot_numbers_train = [shot.number for shot in shot_list_train] shot_numbers_test = [shot.number for shot in shot_list_test] - print(len(shot_numbers_train),len(shot_numbers_test)) - #make sure we only use pre-filtered valid shots + print(len(shot_numbers_train), len(shot_numbers_test)) + # make sure we only use pre-filtered valid shots shots_train = self.filter_by_number(shot_numbers_train) shots_test = self.filter_by_number(shot_numbers_test) - return shots_train.random_sublist(use_shots_train),shots_test.random_sublist(use_shots_test) - + return shots_train.random_sublist( + use_shots_train), shots_test.random_sublist(use_shots_test) - def split_direct(self,frac,do_shuffle=True): - shot_list_one,shot_list_two = train_test_split(self.shots,frac,do_shuffle) - return ShotList(shot_list_one),ShotList(shot_list_two) + def split_direct(self, frac, do_shuffle=True): + shot_list_one, shot_list_two = train_test_split( + self.shots, frac, do_shuffle) + return ShotList(shot_list_one), ShotList(shot_list_two) - - - def filter_by_number(self,numbers): + def filter_by_number(self, numbers): new_shot_list = ShotList() numbers = set(numbers) for shot in self.shots: @@ -156,14 +141,14 @@ def filter_by_number(self,numbers): new_shot_list.append(shot) return new_shot_list - def set_weights(self,weights): + def set_weights(self, weights): assert(len(weights) == len(self.shots)) - for (i,w) in enumerate(weights): + for (i, w) in enumerate(weights): self.shots[i].weight = w - def sample_weighted_given_arr(self,p): + def sample_weighted_given_arr(self, p): p = p/np.sum(p) - idx = np.random.choice(range(len(self.shots)),p=p) + idx = np.random.choice(range(len(self.shots)), p=p) return self.shots[idx] def sample_shot(self): @@ -174,19 +159,20 @@ def sample_weighted(self): p = np.array([shot.weight for shot in self.shots]) return self.sample_weighted_given_arr(p) - def sample_single_class(self,disruptive): + def sample_single_class(self, disruptive): weights_d = 0.0 weights_nd = 1.0 if disruptive: weights_d = 1.0 weights_nd = 0.0 - p = np.array([weights_d if shot.is_disruptive_shot() else weights_nd for shot in self.shots ]) + p = np.array([weights_d if shot.is_disruptive_shot() + else weights_nd for shot in self.shots]) return self.sample_weighted_given_arr(p) - def sample_equal_classes(self): - weights_d,weights_nd = self.get_weights_d_nd() - p = np.array([weights_d if shot.is_disruptive_shot() else weights_nd for shot in self.shots ]) + weights_d, weights_nd = self.get_weights_d_nd() + p = np.array([weights_d if shot.is_disruptive_shot() + else weights_nd for shot in self.shots]) return self.sample_weighted_given_arr(p) def get_weights_d_nd(self): @@ -199,22 +185,22 @@ def get_weights_d_nd(self): else: weights_d = 1.0*num_nd weights_nd = 1.0*num_d - max_weight = np.maximum(weights_d,weights_nd) - return weights_d/max_weight,weights_nd/max_weight + max_weight = np.maximum(weights_d, weights_nd) + return weights_d/max_weight, weights_nd/max_weight - def num_timesteps(self,prepath): + def num_timesteps(self, prepath): ls = [shot.num_timesteps(prepath) for shot in self.shots] timesteps_total = sum(ls) - timesteps_d = sum([ts for (i,ts) in enumerate(ls) if self.shots[i].is_disruptive_shot()]) + timesteps_d = sum([ts for (i, ts) in enumerate( + ls) if self.shots[i].is_disruptive_shot()]) timesteps_nd = timesteps_total-timesteps_d - return timesteps_total,timesteps_d,timesteps_nd - + return timesteps_total, timesteps_d, timesteps_nd def num_disruptive(self): return len([shot for shot in self.shots if shot.is_disruptive_shot()]) def __len__(self): - return len(self.shots) + return len(self.shots) def __str__(self): return str([s.number for s in self.shots]) @@ -225,47 +211,45 @@ def __iter__(self): def next(self): return self.__iter__().next() - def __add__(self,other_list): + def __add__(self, other_list): return ShotList(self.shots + other_list.shots) - def index(self,item): + def index(self, item): return self.shots.index(item) - def __getitem__(self,key): + def __getitem__(self, key): return self.shots[key] - def random_sublist(self,num): - num = min(num,len(self)) - shots_picked = np.random.choice(self.shots,size=num,replace=False) + def random_sublist(self, num): + num = min(num, len(self)) + shots_picked = np.random.choice(self.shots, size=num, replace=False) return ShotList(shots_picked) - def sublists(self,num,do_shuffle=True,equal_size=False): + def sublists(self, num, do_shuffle=True, equal_size=False): lists = [] if do_shuffle: self.shuffle() - for i in range(0,len(self),num): + for i in range(0, len(self), num): subl = self.shots[i:i+num] while equal_size and len(subl) < num: subl.append(rnd.choice(self.shots)) lists.append(subl) return [ShotList(l) for l in lists] - - def shuffle(self): np.random.shuffle(self.shots) def sort(self): - self.shots.sort() #will sort based on machine and number + self.shots.sort() # will sort based on machine and number def as_list(self): return self.shots - def append(self,shot): - assert(isinstance(shot,Shot)) + def append(self, shot): + assert(isinstance(shot, Shot)) self.shots.append(shot) - def remove(self,shot): + def remove(self, shot): assert(shot in self.shots) self.shots.remove(shot) assert(shot not in self.shots) @@ -274,75 +258,93 @@ def make_light(self): for shot in self.shots: shot.make_light() - def append_if_valid(self,shot): + def append_if_valid(self, shot): if shot.valid: self.append(shot) return True else: - #print('Warning: shot {} not valid, omitting'.format(shot.number)) + # print('Warning: shot {} not valid, omitting'.format(shot.number)) return False - class Shot(object): ''' A class representing a shot. - Each shot is a measurement of plasma properties (current, locked mode amplitude, etc.) as a function of time. + Each shot is a measurement of plasma properties (current, locked mode + amplitude, etc.) as a function of time. - For 0D data, each shot is modeled as a 2D Numpy array - time vs a plasma property. + For 0D data, each shot is modeled as a 2D Numpy array - time vs a plasma + property. ''' - def __init__(self,number=None,machine=None,signals=None,signals_dict=None,ttd=None,valid=None,is_disruptive=None,t_disrupt=None): + def __init__( + self, + number=None, + machine=None, + signals=None, + signals_dict=None, + ttd=None, + valid=None, + is_disruptive=None, + t_disrupt=None): ''' Shot objects contain following attributes: - + - number: integer, unique identifier of a shot - - t_disrupt: double, disruption time in milliseconds (second column in the shotlist input file) - - ttd: Numpy array of doubles, time profile of the shot converted to time-to-disruption values - - valid: boolean flag indicating whether plasma property (specifically, current) reaches a certain value during the shot + - t_disrupt: double, disruption time in milliseconds (second column in + the shotlist input file) + + - ttd: Numpy array of doubles, time profile of the shot converted to + time-to-disruption values + - valid: boolean flag indicating whether plasma property + (specifically, current) reaches a certain value during the shot - is_disruptive: boolean flag indicating whether a shot is disruptive ''' - self.number = number #Shot number - self.machine = machine #machine on which it is defined - self.signals = signals - self.signals_dict = signals_dict # - self.ttd = ttd - self.valid =valid + self.number = number # Shot number + self.machine = machine # machine on which it is defined + self.signals = signals + self.signals_dict = signals_dict + self.ttd = ttd + self.valid = valid self.is_disruptive = is_disruptive self.t_disrupt = t_disrupt self.weight = 1.0 self.augmentation_fn = None if t_disrupt is not None: - self.is_disruptive = Shot.is_disruptive_given_disruption_time(t_disrupt) + self.is_disruptive = Shot.is_disruptive_given_disruption_time( + t_disrupt) else: - print('Warning, disruption time (disruptivity) not set! Either set t_disrupt or is_disruptive') + print('Warning, disruption time (disruptivity) not set! ', + 'Either set t_disrupt or is_disruptive') def get_id_str(self): - return '{} : {}'.format(self.machine,self.number) + return '{} : {}'.format(self.machine, self.number) - def __lt__(self,other): + def __lt__(self, other): return self.get_id_str().__lt__(other.get_id_str()) - def __eq__(self,other): + def __eq__(self, other): return self.get_id_str().__eq__(other.get_id_str()) def __hash__(self): - import hashlib - return int(hashlib.md5(self.get_id_str().encode('utf-8')).hexdigest(),16) + import hashlib + return int( + hashlib.md5( + self.get_id_str().encode('utf-8')).hexdigest(), + 16) def __str__(self): string = 'number: {}\n'.format(self.number) string += 'machine: {}\n'.format(self.machine) - string += 'signals: {}\n'.format(self.signals ) - string += 'signals_dict: {}\n'.format(self.signals_dict ) - string += 'ttd: {}\n'.format(self.ttd ) - string += 'valid: {}\n'.format(self.valid ) + string += 'signals: {}\n'.format(self.signals) + string += 'signals_dict: {}\n'.format(self.signals_dict) + string += 'ttd: {}\n'.format(self.ttd) + string += 'valid: {}\n'.format(self.valid) string += 'is_disruptive: {}\n'.format(self.is_disruptive) string += 't_disrupt: {}\n'.format(self.t_disrupt) return string - - def num_timesteps(self,prepath): + def num_timesteps(self, prepath): self.restore(prepath) ts = self.ttd.shape[0] self.make_light() @@ -360,131 +362,155 @@ def is_valid(self): def is_disruptive_shot(self): return self.is_disruptive - def get_data_arrays(self,use_signals,dtype='float32'): + def get_data_arrays(self, use_signals, dtype='float32'): t_array = self.ttd - signal_array = np.zeros((len(t_array),sum([sig.num_channels for sig in use_signals])),dtype=dtype) + signal_array = np.zeros( + (len(t_array), sum([sig.num_channels for sig in use_signals])), + dtype=dtype) curr_idx = 0 for sig in use_signals: - signal_array[:,curr_idx:curr_idx+sig.num_channels] = self.signals_dict[sig] + signal_array[:, curr_idx:curr_idx + + sig.num_channels] = self.signals_dict[sig] curr_idx += sig.num_channels - return t_array,signal_array + return t_array, signal_array def get_individual_signal_arrays(self): - #guarantee ordering + # guarantee ordering return [self.signals_dict[sig] for sig in self.signals] - def preprocess(self,conf): + def preprocess(self, conf): sys.stdout.write('\rrecomputing {}'.format(self.number)) sys.stdout.flush() - #get minmax times - time_arrays,signal_arrays,t_min,t_max,valid = self.get_signals_and_times_from_file(conf) + # get minmax times + time_arrays, signal_arrays, t_min, t_max, valid = ( + self.get_signals_and_times_from_file(conf)) self.valid = valid - #cut and resample + # cut and resample if self.valid: - self.cut_and_resample_signals(time_arrays,signal_arrays,t_min,t_max,conf) + self.cut_and_resample_signals( + time_arrays, signal_arrays, t_min, t_max, conf) - def get_signals_and_times_from_file(self,conf): + def get_signals_and_times_from_file(self, conf): valid = True t_min = -np.Inf t_max = np.Inf - t_thresh = -1 + # t_thresh = -1 signal_arrays = [] time_arrays = [] - #disruptive = self.t_disrupt >= 0 + # disruptive = self.t_disrupt >= 0 signal_prepath = conf['paths']['signal_prepath'] - for (i,signal) in enumerate(self.signals): - t,sig,valid_signal = signal.load_data(signal_prepath,self,conf['data']['floatx']) + for (i, signal) in enumerate(self.signals): + t, sig, valid_signal = signal.load_data( + signal_prepath, self, conf['data']['floatx']) if not valid_signal: - return None,None,None,None,False + return None, None, None, None, False else: assert(len(sig.shape) == 2) assert(len(t.shape) == 1) assert(len(t) > 1) - t_min = max(t_min,np.min(t)) + t_min = max(t_min, np.min(t)) signal_arrays.append(sig) time_arrays.append(t) if self.is_disruptive and self.t_disrupt > np.max(t): - if self.t_disrupt > np.max(t) + signal.get_data_avail_tolerance(self.machine): - print('Shot {}: disruption event is not contained in valid time region of signal {} by {}s, omitting.'.format(self.number,signal,self.t_disrupt - np.max(t))) - valid = False + t_max_total = ( + np.max(t) + signal.get_data_avail_tolerance( + self.machine) + ) + if (self.t_disrupt > t_max_total): + print('Shot {}: disruption event '.format(self.number), + 'is not contained in valid time region of ', + 'signal {} by {}s, omitting.'.format( + self.number, signal, + self.t_disrupt - np.max(t))) + valid = False else: - t_max = np.max(t) + signal.get_data_avail_tolerance(self.machine) + t_max = np.max( + t) + signal.get_data_avail_tolerance(self.machine) else: - t_max = min(t_max,np.max(t)) + t_max = min(t_max, np.max(t)) - #make sure the shot is long enough. + # make sure the shot is long enough. dt = conf['data']['dt'] - if (t_max - t_min)/dt <= (2*conf['model']['length']+conf['data']['T_min_warn']): - print('Shot {} contains insufficient data, omitting.'.format(self.number)) + if (t_max - t_min)/dt <= (2*conf['model'] + ['length']+conf['data']['T_min_warn']): + print( + 'Shot {} contains insufficient data, omitting.'.format( + self.number)) valid = False - # if self.is_disruptive and self.t_disrupt > t_max+conf['data']['data_avail_tolerance']: - # print('Shot {}: disruption event is not contained in valid time region by {}s, omitting.'.format(self.number,self.t_disrupt - t_max)) - # valid = False - assert(t_max > t_min or not valid), "t max: {}, t_min: {}".format(t_max,t_min) - - + assert( + t_max > t_min or not valid), "t max: {}, t_min: {}".format( + t_max, t_min) + if self.is_disruptive: assert(self.t_disrupt <= t_max or not valid) t_max = self.t_disrupt - return time_arrays,signal_arrays,t_min,t_max,valid - + return time_arrays, signal_arrays, t_min, t_max, valid - def cut_and_resample_signals(self,time_arrays,signal_arrays,t_min,t_max,conf): + def cut_and_resample_signals( + self, + time_arrays, + signal_arrays, + t_min, + t_max, + conf): dt = conf['data']['dt'] signals_dict = dict() - #resample signals - assert((len(signal_arrays) == len(time_arrays) == len(self.signals)) and len(signal_arrays) > 0) + # resample signals + assert((len(signal_arrays) == len(time_arrays) + == len(self.signals)) and len(signal_arrays) > 0) tr = 0 - for (i,signal) in enumerate(self.signals): - tr,sigr = cut_and_resample_signal(time_arrays[i],signal_arrays[i],t_min,t_max,dt,conf['data']['floatx']) + for (i, signal) in enumerate(self.signals): + tr, sigr = cut_and_resample_signal( + time_arrays[i], signal_arrays[i], t_min, t_max, dt, + conf['data']['floatx']) signals_dict[signal] = sigr - ttd = self.convert_to_ttd(tr,conf) - self.signals_dict =signals_dict + ttd = self.convert_to_ttd(tr, conf) + self.signals_dict = signals_dict self.ttd = ttd - def convert_to_ttd(self,tr,conf): + def convert_to_ttd(self, tr, conf): T_max = conf['data']['T_max'] dt = conf['data']['dt'] if self.is_disruptive: ttd = max(tr) - tr - ttd = np.clip(ttd,0,T_max) + ttd = np.clip(ttd, 0, T_max) else: ttd = T_max*np.ones_like(tr) ttd = np.log10(ttd + 1.0*dt/10) return ttd - def save(self,prepath): + def save(self, prepath): makedirs_process_safe(prepath) save_path = self.get_save_path(prepath) - np.savez(save_path,valid=self.valid,is_disruptive=self.is_disruptive, - signals_dict=self.signals_dict,ttd=self.ttd) + np.savez(save_path, valid=self.valid, is_disruptive=self.is_disruptive, + signals_dict=self.signals_dict, ttd=self.ttd) print('...saved shot {}'.format(self.number)) - def get_save_path(self,prepath): - return get_individual_shot_file(prepath,self.number,'.npz') + def get_save_path(self, prepath): + return get_individual_shot_file(prepath, self.number, '.npz') - def restore(self,prepath,light=False): + def restore(self, prepath, light=False): assert self.previously_saved(prepath), 'shot was never saved' save_path = self.get_save_path(prepath) - dat = np.load(save_path,encoding="latin1") + dat = np.load(save_path, encoding="latin1") self.valid = dat['valid'][()] self.is_disruptive = dat['is_disruptive'][()] if light: self.signals_dict = None - self.ttd = None + self.ttd = None else: self.signals_dict = dat['signals_dict'][()] self.ttd = dat['ttd'] - - def previously_saved(self,prepath): + + def previously_saved(self, prepath): save_path = self.get_save_path(prepath) return os.path.isfile(save_path) @@ -496,6 +522,8 @@ def make_light(self): def is_disruptive_given_disruption_time(t): return t >= 0 -#it used to be in utilities, but can't import globals in multiprocessing -def get_individual_shot_file(prepath,shot_num,ext='.txt'): - return prepath + str(shot_num) + ext +# it used to be in utilities, but can't import globals in multiprocessing + + +def get_individual_shot_file(prepath, shot_num, ext='.txt'): + return prepath + str(shot_num) + ext From d1f20f9dcf2821acbd35b91da464118df6b4cb70 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 24 Sep 2019 15:58:05 -0500 Subject: [PATCH 091/272] Add flake8 linting to new Travis CI build stage --- .travis.yml | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 50f292fb..49b6e77e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,10 @@ language: python -sudo: required - os: - linux +dist: xenial + #python: # - 2.7 # - 3.6 @@ -15,6 +15,9 @@ matrix: python: 2.7 - env: MPI_LIBRARY=openmpi MPI_LIBRARY_VERSION=2.0.0 python: 3.6 + - stage: python linter + install: pip install flake8 + script: python -m flake8 addons: apt: @@ -22,6 +25,11 @@ addons: - python-numpy - python-setuptools +env: + - TEST_DIR=.; TEST_SCRIPT="python setup.py test" + +# before_install: + install: - sh install-mpi.sh - export MPI_PREFIX="${HOME}/opt/${MPI_LIBRARY}-${MPI_LIBRARY_VERSION}" @@ -30,11 +38,22 @@ install: - pip install --upgrade pip - pip install -r requirements-travis.txt -env: - - TEST_DIR=.; TEST_SCRIPT="python setup.py test" +# before_script: + +script: + - cd $TEST_DIR && $TEST_SCRIPT && cd .. -script: cd $TEST_DIR && $TEST_SCRIPT && cd .. +# Specify order of stages; build matrix of installation and regression tests +# will only run if linter stage passes +stages: + - python linter + - test notifications: + email: + on_success: change + on_failure: always slack: secure: NBjGYpIF2VO/GvhbC7XVPIfi0WLGFEuVhi51UbZjfHg4IOv1UuCF0fImi8GN2UZZBFRnZcbtPVffZnyMUyJI0Krw0M1ropkjA4YDaJx1JUGnYuQLxNmX+jbQmN61Usjg5MQOgcRnAn1bdMN1ttWInqkKejpV5buCjZbt8SZbDePfXz4U3No68P/pRsDTVXSy0xLTtRacuEITJjwxfjp6phbmR3qs127MZMRbVYDC2HA6KsoJW6YSKF1vFyHqnFMl7GSavxYw/XQpqFLJkGKXfnNgPZV6qAbVk5+bzyytAbbwLGvgbuFpnJsGPvAyebV8wVYaIPg7OeK+Sm1A3q0jt774NnqFp2AZ+pSKrxbxIkygDM0zoWLm4i3pt6ToJ6dKcqSbCKbELnSY5NphcyYuiJ8uhLVpAR0Y+vp+fOvhb8td/nH2AWkxFpp6xwOHPXvtodBsyPMkiaeKoVElYBfbfOhDSYH2KafADECTX75S63A9KleeNZh0DSImfFdQPaN3GFLEL8Z9UFABzTkM9eYKP9pyEP82Wh/JGDaqbARazEgNzy8rwghomsEguV247XHdOx32PSd0att541gsRFZ5uyMuVFKzI0jiLijekibY2I1c5b6dDeuK4O8uBiq4FTS+bM55Rj4Job01kdxSCKw3RwOh0amzITRTQvEWKpTAekg= + on_success: always + on_failure: always From 02e49629570c9dc2dca1fe7b9c07abbb94f52503 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 24 Sep 2019 16:24:38 -0500 Subject: [PATCH 092/272] Fix style of files in plasma/utils/ --- .travis.yml | 1 + plasma/utils/batch_jobs.py | 175 +++-- plasma/utils/downloading.py | 162 ++-- plasma/utils/evaluation.py | 31 +- plasma/utils/mpi_launch_tensorflow.py | 340 ++++---- plasma/utils/performance.py | 1032 +++++++++++++++---------- plasma/utils/processing.py | 75 +- plasma/utils/state_reset.py | 51 +- 8 files changed, 1139 insertions(+), 728 deletions(-) diff --git a/.travis.yml b/.travis.yml index 49b6e77e..a983ca04 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ matrix: - env: MPI_LIBRARY=openmpi MPI_LIBRARY_VERSION=2.0.0 python: 3.6 - stage: python linter + env: install: pip install flake8 script: python -m flake8 diff --git a/plasma/utils/batch_jobs.py b/plasma/utils/batch_jobs.py index 4ae45732..292964c8 100644 --- a/plasma/utils/batch_jobs.py +++ b/plasma/utils/batch_jobs.py @@ -1,17 +1,17 @@ -from __future__ import division -from pprint import pprint -import yaml +from __future__ import division import datetime import uuid -import sys,os,getpass +import os +# import getpass import subprocess as sp -import numpy as np + def generate_working_dirname(run_directory): s = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") s += "_{}".format(uuid.uuid4()) return run_directory + s + def get_executable_name(conf): shallow = conf['model']['shallow'] if shallow: @@ -20,21 +20,43 @@ def get_executable_name(conf): else: executable_name = conf['paths']['executable'] use_mpi = True - return executable_name,use_mpi - - -def start_slurm_job(subdir,num_nodes,i,conf,shallow,env_name="frnn",env_type="anaconda"): - executable_name,use_mpi = get_executable_name(conf) - os.system(" ".join(["cp -p",executable_name,subdir])) - script = create_slurm_script(subdir,num_nodes,i,executable_name,use_mpi,env_name,env_type) - sp.Popen("sbatch "+script,shell=True) - - -def create_jenkins_script(subdir,num_nodes,executable_name,test_configuration,env_name="frnn",env_type="anaconda"): - filename = "jenkins_{}_{}.cmd".format(test_configuration[0],test_configuration[1]) #version of Python and the dataset - filepath = os.path.join(subdir,filename) - user = getpass.getuser() - with open(filepath,"w") as f: + return executable_name, use_mpi + + +def start_slurm_job( + subdir, + num_nodes, + i, + conf, + shallow, + env_name="frnn", + env_type="anaconda"): + executable_name, use_mpi = get_executable_name(conf) + os.system(" ".join(["cp -p", executable_name, subdir])) + script = create_slurm_script( + subdir, + num_nodes, + i, + executable_name, + use_mpi, + env_name, + env_type) + sp.Popen("sbatch "+script, shell=True) + + +def create_jenkins_script( + subdir, + num_nodes, + executable_name, + test_configuration, + env_name="frnn", + env_type="anaconda"): + filename = "jenkins_{}_{}.cmd".format( + test_configuration[0], + test_configuration[1]) # version of Python and the dataset + filepath = os.path.join(subdir, filename) + # user = getpass.getuser() + with open(filepath, "w") as f: f.write('#!/usr/bin/bash\n') f.write('export OMPI_MCA_btl=\"tcp,self,sm\"\n') f.write('echo \"Jenkins test {}\"\n'.format(test_configuration[1])) @@ -48,36 +70,74 @@ def create_jenkins_script(subdir,num_nodes,executable_name,test_configuration,en f.write('module load cudnn/cuda-8.0/6.0\n') f.write('module load openmpi/cuda-8.0/intel-17.0/2.1.0/64\n') f.write('module load intel/17.0/64/17.0.4.196\n') - f.write('cd /home/alexeys/jenkins/workspace/FRNM/PPPL\n') + f.write('cd /home/alexeys/jenkins/workspace/FRNM/PPPL\n') f.write('python setup.py install\n') f.write('cd {}\n'.format(subdir)) - f.write('srun -N {} -n {} python {}\n'.format(num_nodes//2, num_nodes//2*4, executable_name)) + f.write('srun -N {} -n {} python {}\n'.format( + num_nodes // 2, num_nodes // 2 * 4, executable_name)) return filepath -def start_jenkins_job(subdir,num_nodes,executable_name,test_configuration,env_name,env_type): - os.system(" ".join(["cp -p",executable_name,subdir])) - script = create_jenkins_script(subdir,num_nodes,executable_name,test_configuration,env_name,env_type) - sp.Popen("sh "+script,shell=True) - -def start_pbs_job(subdir,num_nodes,i,conf,shallow,env_name="frnn",env_type="anaconda"): - executable_name,use_mpi = get_executable_name(conf) - os.system(" ".join(["cp -p",executable_name,subdir])) - script = create_pbs_script(subdir,num_nodes,i,executable_name,use_mpi,env_name,env_type) - sp.Popen("qsub "+script,shell=True) - -def create_slurm_script(subdir,num_nodes,idx,executable_name,use_mpi,env_name="frnn",env_type="anaconda"): +def start_jenkins_job( + subdir, + num_nodes, + executable_name, + test_configuration, + env_name, + env_type): + os.system(" ".join(["cp -p", executable_name, subdir])) + script = create_jenkins_script( + subdir, + num_nodes, + executable_name, + test_configuration, + env_name, + env_type) + sp.Popen("sh "+script, shell=True) + + +def start_pbs_job( + subdir, + num_nodes, + i, + conf, + shallow, + env_name="frnn", + env_type="anaconda"): + executable_name, use_mpi = get_executable_name(conf) + os.system(" ".join(["cp -p", executable_name, subdir])) + script = create_pbs_script( + subdir, + num_nodes, + i, + executable_name, + use_mpi, + env_name, + env_type) + sp.Popen("qsub "+script, shell=True) + + +def create_slurm_script( + subdir, + num_nodes, + idx, + executable_name, + use_mpi, + env_name="frnn", + env_type="anaconda"): filename = "run_{}_nodes.cmd".format(num_nodes) filepath = subdir+filename - user = getpass.getuser() - sbatch_header = create_slurm_header(num_nodes,use_mpi,idx) - with open(filepath,"w") as f: + # user = getpass.getuser() + sbatch_header = create_slurm_header(num_nodes, use_mpi, idx) + with open(filepath, "w") as f: for line in sbatch_header: f.write(line) f.write('module load '+env_type+'\n') f.write('source activate '+env_name+'\n') - f.write('module load cudatoolkit/8.0 cudnn/cuda-8.0/6.0 openmpi/cuda-8.0/intel-17.0/2.1.0/64\n') + f.write(( + 'module load cudatoolkit/8.0 cudnn/cuda-8.0/6.0 ' + 'openmpi/cuda-8.0/intel-17.0/2.1.0/64\n')) f.write('module load intel/17.0/64/17.0.5.239 intel-mkl/2017.3/4/64\n') # f.write('rm -f /tigress/{}/model_checkpoints/*.h5\n'.format(user)) f.write('cd {}\n'.format(subdir)) @@ -87,27 +147,38 @@ def create_slurm_script(subdir,num_nodes,idx,executable_name,use_mpi,env_name="f return filepath -def create_pbs_script(subdir,num_nodes,idx,executable_name,use_mpi,env_name="frnn",env_type="anaconda"): + +def create_pbs_script( + subdir, + num_nodes, + idx, + executable_name, + use_mpi, + env_name="frnn", + env_type="anaconda"): filename = "run_{}_nodes.cmd".format(num_nodes) filepath = subdir+filename - user = getpass.getuser() - sbatch_header = create_pbs_header(num_nodes,use_mpi,idx) - with open(filepath,"w") as f: + # user = getpass.getuser() + sbatch_header = create_pbs_header(num_nodes, use_mpi, idx) + with open(filepath, "w") as f: for line in sbatch_header: f.write(line) - #f.write('export HOME=/lustre/atlas/proj-shared/fus117\n') - #f.write('cd $HOME/PPPL/plasma-python/examples\n') + # f.write('export HOME=/lustre/atlas/proj-shared/fus117\n') + # f.write('cd $HOME/PPPL/plasma-python/examples\n') f.write('source $MODULESHOME/init/bash\n') f.write('module load tensorflow\n') # f.write('rm $HOME/tigress/alexeys/model_checkpoints/*\n') f.write('cd {}\n'.format(subdir)) - f.write('aprun -n {} -N1 env KERAS_HOME={} singularity exec $TENSORFLOW_CONTAINER python3 {}\n'.format(str(num_nodes),subdir,executable_name)) + f.write( + 'aprun -n {} -N1 env KERAS_HOME={} singularity exec ' + '$TENSORFLOW_CONTAINER python3 {}\n'.format(str(num_nodes), subdir, + executable_name)) f.write('echo "done."') return filepath -def create_slurm_header(num_nodes,use_mpi,idx): +def create_slurm_header(num_nodes, use_mpi, idx): if not use_mpi: assert(num_nodes == 1) lines = [] @@ -127,7 +198,8 @@ def create_slurm_header(num_nodes,use_mpi,idx): lines.append('\n\n') return lines -def create_pbs_header(num_nodes,use_mpi,idx): + +def create_pbs_header(num_nodes, use_mpi, idx): if not use_mpi: assert(num_nodes == 1) lines = [] @@ -143,7 +215,12 @@ def create_pbs_header(num_nodes,use_mpi,idx): def copy_files_to_environment(subdir): from plasma.conf import conf - normalization_dir = os.path.dirname(conf['paths']['global_normalizer_path']) + normalization_dir = os.path.dirname( + conf['paths']['global_normalizer_path']) if os.path.isdir(normalization_dir): print("Copying normalization to") - os.system(" ".join(["cp -rp",normalization_dir,os.path.join(subdir,os.path.basename(normalization_dir))])) + os.system(" ".join( + ["cp -rp", normalization_dir, + os.path.join(subdir, os.path.basename(normalization_dir))] + ) + ) diff --git a/plasma/utils/downloading.py b/plasma/utils/downloading.py index 8980ebde..4822a99b 100644 --- a/plasma/utils/downloading.py +++ b/plasma/utils/downloading.py @@ -1,4 +1,12 @@ from __future__ import print_function +import errno +import os +# from multiprocessing import Queue +from functools import partial +import multiprocessing as mp +import sys +import time +import numpy as np ''' http://www.mdsplus.org/index.php?title=Documentation:Tutorial:RemoteAccess&open=76203664636339686324830207&page=Documentation%2FThe+MDSplus+tutorial%2FRemote+data+access+in+MDSplus http://piscope.psfc.mit.edu/index.php/MDSplus_%26_python#Simple_example_of_reading_MDSplus_data @@ -14,58 +22,57 @@ - TEST ''' try: - from MDSplus import * + from MDSplus import Connection except ImportError: pass -#from pylab import * -import numpy as np -import sys -import multiprocessing as mp -from functools import partial -from multiprocessing import Queue -import os -import errno # import gadata - # from plasma.primitives.shots import ShotList -#from signals import * - -#print("Importing numpy version"+np.__version__) - def get_missing_value_array(): return np.array([-1.0]) + def makedirs_process_safe(dirpath): - try: #can lead to race condition + try: # can lead to race condition os.makedirs(dirpath) except OSError as e: - if e.errno == errno.EEXIST:# File exists, and it's a directory, another process beat us to creating this dir, that's OK. + # File exists, and it's a directory, another process beat us to + # creating this dir, that's OK. + if e.errno == errno.EEXIST: pass - else:# Our target dir exists as a file, or different error, reraise the error! + else: + # Our target dir exists as a file, or different error, reraise the + # error! raise + def makedirdepth_process_safe(dirpath): - try: #can lead to race condition + try: # can lead to race condition mkdirdepth(dirpath) except OSError as e: - if e.errno == errno.EEXIST:# File exists, and it's a directory, another process beat us to creating this dir, that's OK. + # File exists, and it's a directory, another process beat us to + # creating this dir, that's OK. + if e.errno == errno.EEXIST: pass - else:# Our target dir exists as a file, or different error, reraise the error! + else: + # Our target dir exists as a file, or different error, reraise the + # error! raise + def mkdirdepth(filename): - folder=os.path.dirname(filename) + folder = os.path.dirname(filename) if not os.path.exists(folder): os.makedirs(folder) -def format_save_path(prepath,signal_path,shot_num): - return prepath + signal_path + '/{}.txt'.format(shot_num) + +def format_save_path(prepath, signal_path, shot_num): + return prepath + signal_path + '/{}.txt'.format(shot_num) -def save_shot(shot_num_queue,c,signals,save_prepath,machine,sentinel=-1): +def save_shot(shot_num_queue, c, signals, save_prepath, machine, sentinel=-1): missing_values = 0 # if machine == 'd3d': # reload(gadata) #reloads Gadata object with connection @@ -76,99 +83,132 @@ def save_shot(shot_num_queue,c,signals,save_prepath,machine,sentinel=-1): shot_complete = True for signal in signals: signal_path = signal.get_path(machine) - save_path_full = signal.get_file_path(save_prepath,machine,shot_num) + save_path_full = signal.get_file_path( + save_prepath, machine, shot_num) success = False mapping = None if os.path.isfile(save_path_full): if os.path.getsize(save_path_full) > 0: - print('-',end='') + print('-', end='') success = True else: - print('Signal {}, shot {} was downloaded incorrectly (empty file). Redownloading.'.format(signal_path,shot_num)) + print( + 'Signal {}, shot {} '.format(signal_path, shot_num), + 'was downloaded incorrectly (empty file). ', + 'Redownloading.') if not success: try: try: - time,data,mapping,success = signal.fetch_data(machine,shot_num,c) + time, data, mapping, success = ( + signal.fetch_data(machine, shot_num, c)) if not success: - print('No success shot {}, signal {}'.format(shot_num,signal)) + print('No success shot {}, signal {}'.format( + shot_num, signal)) except Exception as e: print(e) sys.stdout.flush() - #missing_values += 1 - print('Signal {}, shot {} missing. Filling with zeros'.format(signal_path,shot_num)) + # missing_values += 1 + print('Signal {}, shot {} missing. '.format( + signal_path, shot_num), 'Filling with zeros.') success = False - if success: - data_two_column = np.vstack((np.atleast_2d(time),np.atleast_2d(data))).transpose() + data_two_column = np.vstack((np.atleast_2d(time), + np.atleast_2d(data)) + ).transpose() if mapping is not None: - mapping_two_column = np.vstack((np.atleast_2d(time),np.atleast_2d(mapping))).transpose() - # mapping_two_column = np.hstack((np.array([[0.0]]),np.atleast_2d(mapping))) - data_two_column = np.vstack((mapping_two_column,data_two_column)) + mapping_two_column = np.vstack(( + np.atleast_2d(time), np.atleast_2d(mapping)) + ).transpose() + data_two_column = np.vstack( + (mapping_two_column, data_two_column)) makedirdepth_process_safe(save_path_full) if success: - np.savetxt(save_path_full,data_two_column,fmt = '%.5e')#fmt = '%f %f') + np.savetxt(save_path_full, data_two_column, fmt='%.5e') else: - np.savetxt(save_path_full,get_missing_value_array(),fmt = '%.5e') - print('.',end='') - except: - print('Could not save shot {}, signal {}'.format(shot_num,signal_path)) + np.savetxt(save_path_full, get_missing_value_array(), + fmt='%.5e') + print('.', end='') + except BaseException: + print( + 'Could not save shot {}, signal {}'.format( + shot_num, signal_path)) print('Warning: Incomplete!!!') raise sys.stdout.flush() if not success: missing_values += 1 shot_complete = False - #only add shot to list if it was complete + # only add shot to list if it was complete if shot_complete: print('saved shot {}'.format(shot_num)) - #complete_queue.put(shot_num) + # complete_queue.put(shot_num) else: print('shot {} not complete. removing from list.'.format(shot_num)) print('Finished with {} missing values total'.format(missing_values)) return -def download_shot_numbers(shot_numbers,save_prepath,machine,signals): +def download_shot_numbers(shot_numbers, save_prepath, machine, signals): max_cores = machine.max_cores sentinel = -1 - fn = partial(save_shot,signals=signals,save_prepath=save_prepath,machine=machine,sentinel=sentinel) - num_cores = min(mp.cpu_count(),max_cores) #can only handle 8 connections at once :( + fn = partial(save_shot, signals=signals, save_prepath=save_prepath, + machine=machine, sentinel=sentinel) + # can only handle 8 connections at once :( + num_cores = min(mp.cpu_count(), max_cores) queue = mp.Queue() - #complete_shots = Array('i',zeros(len(shot_numbers)))# = mp.Queue() - - assert(len(shot_numbers) < 32000) # mp.queue can't handle larger queues yet! + # complete_shots = Array('i',zeros(len(shot_numbers)))# = mp.Queue() + + # mp.queue can't handle larger queues yet! + assert(len(shot_numbers) < 32000) for shot_num in shot_numbers: queue.put(shot_num) for i in range(num_cores): queue.put(sentinel) connections = [Connection(machine.server) for _ in range(num_cores)] - processes = [mp.Process(target=fn,args=(queue,connections[i])) for i in range(num_cores)] - + processes = [ + mp.Process( + target=fn, + args=( + queue, + connections[i])) for i in range(num_cores)] + print('running in parallel on {} processes'.format(num_cores)) - + for p in processes: p.start() for p in processes: p.join() -def download_all_shot_numbers(prepath,save_path,shot_list_files,signals_full): +def download_all_shot_numbers( + prepath, + save_path, + shot_list_files, + signals_full): max_len = 30000 machine = shot_list_files.machine signals = [] for sig in signals_full: if not sig.is_defined_on_machine(machine): - print('Signal {} not defined on machine {}, omitting'.format(sig,machine)) + print( + 'Signal {} not defined on machine {}, omitting'.format( + sig, machine)) else: signals.append(sig) - save_prepath = prepath+save_path + '/' - shot_numbers,_ = shot_list_files.get_shot_numbers_and_disruption_times() - shot_numbers_chunks = [shot_numbers[i:i+max_len] for i in xrange(0,len(shot_numbers),max_len)]#can only use queue of max size 30000 + save_prepath = prepath+save_path + '/' + shot_numbers, _ = shot_list_files.get_shot_numbers_and_disruption_times() + # can only use queue of max size 30000 + shot_numbers_chunks = [shot_numbers[i:i+max_len] + for i in np.xrange(0, len(shot_numbers), max_len)] start_time = time.time() for shot_numbers_chunk in shot_numbers_chunks: - download_shot_numbers(shot_numbers_chunk,save_prepath,machine,signals) - - print('Finished downloading {} shots in {} seconds'.format(len(shot_numbers),time.time()-start_time)) - + download_shot_numbers( + shot_numbers_chunk, + save_prepath, + machine, + signals) + + print('Finished downloading {} shots in {} seconds'.format( + len(shot_numbers), time.time()-start_time)) diff --git a/plasma/utils/evaluation.py b/plasma/utils/evaluation.py index 9993ee79..5b0305b9 100644 --- a/plasma/utils/evaluation.py +++ b/plasma/utils/evaluation.py @@ -1,27 +1,32 @@ import numpy as np epsilon = 1e-7 -def get_loss_from_list(y_pred_list,y_true_list,target): - return np.mean([get_loss(yg,yp,target) for yp,yg in zip(y_pred_list,y_true_list)]) -def get_loss(y_true,y_pred,target): - return target.loss_np(y_true,y_pred) +def get_loss_from_list(y_pred_list, y_true_list, target): + return np.mean([get_loss(yg, yp, target) + for yp, yg in zip(y_pred_list, y_true_list)]) -def mae_np(y_true,y_pred): + +def get_loss(y_true, y_pred, target): + return target.loss_np(y_true, y_pred) + + +def mae_np(y_true, y_pred): return np.mean(np.abs(y_pred-y_true)) -def mse_np(y_true,y_pred): + +def mse_np(y_true, y_pred): return np.mean((y_pred-y_true)**2) -def binary_crossentropy_np(y_true,y_pred): - y_pred = np.clip(y_pred,epsilon,1-epsilon) - return np.mean(- (y_true*np.log(y_pred) + (1-y_true)*np.log(1 - y_pred))) -def hinge_np(y_true,y_pred): - return np.mean(np.maximum(0.0,1 - y_pred*y_true)) +def binary_crossentropy_np(y_true, y_pred): + y_pred = np.clip(y_pred, epsilon, 1-epsilon) + return np.mean(- (y_true*np.log(y_pred) + (1-y_true)*np.log(1 - y_pred))) -def squared_hinge_np(y_true,y_pred): - return np.mean(np.maximum(0.0,1 - y_pred*y_true)**2) +def hinge_np(y_true, y_pred): + return np.mean(np.maximum(0.0, 1 - y_pred*y_true)) +def squared_hinge_np(y_true, y_pred): + return np.mean(np.maximum(0.0, 1 - y_pred*y_true)**2) diff --git a/plasma/utils/mpi_launch_tensorflow.py b/plasma/utils/mpi_launch_tensorflow.py index f372683c..c1ea9fec 100644 --- a/plasma/utils/mpi_launch_tensorflow.py +++ b/plasma/utils/mpi_launch_tensorflow.py @@ -2,156 +2,206 @@ from mpi4py import MPI from hostlist import expand_hostlist -import socket,os,math +import socket +import os +import math + def get_host_to_id_mapping(): - return {host:i for (i,host) in enumerate(expand_hostlist( os.environ['SLURM_NODELIST']))} + return { + host: i for ( + i, host) in enumerate( + expand_hostlist( + os.environ['SLURM_NODELIST']))} + def get_my_host_id(): - return get_host_to_id_mapping()[socket.gethostname()] + return get_host_to_id_mapping()[socket.gethostname()] def get_host_list(port): - return [ '{}:{}'.format(host,port) for host in expand_hostlist( os.environ['SLURM_NODELIST']) ] - -def get_worker_host_list(base_port,workers_per_host): - hosts = expand_hostlist( os.environ['SLURM_NODELIST']) - ports = [base_port + i for i in range(workers_per_host)] - l = [] - for h in hosts: - for p in ports: - l.append('{}:{}'.format(h,p)) - return l - -def get_worker_host(base_port,workers_per_host,task_id): - return get_worker_host_list(base_port,workers_per_host)[task_id] - -def get_ps_host_list(base_port,num_ps): - assert(num_ps < 10000000) - port = base_port - l = [] - hosts = expand_hostlist( os.environ['SLURM_NODELIST']) - while True: - for host in hosts: - if len(l) >= num_ps: - return l - l.append('{}:{}'.format(host,port)) - port += 1 - -def get_ps_host(base_port,num_ps,num_workers,task_id): - return get_ps_host_list(base_port,num_ps)[task_id - num_workers] - -def get_mpi_cluster_server_jobname(num_ps = 1,num_workers = None): - import tensorflow as tf - NUM_GPUS = 4 - comm = MPI.COMM_WORLD - task_index = comm.Get_rank() - task_num = comm.Get_size() - num_hosts = len(get_host_list(1)) - - num_workers_per_host = NUM_GPUS - if num_workers is None or num_workers == 0 or num_workers > num_workers_per_host*num_hosts: - if num_workers > num_workers_per_host*num_hosts: - print('Num workers too large (more than one per GPU). Setting to default of {} for {} hosts'.format(num_workers_per_host*num_hosts,num_hosts)) - if num_workers == 0: - print('Num workers set to 0, should be positive. Setting to default of {} for {} hosts'.format(num_workers_per_host*num_hosts,num_hosts)) - num_workers = num_workers_per_host*num_hosts - - - tasks_per_node = task_num / num_hosts - - max_ps = num_hosts*(tasks_per_node - num_workers_per_host) - print("tasks_per_node {} num_workers_per_host {} num_hosts {}".format(tasks_per_node,num_workers_per_host,num_hosts)) - if num_ps == 0 or num_ps > max_ps: - print('Invalid number of ps {} (maximum {}, minimum 0)'.format(num_ps,max_ps)) - if num_ps == 0: - print('Setting to 1') - num_ps = 1 + return [ + '{}:{}'.format( + host, + port) for host in expand_hostlist( + os.environ['SLURM_NODELIST'])] + + +def get_worker_host_list(base_port, workers_per_host): + hosts = expand_hostlist(os.environ['SLURM_NODELIST']) + ports = [base_port + i for i in range(workers_per_host)] + worker_hlist = [] + for h in hosts: + for p in ports: + worker_hlist.append('{}:{}'.format(h, p)) + return worker_hlist + + +def get_worker_host(base_port, workers_per_host, task_id): + return get_worker_host_list(base_port, workers_per_host)[task_id] + + +def get_ps_host_list(base_port, num_ps): + assert(num_ps < 10000000) + port = base_port + ps_hlist = [] + hosts = expand_hostlist(os.environ['SLURM_NODELIST']) + while True: + for host in hosts: + if len(ps_hlist) >= num_ps: + return ps_hlist + ps_hlist.append('{}:{}'.format(host, port)) + port += 1 + + +def get_ps_host(base_port, num_ps, num_workers, task_id): + return get_ps_host_list(base_port, num_ps)[task_id - num_workers] + + +def get_mpi_cluster_server_jobname(num_ps=1, num_workers=None): + import tensorflow as tf + NUM_GPUS = 4 + comm = MPI.COMM_WORLD + task_index = comm.Get_rank() + task_num = comm.Get_size() + num_hosts = len(get_host_list(1)) + + num_workers_per_host = NUM_GPUS + # TODO(KGF): this error handling is completely duplicated below + if (num_workers is None or num_workers == 0 + or num_workers > num_workers_per_host*num_hosts): + if num_workers > num_workers_per_host*num_hosts: + print('Num workers too large (more than one per GPU). ', + 'Setting to default of {} for {} hosts'.format( + num_workers_per_host * num_hosts, num_hosts)) + if num_workers == 0: + print('Num workers set to 0, should be positive. ', + 'Setting to default of {} for {} hosts'.format( + num_workers_per_host * num_hosts, num_hosts)) + num_workers = num_workers_per_host*num_hosts + + tasks_per_node = task_num / num_hosts + + max_ps = num_hosts*(tasks_per_node - num_workers_per_host) + print("tasks_per_node {} num_workers_per_host {} num_hosts {}".format( + tasks_per_node, num_workers_per_host, num_hosts)) + if num_ps == 0 or num_ps > max_ps: + print( + 'Invalid number of ps {} (maximum {}, minimum 0)'.format( + num_ps, max_ps)) + if num_ps == 0: + print('Setting to 1') + num_ps = 1 + else: + print('Setting to {}'.format(max_ps)) + num_ps = max_ps + num_ps_per_host = int(math.ceil(1.0*num_ps/num_hosts)) + + task_index = task_index % tasks_per_node + + if task_index < NUM_GPUS: + job_name = 'worker' + num_per_host = num_workers_per_host + global_task_index = get_my_host_id()*num_per_host+task_index + else: + job_name = 'ps' + task_index = task_index - NUM_GPUS + num_per_host = num_ps_per_host + os.environ['CUDA_VISIBLE_DEVICES'] = '' + global_task_index = num_hosts*task_index+get_my_host_id() + + # if job_name == "ps": + # if job_name == "worker": + # os.environ['CUDA_VISIBLE_DEVICES'] = '{}'.format(MY_GPU) + num_total = num_workers + num_ps + assert(task_num >= num_total) + + if task_index == 0: + print('{} superfluous processes'.format(task_num - num_total)) + print( + '{} superfluous workers'.format( + num_workers_per_host + * num_hosts + - num_workers)) + print('{} superfluous ps'.format(num_ps_per_host*num_hosts - num_ps)) + + print( + '{}, task_id: {}, host_id: {}'.format( + socket.gethostname(), + task_index, + get_my_host_id())) + + worker_hosts = get_worker_host_list( + 2222, num_workers_per_host)[:num_workers] + ps_hosts = get_ps_host_list(2322, num_ps) + if global_task_index == 0: + print( + 'ps_hosts: {}\n, worker hosts: {}\n'.format( + ps_hosts, + worker_hosts)) + # Create a cluster from the parameter server and worker hosts. + if job_name == 'ps' and global_task_index >= num_ps: + exit(0) + if job_name == 'worker' and global_task_index >= num_workers: + exit(0) + + cluster = tf.train.ClusterSpec({"ps": ps_hosts, "worker": worker_hosts}) + # Create and start a server for the local task. + server = tf.train.Server( + cluster, + job_name=job_name, + task_index=global_task_index) + + return cluster, server, job_name, global_task_index, num_workers + + +def get_mpi_task_index(num_workers=None): + NUM_GPUS = 4 + comm = MPI.COMM_WORLD + task_index = comm.Get_rank() + task_num = comm.Get_size() + num_hosts = len(get_host_list(1)) + + num_workers_per_host = NUM_GPUS + if (num_workers is None or num_workers == 0 + or num_workers > num_workers_per_host*num_hosts): + if num_workers > num_workers_per_host*num_hosts: + print('Num workers too large (more than one per GPU). ', + 'Setting to default of {} for {} hosts'.format( + num_workers_per_host * num_hosts, num_hosts)) + if num_workers == 0: + print('Num workers set to 0, should be positive. ', + 'Setting to default of {} for {} hosts'.format( + num_workers_per_host * num_hosts, num_hosts)) + num_workers = num_workers_per_host*num_hosts + + tasks_per_node = task_num / num_hosts + task_index = task_index % tasks_per_node + + if task_index < NUM_GPUS: + job_name = 'worker' + num_per_host = num_workers_per_host + global_task_index = get_my_host_id()*num_per_host+task_index else: - print('Setting to {}'.format(max_ps)) - num_ps = max_ps - num_ps_per_host = int(math.ceil(1.0*num_ps/num_hosts)) - - - task_index = task_index % tasks_per_node - - if task_index < NUM_GPUS: - job_name = 'worker' - num_per_host = num_workers_per_host - global_task_index = get_my_host_id()*num_per_host+task_index - else: - job_name = 'ps' - task_index = task_index - NUM_GPUS - num_per_host = num_ps_per_host - os.environ['CUDA_VISIBLE_DEVICES'] = '' - global_task_index = num_hosts*task_index+get_my_host_id() - - # if job_name == "ps": - # if job_name == "worker": - # os.environ['CUDA_VISIBLE_DEVICES'] = '{}'.format(MY_GPU) - num_total = num_workers + num_ps - assert(task_num >= num_total) - - if task_index == 0: - print('{} superfluous processes'.format(task_num - num_total)) - print('{} superfluous workers'.format(num_workers_per_host*num_hosts - num_workers)) - print('{} superfluous ps'.format(num_ps_per_host*num_hosts - num_ps)) - - - print('{}, task_id: {}, host_id: {}'.format(socket.gethostname(),task_index,get_my_host_id())) - - worker_hosts = get_worker_host_list(2222,num_workers_per_host)[:num_workers] - ps_hosts = get_ps_host_list(2322,num_ps) - if global_task_index == 0: - print('ps_hosts: {}\n, worker hosts: {}\n'.format(ps_hosts,worker_hosts)) - # Create a cluster from the parameter server and worker hosts. - if job_name == 'ps' and global_task_index >= num_ps: - exit(0) - if job_name == 'worker' and global_task_index >= num_workers: - exit(0) - - cluster = tf.train.ClusterSpec({"ps": ps_hosts, "worker": worker_hosts}) - # Create and start a server for the local task. - server = tf.train.Server(cluster,job_name=job_name,task_index=global_task_index) - - return cluster,server,job_name,global_task_index,num_workers - -def get_mpi_task_index(num_workers = None): - NUM_GPUS = 4 - comm = MPI.COMM_WORLD - task_index = comm.Get_rank() - task_num = comm.Get_size() - num_hosts = len(get_host_list(1)) - - num_workers_per_host = NUM_GPUS - if num_workers is None or num_workers == 0 or num_workers > num_workers_per_host*num_hosts: - if num_workers > num_workers_per_host*num_hosts: - print('Num workers too large (more than one per GPU). Setting to default of {} for {} hosts'.format(num_workers_per_host*num_hosts,num_hosts)) - if num_workers == 0: - print('Num workers set to 0, should be positive. Setting to default of {} for {} hosts'.format(num_workers_per_host*num_hosts,num_hosts)) - num_workers = num_workers_per_host*num_hosts - - - tasks_per_node = task_num / num_hosts - task_index = task_index % tasks_per_node - - if task_index < NUM_GPUS: - job_name = 'worker' - num_per_host = num_workers_per_host - global_task_index = get_my_host_id()*num_per_host+task_index - else: - exit(0) - - num_total = num_workers - assert(task_num >= num_total) - - if task_index == 0: - print('{} superfluous workers'.format(num_workers_per_host*num_hosts - num_workers)) - print('{} total workers'.format(num_workers)) - - - print('{}, task_id: {}, host_id: {}'.format(socket.gethostname(),task_index,get_my_host_id())) - if job_name == 'worker' and global_task_index >= num_workers: - exit(0) - - return global_task_index,num_workers + exit(0) + + num_total = num_workers + assert(task_num >= num_total) + + if task_index == 0: + print( + '{} superfluous workers'.format( + num_workers_per_host + * num_hosts + - num_workers)) + print('{} total workers'.format(num_workers)) + + print( + '{}, task_id: {}, host_id: {}'.format( + socket.gethostname(), + task_index, + get_my_host_id())) + if job_name == 'worker' and global_task_index >= num_workers: + exit(0) + + return global_task_index, num_workers diff --git a/plasma/utils/performance.py b/plasma/utils/performance.py index 7bbb9538..5df6a108 100644 --- a/plasma/utils/performance.py +++ b/plasma/utils/performance.py @@ -1,29 +1,38 @@ from __future__ import print_function -import matplotlib -matplotlib.use('Agg')#for machines that don't have a display +from plasma.preprocessor.normalize import VarNormalizer as Normalizer +from plasma.primitives.shots import ShotList # , Shot +from scipy import stats +import numpy as np +from pprint import pprint +import os +import matplotlib.pyplot as plt from matplotlib import rc -rc('font',**{'family':'serif','sans-serif':['Times']}) +import matplotlib +matplotlib.use('Agg') # for machines that don't have a display +rc('font', **{'family': 'serif', 'sans-serif': ['Times']}) rc('text', usetex=True) -import matplotlib.pyplot as plt -import os -from pprint import pprint -import numpy as np -from scipy import stats -from plasma.preprocessor.normalize import VarNormalizer as Normalizer -from plasma.conf import conf -from plasma.primitives.shots import Shot,ShotList class PerformanceAnalyzer(): - def __init__(self,results_dir=None,shots_dir=None,i = 0,T_min_warn = None,T_max_warn = None, verbose = False,pred_ttd=False,conf=None): + def __init__( + self, + results_dir=None, + shots_dir=None, + i=0, + T_min_warn=None, + T_max_warn=None, + verbose=False, + pred_ttd=False, + conf=None): self.T_min_warn = T_min_warn self.T_max_warn = T_max_warn dt = conf['data']['dt'] T_max_warn_def = int(round(conf['data']['T_warning']/dt)) - T_min_warn_def = conf['data']['T_min_warn']#int(round(conf['data']['T_min_warn']/dt)) - if T_min_warn == None: + # int(round(conf['data']['T_min_warn']/dt)) + T_min_warn_def = conf['data']['T_min_warn'] + if T_min_warn is None: self.T_min_warn = T_min_warn_def - if T_max_warn == None: + if T_max_warn is None: self.T_max_warn = T_max_warn_def self.verbose = verbose self.results_dir = results_dir @@ -47,214 +56,239 @@ def __init__(self,results_dir=None,shots_dir=None,i = 0,T_min_warn = None,T_max_ self.normalizer = None - - - def get_metrics_vs_p_thresh(self,mode): + def get_metrics_vs_p_thresh(self, mode): if mode == 'train': all_preds = self.pred_train all_truths = self.truth_train all_disruptive = self.disruptive_train - elif mode == 'test': all_preds = self.pred_test all_truths = self.truth_test all_disruptive = self.disruptive_test - return self.get_metrics_vs_p_thresh_custom(all_preds,all_truths,all_disruptive) - - + return self.get_metrics_vs_p_thresh_custom( + all_preds, all_truths, all_disruptive) - def get_metrics_vs_p_thresh_custom(self,all_preds,all_truths,all_disruptive): - return self.get_metrics_vs_p_thresh_fast(all_preds,all_truths,all_disruptive) + def get_metrics_vs_p_thresh_custom( + self, all_preds, all_truths, all_disruptive): + return self.get_metrics_vs_p_thresh_fast( + all_preds, all_truths, all_disruptive) P_thresh_range = self.get_p_thresh_range() correct_range = np.zeros_like(P_thresh_range) accuracy_range = np.zeros_like(P_thresh_range) fp_range = np.zeros_like(P_thresh_range) missed_range = np.zeros_like(P_thresh_range) early_alarm_range = np.zeros_like(P_thresh_range) - - for i,P_thresh in enumerate(P_thresh_range): - correct,accuracy,fp_rate,missed,early_alarm_rate = self.summarize_shot_prediction_stats(P_thresh,all_preds,all_truths,all_disruptive) + + for i, P_thresh in enumerate(P_thresh_range): + correct, accuracy, fp_rate, missed, early_alarm_rate = ( + self.summarize_shot_prediction_stats( + P_thresh, all_preds, all_truths, all_disruptive)) correct_range[i] = correct - accuracy_range[i] = accuracy - fp_range[i] = fp_rate + accuracy_range[i] = accuracy + fp_range[i] = fp_rate missed_range[i] = missed early_alarm_range[i] = early_alarm_rate - - return correct_range,accuracy_range,fp_range,missed_range,early_alarm_range + return (correct_range, accuracy_range, fp_range, missed_range, + early_alarm_range) def get_p_thresh_range(self): - #return self.conf['data']['target'].threshold_range(self.conf['data']['T_warning']) - if np.any(self.p_thresh_range) == None: - all_preds_tr = self.pred_train - all_truths_tr = self.truth_train - all_disruptive_tr = self.disruptive_train - all_preds_te = self.pred_test - all_truths_te = self.truth_test - all_disruptive_te = self.disruptive_test - - early_th_tr,correct_th_tr,late_th_tr,nd_th_tr = self.get_threshold_arrays(all_preds_tr,all_truths_tr,all_disruptive_tr) - early_th_te,correct_th_te,late_th_te,nd_th_te = self.get_threshold_arrays(all_preds_te,all_truths_te,all_disruptive_te) - all_thresholds = np.sort(np.concatenate((early_th_tr,correct_th_tr,late_th_tr,nd_th_tr,early_th_te,correct_th_te,late_th_te,nd_th_te))) - self.p_thresh_range = all_thresholds - #print(np.unique(self.p_thresh_range)) + if np.any(self.p_thresh_range) is None: + all_preds_tr = self.pred_train + all_truths_tr = self.truth_train + all_disruptive_tr = self.disruptive_train + all_preds_te = self.pred_test + all_truths_te = self.truth_test + all_disruptive_te = self.disruptive_test + + early_th_tr, correct_th_tr, late_th_tr, nd_th_tr = ( + self.get_threshold_arrays(all_preds_tr, all_truths_tr, + all_disruptive_tr)) + early_th_te, correct_th_te, late_th_te, nd_th_te = ( + self.get_threshold_arrays(all_preds_te, all_truths_te, + all_disruptive_te)) + all_thresholds = np.sort(np.concatenate( + (early_th_tr, correct_th_tr, late_th_tr, nd_th_tr, early_th_te, + correct_th_te, late_th_te, nd_th_te))) + self.p_thresh_range = all_thresholds + # print(np.unique(self.p_thresh_range)) return self.p_thresh_range - - def get_metrics_vs_p_thresh_fast(self,all_preds,all_truths,all_disruptive): - all_disruptive = np.array(all_disruptive) + def get_metrics_vs_p_thresh_fast( + self, all_preds, all_truths, all_disruptive): + all_disruptive = np.array(all_disruptive) if self.pred_train is not None: - p_thresh_range = self.get_p_thresh_range() + p_thresh_range = self.get_p_thresh_range() else: - early_th,correct_th,late_th,nd_th = self.get_threshold_arrays(all_preds,all_truths,all_disruptive) - p_thresh_range = np.sort(np.concatenate((early_th,correct_th,late_th,nd_th))) + early_th, correct_th, late_th, nd_th = self.get_threshold_arrays( + all_preds, all_truths, all_disruptive) + p_thresh_range = np.sort(np.concatenate( + (early_th, correct_th, late_th, nd_th))) correct_range = np.zeros_like(p_thresh_range) accuracy_range = np.zeros_like(p_thresh_range) fp_range = np.zeros_like(p_thresh_range) missed_range = np.zeros_like(p_thresh_range) early_alarm_range = np.zeros_like(p_thresh_range) - early_th,correct_th,late_th,nd_th = self.get_threshold_arrays(all_preds,all_truths,all_disruptive) + early_th, correct_th, late_th, nd_th = self.get_threshold_arrays( + all_preds, all_truths, all_disruptive) - for i,thresh in enumerate(p_thresh_range): - #correct,accuracy,fp_rate,missed,early_alarm_rate = self.summarize_shot_prediction_stats(thresh,all_preds,all_truths,all_disruptive) - correct,accuracy,fp_rate,missed,early_alarm_rate = self.get_shot_prediction_stats_from_threshold_arrays(early_th,correct_th,late_th,nd_th,thresh) + for i, thresh in enumerate(p_thresh_range): + correct, accuracy, fp_rate, missed, early_alarm_rate = ( + self.get_shot_prediction_stats_from_threshold_arrays( + early_th, correct_th, late_th, nd_th, thresh)) correct_range[i] = correct - accuracy_range[i] = accuracy - fp_range[i] = fp_rate + accuracy_range[i] = accuracy + fp_range[i] = fp_rate missed_range[i] = missed early_alarm_range[i] = early_alarm_rate - - return correct_range,accuracy_range,fp_range,missed_range,early_alarm_range - def get_shot_prediction_stats_from_threshold_arrays(self,early_th,correct_th,late_th,nd_th,thresh): - indices = np.where(np.logical_and(correct_th > thresh,early_th <= thresh))[0] - FPs = np.sum(nd_th > thresh) - TNs = len(nd_th) - FPs + return (correct_range, accuracy_range, fp_range, missed_range, + early_alarm_range) - earlies = np.sum(early_th > thresh) - TPs = np.sum(np.logical_and(early_th <= thresh,correct_th > thresh)) - lates = np.sum(np.logical_and(np.logical_and(early_th <= thresh,correct_th <= thresh),late_th > thresh)) - FNs = np.sum(np.logical_and(np.logical_and(early_th <= thresh,correct_th <= thresh),late_th <= thresh)) + def get_shot_prediction_stats_from_threshold_arrays( + self, early_th, correct_th, late_th, nd_th, thresh): + # indices = np.where(np.logical_and( + # correct_th > thresh, early_th <= thresh))[0] + FPs = np.sum(nd_th > thresh) + TNs = len(nd_th) - FPs - return self.get_accuracy_and_fp_rate_from_stats(TPs,FPs,FNs,TNs,earlies,lates) + earlies = np.sum(early_th > thresh) + TPs = np.sum(np.logical_and(early_th <= thresh, correct_th > thresh)) + lates = np.sum(np.logical_and(np.logical_and( + early_th <= thresh, correct_th <= thresh), late_th > thresh)) + FNs = np.sum(np.logical_and(np.logical_and( + early_th <= thresh, correct_th <= thresh), late_th <= thresh)) + return self.get_accuracy_and_fp_rate_from_stats( + TPs, FPs, FNs, TNs, earlies, lates) - def get_shot_difficulty(self,preds,truths,disruptives): + def get_shot_difficulty(self, preds, truths, disruptives): disruptives = np.array(disruptives) - d_early_thresholds, d_correct_thresholds,d_late_thresholds, nd_thresholds = self.get_threshold_arrays(preds,truths,disruptives) - d_thresholds = np.maximum(d_early_thresholds,d_correct_thresholds) - #rank shots by difficulty. rank 1 is assigned to lowest value, should be highest difficulty - d_ranks = stats.rankdata(d_thresholds,method='min')#difficulty is highest when threshold is low, can't detect disruption - nd_ranks = stats.rankdata(-nd_thresholds,method='min')#difficulty is highest when threshold is high, can't avoid false positive + (d_early_thresholds, d_correct_thresholds, d_late_thresholds, + nd_thresholds) = self.get_threshold_arrays(preds, truths, disruptives) + d_thresholds = np.maximum(d_early_thresholds, d_correct_thresholds) + # rank shots by difficulty. rank 1 is assigned to lowest value, should + # be highest difficulty + # difficulty is highest when threshold is low, can't detect disruption + d_ranks = stats.rankdata(d_thresholds, method='min') + # difficulty is highest when threshold is high, can't avoid false + # positive + nd_ranks = stats.rankdata(-nd_thresholds, method='min') ranking_fac = self.saved_conf['training']['ranking_difficulty_fac'] - facs_d = np.linspace(ranking_fac,1,len(d_ranks))[d_ranks-1] - facs_nd = np.linspace(ranking_fac,1,len(nd_ranks))[nd_ranks-1] + facs_d = np.linspace(ranking_fac, 1, len(d_ranks))[d_ranks-1] + facs_nd = np.linspace(ranking_fac, 1, len(nd_ranks))[nd_ranks-1] ret_facs = np.ones(len(disruptives)) ret_facs[disruptives] = facs_d ret_facs[~disruptives] = facs_nd - #print("setting shot difficulty") - #print(disruptives) - #print(d_thresholds) - #print(nd_thresholds) - #print(ret_facs) + # print("setting shot difficulty") + # print(disruptives) + # print(d_thresholds) + # print(nd_thresholds) + # print(ret_facs) return ret_facs - def get_threshold_arrays(self,preds,truths,disruptives): - num_d = np.sum(disruptives) - num_nd = np.sum(~disruptives) - nd_thresholds = [] - d_early_thresholds = [] - d_correct_thresholds = [] - d_late_thresholds = [] + def get_threshold_arrays(self, preds, truths, disruptives): + # num_d = np.sum(disruptives) + # num_nd = np.sum(~disruptives) + nd_thresholds = [] + d_early_thresholds = [] + d_correct_thresholds = [] + d_late_thresholds = [] for i in range(len(preds)): pred = 1.0*preds[i] truth = truths[i] pred[:self.get_ignore_indices()] = -np.inf - is_disruptive = disruptives[i] + is_disruptive = disruptives[i] if is_disruptive: - max_acceptable = self.create_acceptable_region(truth,'max') - min_acceptable = self.create_acceptable_region(truth,'min') - correct_indices = np.logical_and(max_acceptable, ~min_acceptable) + max_acceptable = self.create_acceptable_region(truth, 'max') + min_acceptable = self.create_acceptable_region(truth, 'min') + correct_indices = np.logical_and( + max_acceptable, ~min_acceptable) early_indices = ~max_acceptable late_indices = min_acceptable if np.sum(late_indices) == 0: - d_late_thresholds.append(-np.inf) + d_late_thresholds.append(-np.inf) else: - d_late_thresholds.append(np.max(pred[late_indices])) + d_late_thresholds.append(np.max(pred[late_indices])) if np.sum(early_indices) == 0: - d_early_thresholds.append(-np.inf) + d_early_thresholds.append(-np.inf) else: - d_early_thresholds.append(np.max(pred[early_indices])) - + d_early_thresholds.append(np.max(pred[early_indices])) + d_correct_thresholds.append(np.max(pred[correct_indices])) else: nd_thresholds.append(np.max(pred)) - return np.array(d_early_thresholds), np.array(d_correct_thresholds),np.array(d_late_thresholds), np.array(nd_thresholds) - - - - - + return (np.array(d_early_thresholds), np.array(d_correct_thresholds), + np.array(d_late_thresholds), np.array(nd_thresholds)) - def summarize_shot_prediction_stats_by_mode(self,P_thresh,mode,verbose=False): + def summarize_shot_prediction_stats_by_mode( + self, P_thresh, mode, verbose=False): if mode == 'train': all_preds = self.pred_train all_truths = self.truth_train all_disruptive = self.disruptive_train - elif mode == 'test': all_preds = self.pred_test all_truths = self.truth_test all_disruptive = self.disruptive_test - return self.summarize_shot_prediction_stats(P_thresh,all_preds,all_truths,all_disruptive,verbose) - + return self.summarize_shot_prediction_stats( + P_thresh, all_preds, all_truths, all_disruptive, verbose) - def summarize_shot_prediction_stats(self,P_thresh,all_preds,all_truths,all_disruptive,verbose=False): - TPs,FPs,FNs,TNs,earlies,lates = (0,0,0,0,0,0) + def summarize_shot_prediction_stats( + self, + P_thresh, + all_preds, + all_truths, + all_disruptive, + verbose=False): + TPs, FPs, FNs, TNs, earlies, lates = (0, 0, 0, 0, 0, 0) for i in range(len(all_preds)): preds = all_preds[i] truth = all_truths[i] is_disruptive = all_disruptive[i] - - TP,FP,FN,TN,early,late = self.get_shot_prediction_stats(P_thresh,preds,truth,is_disruptive) + TP, FP, FN, TN, early, late = self.get_shot_prediction_stats( + P_thresh, preds, truth, is_disruptive) TPs += TP FPs += FP FNs += FN TNs += TN earlies += early lates += late - + disr = earlies + lates + TPs + FNs nondisr = FPs + TNs if verbose: - print('total: {}, tp: {} fp: {} fn: {} tn: {} early: {} late: {} disr: {} nondisr: {}'.format(len(all_preds),TPs,FPs,FNs,TNs,earlies,lates,disr,nondisr)) - - return self.get_accuracy_and_fp_rate_from_stats(TPs,FPs,FNs,TNs,earlies,lates,verbose) + print('total: {}, tp: {} fp: {} fn: {} tn: {} '.format( + len(all_preds), TPs, FPs, FNs, TNs,), + 'early: {} late: {} disr: {} nondisr: {}'.format( + earlies, lates, disr, nondisr)) + return self.get_accuracy_and_fp_rate_from_stats( + TPs, FPs, FNs, TNs, earlies, lates, verbose) + # we are interested in the predictions of the *first alarm* - #we are interested in the predictions of the *first alarm* - def get_shot_prediction_stats(self,P_thresh,pred,truth,is_disruptive): + def get_shot_prediction_stats(self, P_thresh, pred, truth, is_disruptive): if self.pred_ttd: predictions = pred < P_thresh else: predictions = pred > P_thresh - predictions = np.reshape(predictions,(len(predictions),)) - - max_acceptable = self.create_acceptable_region(truth,'max') - min_acceptable = self.create_acceptable_region(truth,'min') - + predictions = np.reshape(predictions, (len(predictions),)) + + max_acceptable = self.create_acceptable_region(truth, 'max') + min_acceptable = self.create_acceptable_region(truth, 'min') + early = late = TP = TN = FN = FP = 0 - - positives = self.get_positives(predictions)#where(predictions)[0] + + positives = self.get_positives(predictions) # where(predictions)[0] if len(positives) == 0: if is_disruptive: FN = 1 @@ -263,7 +297,8 @@ def get_shot_prediction_stats(self,P_thresh,pred,truth,is_disruptive): else: if is_disruptive: first_pred_idx = positives[0] - if max_acceptable[first_pred_idx] and ~min_acceptable[first_pred_idx]: + if (max_acceptable[first_pred_idx] + and ~min_acceptable[first_pred_idx]): TP = 1 elif min_acceptable[first_pred_idx]: late = 1 @@ -271,51 +306,52 @@ def get_shot_prediction_stats(self,P_thresh,pred,truth,is_disruptive): early = 1 else: FP = 1 - return TP,FP,FN,TN,early,late + return TP, FP, FN, TN, early, late def get_ignore_indices(self): return self.saved_conf['model']['ignore_timesteps'] - - def get_positives(self,predictions): + def get_positives(self, predictions): indices = np.arange(len(predictions)) - return np.where(np.logical_and(predictions,indices >= self.get_ignore_indices()))[0] + return np.where( + np.logical_and( + predictions, + indices >= self.get_ignore_indices()))[0] - - def create_acceptable_region(self,truth,mode): + def create_acceptable_region(self, truth, mode): if mode == 'min': acceptable_timesteps = self.T_min_warn elif mode == 'max': acceptable_timesteps = self.T_max_warn else: print('Error Invalid Mode for acceptable region') - exit(1) + exit(1) - acceptable = np.zeros_like(truth,dtype=bool) + acceptable = np.zeros_like(truth, dtype=bool) if acceptable_timesteps > 0: acceptable[-acceptable_timesteps:] = True return acceptable - - def get_accuracy_and_fp_rate_from_stats(self,tp,fp,fn,tn,early,late,verbose=False): + def get_accuracy_and_fp_rate_from_stats( + self, tp, fp, fn, tn, early, late, verbose=False): total = tp + fp + fn + tn + early + late - disr = early + late + tp + fn + disr = early + late + tp + fn nondisr = fp + tn - + if disr == 0: early_alarm_rate = 0 missed = 0 - accuracy = 0 + accuracy = 0 else: early_alarm_rate = 1.0*early/disr missed = 1.0*(late + fn)/disr accuracy = 1.0*tp/disr if nondisr == 0: fp_rate = 0 - else: + else: fp_rate = 1.0*fp/nondisr correct = 1.0*(tp + tn)/total - + if verbose: print('accuracy: {}'.format(accuracy)) print('missed: {}'.format(missed)) @@ -323,15 +359,14 @@ def get_accuracy_and_fp_rate_from_stats(self,tp,fp,fn,tn,early,late,verbose=Fals print('false positive rate: {}'.format(fp_rate)) print('correct: {}'.format(correct)) - return correct,accuracy,fp_rate,missed,early_alarm_rate - - + return correct, accuracy, fp_rate, missed, early_alarm_rate def load_ith_file(self): results_files = os.listdir(self.results_dir) print(results_files) dat = np.load(self.results_dir + results_files[self.i]) - print("Loading results file {}".format(self.results_dir + results_files[self.i])) + print("Loading results file {}".format( + self.results_dir + results_files[self.i])) if self.verbose: print('configuration: {} '.format(dat['conf'])) @@ -345,44 +380,54 @@ def load_ith_file(self): self.shot_list_test = ShotList(dat['shot_list_test'][()]) self.shot_list_train = ShotList(dat['shot_list_train'][()]) self.saved_conf = dat['conf'][()] - self.conf['data']['T_warning'] = self.saved_conf['data']['T_warning'] #all files must agree on T_warning due to output of truth vs. normalized shot ttd. - for mode in ['test','train']: - print('{}: loaded {} shot ({}) disruptive'.format(mode,self.get_num_shots(mode),self.get_num_disruptive_shots(mode))) + # all files must agree on T_warning due to output of truth vs. + # normalized shot ttd. + self.conf['data']['T_warning'] = self.saved_conf['data']['T_warning'] + for mode in ['test', 'train']: + print( + '{}: loaded {} shot ({}) disruptive'.format( + mode, + self.get_num_shots(mode), + self.get_num_disruptive_shots(mode))) if self.verbose: self.print_conf() - #self.assert_same_lists(self.shot_list_test,self.truth_test,self.disruptive_test) - #self.assert_same_lists(self.shot_list_train,self.truth_train,self.disruptive_train) + # self.assert_same_lists(self.shot_list_test,self.truth_test,self.disruptive_test) + # self.assert_same_lists(self.shot_list_train,self.truth_train,self.disruptive_train) - def assert_same_lists(self,shot_list,truth_arr,disr_arr): + def assert_same_lists(self, shot_list, truth_arr, disr_arr): assert(len(shot_list) == len(truth_arr)) for i in range(len(shot_list)): - shot_list.shots[i].restore("/tigress/jk7/processed_shots/") - s = shot_list.shots[i].ttd - if not truth_arr[i].shape[0] == s.shape[0]-30: - print(i) - print(shot_list.shots[i].number) - print((s.shape,truth_arr[i].shape,disr_arr[i])) - assert(truth_arr[i].shape[0] == s.shape[0]-30) + shot_list.shots[i].restore("/tigress/jk7/processed_shots/") + s = shot_list.shots[i].ttd + if not truth_arr[i].shape[0] == s.shape[0]-30: + print(i) + print(shot_list.shots[i].number) + print((s.shape, truth_arr[i].shape, disr_arr[i])) + assert(truth_arr[i].shape[0] == s.shape[0]-30) print("Same Shape!") - + def print_conf(self): - pprint(self.saved_conf) + pprint(self.saved_conf) - def get_num_shots(self,mode): + def get_num_shots(self, mode): if mode == 'test': return len(self.disruptive_test) if mode == 'train': return len(self.disruptive_train) - def get_num_disruptive_shots(self,mode): + def get_num_disruptive_shots(self, mode): if mode == 'test': return sum(self.disruptive_test) if mode == 'train': return sum(self.disruptive_train) - - def hist_alarms(self,alarms,title_str='alarms',save_figure=False,linestyle='-'): - fontsize=15 + def hist_alarms( + self, + alarms, + title_str='alarms', + save_figure=False, + linestyle='-'): + fontsize = 15 T_min_warn = self.T_min_warn T_max_warn = self.T_max_warn if len(alarms) > 0: @@ -392,40 +437,45 @@ def hist_alarms(self,alarms,title_str='alarms',save_figure=False,linestyle='-'): T_max_warn /= 1000.0 plt.figure() alarms += 0.0001 - bins=np.logspace(np.log10(min(alarms)),np.log10(max(alarms)),40) - #bins=linspace(min(alarms),max(alarms),100) - # hist(alarms,bins=bins,alpha=1.0,histtype='step',normed=True,log=False,cumulative=-1) - # - plt.step(np.concatenate((alarms[::-1], alarms[[0]])), 1.0*np.arange(alarms.size+1)/(alarms.size),linestyle=linestyle,linewidth=1.5) + # bins = np.logspace(np.log10(min(alarms)), np.log10(max(alarms)), + # 40) + + # bins=linspace(min(alarms), max(alarms), 100) + # hist(alarms, bins=bins, alpha=1.0, histtype='step', + # normed=True, log=False, cumulative=-1) + plt.step(np.concatenate((alarms[::-1], alarms[[0]])), + 1.0*np.arange(alarms.size+1)/(alarms.size), + linestyle=linestyle, linewidth=1.5) plt.gca().set_xscale('log') - plt.axvline(T_min_warn,color='r',linewidth=0.5) - #if T_max_warn < np.max(alarms): + plt.axvline(T_min_warn, color='r', linewidth=0.5) + # if T_max_warn < np.max(alarms): # plt.axvline(T_max_warn,color='r',linewidth=0.5) - plt.xlabel('Time to disruption [s]',size=fontsize) - plt.ylabel('Fraction of detected disruptions',size=fontsize) - plt.xlim([1e-4,4e1])#max(alarms)*10]) - plt.ylim([0,1]) + plt.xlabel('Time to disruption [s]', size=fontsize) + plt.ylabel('Fraction of detected disruptions', size=fontsize) + plt.xlim([1e-4, 4e1]) # max(alarms)*10]) + plt.ylim([0, 1]) plt.grid() plt.title(title_str) - plt.setp(plt.gca().get_yticklabels(),fontsize=fontsize) - plt.setp(plt.gca().get_xticklabels(),fontsize=fontsize) + plt.setp(plt.gca().get_yticklabels(), fontsize=fontsize) + plt.setp(plt.gca().get_xticklabels(), fontsize=fontsize) plt.show() if save_figure: - plt.savefig('accum_disruptions.png',dpi=200,bbox_inches='tight') + plt.savefig( + 'accum_disruptions.png', + dpi=200, + bbox_inches='tight') else: print(title_str + ": No alarms!") - - - def gather_first_alarms(self,P_thresh,mode): + def gather_first_alarms(self, P_thresh, mode): if mode == 'train': - pred_list = self.pred_train - disruptive_list = self.disruptive_train + pred_list = self.pred_train + disruptive_list = self.disruptive_train elif mode == 'test': - pred_list = self.pred_test - disruptive_list = self.disruptive_test - + pred_list = self.pred_test + disruptive_list = self.disruptive_test + alarms = [] disr_alarms = [] nondisr_alarms = [] @@ -435,8 +485,9 @@ def gather_first_alarms(self,P_thresh,mode): predictions = pred < P_thresh else: predictions = pred > P_thresh - predictions = np.reshape(predictions,(len(predictions),)) - positives = self.get_positives(predictions) #where(predictions)[0] + predictions = np.reshape(predictions, (len(predictions),)) + positives = self.get_positives( + predictions) # where(predictions)[0] if len(positives) > 0: alarm_ttd = len(pred) - 1.0 - positives[0] alarms.append(alarm_ttd) @@ -447,36 +498,45 @@ def gather_first_alarms(self,P_thresh,mode): else: if disruptive_list[i]: disr_alarms.append(-1) - return np.array(alarms),np.array(disr_alarms),np.array(nondisr_alarms) - + return np.array(alarms), np.array( + disr_alarms), np.array(nondisr_alarms) - def compute_tradeoffs_and_print(self,mode): + def compute_tradeoffs_and_print(self, mode): P_thresh_range = self.get_p_thresh_range() - correct_range, accuracy_range, fp_range,missed_range,early_alarm_range = self.get_metrics_vs_p_thresh(mode) - fp_threshs = [0.01,0.05,0.1] - missed_threshs = [0.01,0.05,0.0] - # missed_threshs = [0.01,0.05,0.1,0.2,0.3] - - #first index where... - for fp_thresh in fp_threshs: + (correct_range, accuracy_range, fp_range, missed_range, + early_alarm_range) = self.get_metrics_vs_p_thresh(mode) + fp_threshs = [0.01, 0.05, 0.1] + missed_threshs = [0.01, 0.05, 0.0] + # missed_threshs = [0.01, 0.05, 0.1, 0.2, 0.3] + + # first index where... + for fp_thresh in fp_threshs: print('============= FP RATE < {} ============='.format(fp_thresh)) if(any(fp_range < fp_thresh)): idx = np.where(fp_range <= fp_thresh)[0][0] P_thresh_opt = P_thresh_range[idx] - self.summarize_shot_prediction_stats_by_mode(P_thresh_opt,mode,verbose=True) - print('============= AT P_THRESH = {} ============='.format(P_thresh_opt)) + self.summarize_shot_prediction_stats_by_mode( + P_thresh_opt, mode, verbose=True) + print( + '============= AT P_THRESH = {} ============='.format( + P_thresh_opt)) else: - print('No such P_thresh found') + print('No such P_thresh found') print('') - #last index where - for missed_thresh in missed_threshs: - print('============= MISSED RATE < {} ============='.format(missed_thresh)) + # last index where + for missed_thresh in missed_threshs: + print( + '============= MISSED RATE < {} ============='.format( + missed_thresh)) if(any(missed_range < missed_thresh)): idx = np.where(missed_range <= missed_thresh)[0][-1] P_thresh_opt = P_thresh_range[idx] - self.summarize_shot_prediction_stats_by_mode(P_thresh_opt,mode,verbose=True) - print('============= AT P_THRESH = {} ============='.format(P_thresh_opt)) + self.summarize_shot_prediction_stats_by_mode( + P_thresh_opt, mode, verbose=True) + print( + '============= AT P_THRESH = {} ============='.format( + P_thresh_opt)) else: print('No such P_thresh found') print('') @@ -485,17 +545,18 @@ def compute_tradeoffs_and_print(self,mode): print('============= TEST PERFORMANCE: =============') idx = np.where(missed_range <= fp_range)[0][-1] P_thresh_opt = P_thresh_range[idx] - self.summarize_shot_prediction_stats_by_mode(P_thresh_opt,mode,verbose=True) + self.summarize_shot_prediction_stats_by_mode( + P_thresh_opt, mode, verbose=True) P_thresh_ret = P_thresh_opt return P_thresh_ret - def compute_tradeoffs_and_print_from_training(self): P_thresh_range = self.get_p_thresh_range() - correct_range, accuracy_range, fp_range,missed_range,early_alarm_range = self.get_metrics_vs_p_thresh('train') + (correct_range, accuracy_range, fp_range, missed_range, + early_alarm_range) = self.get_metrics_vs_p_thresh('train') - fp_threshs = [0.01,0.05,0.1] - missed_threshs = [0.01,0.05,0.0] + fp_threshs = [0.01, 0.05, 0.1] + missed_threshs = [0.01, 0.05, 0.0] # missed_threshs = [0.01,0.05,0.1,0.2,0.3] P_thresh_default = 0.03 P_thresh_ret = P_thresh_default @@ -503,33 +564,38 @@ def compute_tradeoffs_and_print_from_training(self): first_idx = 0 if not self.pred_ttd else -1 last_idx = -1 if not self.pred_ttd else 0 - #first index where... - for fp_thresh in fp_threshs: + # first index where... + for fp_thresh in fp_threshs: - print('============= TRAINING FP RATE < {} ============='.format(fp_thresh)) + print('============= TRAINING FP RATE < {} ============='.format( + fp_thresh)) print('============= TEST PERFORMANCE: =============') if(any(fp_range < fp_thresh)): idx = np.where(fp_range <= fp_thresh)[0][first_idx] P_thresh_opt = P_thresh_range[idx] - self.summarize_shot_prediction_stats_by_mode(P_thresh_opt,'test',verbose=True) - print('============= AT P_THRESH = {} ============='.format(P_thresh_opt)) + self.summarize_shot_prediction_stats_by_mode( + P_thresh_opt, 'test', verbose=True) + print('============= AT P_THRESH = {} ============='.format( + P_thresh_opt)) else: print('No such P_thresh found') P_thresh_opt = P_thresh_default print('') - #last index where - for missed_thresh in missed_threshs: - - print('============= TRAINING MISSED RATE < {} ============='.format(missed_thresh)) + # last index where + for missed_thresh in missed_threshs: + print('============= TRAINING MISSED RATE < {} ==========='.format( + missed_thresh)) print('============= TEST PERFORMANCE: =============') if(any(missed_range < missed_thresh)): idx = np.where(missed_range <= missed_thresh)[0][last_idx] P_thresh_opt = P_thresh_range[idx] - self.summarize_shot_prediction_stats_by_mode(P_thresh_opt,'test',verbose=True) + self.summarize_shot_prediction_stats_by_mode( + P_thresh_opt, 'test', verbose=True) if missed_thresh == 0.05: P_thresh_ret = P_thresh_opt - print('============= AT P_THRESH = {} ============='.format(P_thresh_opt)) + print('============= AT P_THRESH = {} ============='.format( + P_thresh_opt)) else: print('No such P_thresh found') P_thresh_opt = P_thresh_default @@ -540,20 +606,25 @@ def compute_tradeoffs_and_print_from_training(self): if(any(missed_range <= fp_range)): idx = np.where(missed_range <= fp_range)[0][last_idx] P_thresh_opt = P_thresh_range[idx] - self.summarize_shot_prediction_stats_by_mode(P_thresh_opt,'test',verbose=True) + self.summarize_shot_prediction_stats_by_mode( + P_thresh_opt, 'test', verbose=True) P_thresh_ret = P_thresh_opt - print('============= AT P_THRESH = {} ============='.format(P_thresh_opt)) + print('============= AT P_THRESH = {} ============='.format( + P_thresh_opt)) else: print('No such P_thresh found') return P_thresh_ret + def compute_tradeoffs_and_plot(self, mode, save_figure=True, + plot_string='', linestyle="-"): + (correct_range, accuracy_range, fp_range, missed_range, + early_alarm_range) = self.get_metrics_vs_p_thresh(mode) - def compute_tradeoffs_and_plot(self,mode,save_figure=True,plot_string='',linestyle="-"): - correct_range, accuracy_range, fp_range,missed_range,early_alarm_range = self.get_metrics_vs_p_thresh(mode) - - return self.tradeoff_plot(accuracy_range,missed_range,fp_range,early_alarm_range,save_figure=save_figure,plot_string=plot_string,linestyle=linestyle) + return self.tradeoff_plot(accuracy_range, missed_range, fp_range, + early_alarm_range, save_figure=save_figure, + plot_string=plot_string, linestyle=linestyle) - def get_prediction_type(self,TP,FP,FN,TN,early,late): + def get_prediction_type(self, TP, FP, FN, TN, early, late): if TP: return 'TP' elif FP: @@ -567,9 +638,14 @@ def get_prediction_type(self,TP,FP,FN,TN,early,late): elif late: return 'late' - def plot_individual_shot(self,P_thresh_opt,shot_num,normalize=True,plot_signals=True): + def plot_individual_shot( + self, + P_thresh_opt, + shot_num, + normalize=True, + plot_signals=True): success = False - for mode in ['test','train']: + for mode in ['test', 'train']: if mode == 'test': pred = self.pred_test truth = self.truth_test @@ -580,30 +656,49 @@ def plot_individual_shot(self,P_thresh_opt,shot_num,normalize=True,plot_signals= truth = self.truth_train is_disruptive = self.disruptive_train shot_list = self.shot_list_train - for i,shot in enumerate(shot_list): + for i, shot in enumerate(shot_list): if shot.number == shot_num: t = truth[i] p = pred[i] is_disr = is_disruptive[i] - TP,FP,FN,TN,early,late =self.get_shot_prediction_stats(P_thresh_opt,p,t,is_disr) - prediction_type = self.get_prediction_type(TP,FP,FN,TN,early,late) + TP, FP, FN, TN, early, late = ( + self.get_shot_prediction_stats(P_thresh_opt, p, t, + is_disr)) + prediction_type = self.get_prediction_type(TP, FP, FN, TN, + early, late) print(prediction_type) - self.plot_shot(shot,True,normalize,t,p,P_thresh_opt,prediction_type,extra_filename = '_indiv') + self.plot_shot( + shot, + True, + normalize, + t, + p, + P_thresh_opt, + prediction_type, + extra_filename='_indiv') success = True if not success: print("Shot {} not found".format(shot_num)) - - def get_prediction_type_for_individual_shot(self,P_thresh,shot,mode='test'): - p,t,is_disr = self.get_pred_truth_disr_by_shot(shot) + def get_prediction_type_for_individual_shot( + self, P_thresh, shot, mode='test'): + p, t, is_disr = self.get_pred_truth_disr_by_shot(shot) - TP,FP,FN,TN,early,late =self.get_shot_prediction_stats(P_thresh,p,t,is_disr) - prediction_type = self.get_prediction_type(TP,FP,FN,TN,early,late) + TP, FP, FN, TN, early, late = self.get_shot_prediction_stats( + P_thresh, p, t, is_disr) + prediction_type = self.get_prediction_type(TP, FP, FN, TN, early, late) return prediction_type - - def example_plots(self,P_thresh_opt,mode='test',types_to_plot = ['FP'],max_plot = 5,normalize=True,plot_signals=True,extra_filename=''): + def example_plots( + self, + P_thresh_opt, + mode='test', + types_to_plot=['FP'], + max_plot=5, + normalize=True, + plot_signals=True, + extra_filename=''): if mode == 'test': pred = self.pred_test truth = self.truth_test @@ -623,126 +718,201 @@ def example_plots(self,P_thresh_opt,mode='test',types_to_plot = ['FP'],max_plot is_disr = is_disruptive[i] shot = shot_list.shots[i] - TP,FP,FN,TN,early,late =self.get_shot_prediction_stats(P_thresh_opt,p,t,is_disr) - prediction_type = self.get_prediction_type(TP,FP,FN,TN,early,late) - if not all(_ in set(['FP','TP','FN','TN','late','early','any']) for _ in types_to_plot): + TP, FP, FN, TN, early, late = self.get_shot_prediction_stats( + P_thresh_opt, p, t, is_disr) + prediction_type = self.get_prediction_type( + TP, FP, FN, TN, early, late) + if not all(_ in set(['FP', 'TP', 'FN', 'TN', 'late', + 'early', 'any']) for _ in types_to_plot): print('warning, unkown types_to_plot') return - if ('any' in types_to_plot or prediction_type in types_to_plot) and plotted < max_plot: + if (('any' in types_to_plot or prediction_type in types_to_plot) + and plotted < max_plot): if plot_signals: - self.plot_shot(shot,True,normalize,t,p,P_thresh_opt,prediction_type,extra_filename=extra_filename) + self.plot_shot( + shot, + True, + normalize, + t, + p, + P_thresh_opt, + prediction_type, + extra_filename=extra_filename) else: plt.figure() - plt.semilogy((t+0.001)[::-1],label='ground truth') - plt.plot(p[::-1],'g',label='neural net prediction') - plt.axvline(self.T_min_warn,color='r',label='max warning time') - plt.axvline(self.T_max_warn,color='r',label='min warning time') - plt.axhline(P_thresh_opt,color='k',label='trigger threshold') + plt.semilogy((t+0.001)[::-1], label='ground truth') + plt.plot(p[::-1], 'g', label='neural net prediction') + plt.axvline( + self.T_min_warn, + color='r', + label='max warning time') + plt.axvline( + self.T_max_warn, + color='r', + label='min warning time') + plt.axhline( + P_thresh_opt, + color='k', + label='trigger threshold') plt.xlabel('TTD [ms]') - plt.legend(loc = (1.0,0.6)) - plt.ylim([1e-7,1.1e0]) + plt.legend(loc=(1.0, 0.6)) + plt.ylim([1e-7, 1.1e0]) plt.grid() - plt.savefig('fig_{}.png'.format(shot.number),bbox_inches='tight') + plt.savefig( + 'fig_{}.png'.format( + shot.number), + bbox_inches='tight') plotted += 1 - def plot_shot(self,shot,save_fig=True,normalize=True,truth=None,prediction=None,P_thresh_opt=None,prediction_type='',extra_filename=''): + def plot_shot( + self, + shot, + save_fig=True, + normalize=True, + truth=None, + prediction=None, + P_thresh_opt=None, + prediction_type='', + extra_filename=''): if self.normalizer is None and normalize: if self.conf is not None: - self.saved_conf['paths']['normalizer_path'] = self.conf['paths']['normalizer_path'] + self.saved_conf['paths']['normalizer_path'] = ( + self.conf['paths']['normalizer_path']) nn = Normalizer(self.saved_conf) nn.train() self.normalizer = nn self.normalizer.set_inference_mode(True) - + if(shot.previously_saved(self.shots_dir)): shot.restore(self.shots_dir) - if shot.signals_dict is not None: #make sure shot was saved with data - t_disrupt = shot.t_disrupt - is_disruptive = shot.is_disruptive + if shot.signals_dict is not None: + # make sure shot was saved with data + # t_disrupt = shot.t_disrupt + # is_disruptive = shot.is_disruptive if normalize: self.normalizer.apply(shot) - + use_signals = self.saved_conf['paths']['use_signals'] - fontsize= 15 - lower_lim = 0 #len(pred) + fontsize = 15 + lower_lim = 0 # len(pred) plt.close() - colors = ["b","k"] - lss = ["-","--"] - f,axarr = plt.subplots(len(use_signals)+1,1,sharex=True,figsize=(10,15))#, squeeze=False) + # colors = ["b", "k"] + # lss = ["-", "--"] + f, axarr = plt.subplots( + len(use_signals)+1, 1, sharex=True, figsize=(10, 15)) plt.title(prediction_type) assert(np.all(shot.ttd.flatten() == truth.flatten())) - xx = range(len(prediction)) #list(reversed(range(len(pred)))) - for i,sig in enumerate(use_signals): + xx = range(len(prediction)) # list(reversed(range(len(pred)))) + for i, sig in enumerate(use_signals): ax = axarr[i] num_channels = sig.num_channels sig_arr = shot.signals_dict[sig] if num_channels == 1: - # if j == 0: - ax.plot(xx,sig_arr[:,0],linewidth=2)#,linestyle=lss[j],color=colors[j]) - # else: - # ax.plot(xx,sig_arr[:,0],linewidth=2)#,linestyle=lss[j],color=colors[j],label = labels[sig]) - ax.plot([],linestyle="none",label = sig.description)#labels[sig]) - if np.min(sig_arr[:,0]) < 0: - ax.set_ylim([-6,6]) - ax.set_yticks([-5,0,5]) - # ax.plot(xx,sig_arr[:,0],linewidth=2)#,linestyle=lss[j],color=colors[j],label = labels[sig]) - ax.plot([],linestyle="none",label = sig.description)#labels[sig]) - if np.min(sig_arr[:,0]) < 0: - ax.set_ylim([-6,6]) - ax.set_yticks([-5,0,5]) + ax.plot(xx, sig_arr[:, 0], linewidth=2) + ax.plot([], linestyle="none", label=sig.description) + if np.min(sig_arr[:, 0]) < 0: + ax.set_ylim([-6, 6]) + ax.set_yticks([-5, 0, 5]) + ax.plot([], linestyle="none", label=sig.description) + if np.min(sig_arr[:, 0]) < 0: + ax.set_ylim([-6, 6]) + ax.set_yticks([-5, 0, 5]) else: - ax.set_ylim([0,8]) - ax.set_yticks([0,5]) - # ax.set_ylabel(labels[sig],size=fontsize) + ax.set_ylim([0, 8]) + ax.set_yticks([0, 5]) else: - ax.imshow(sig_arr[:,:].T, aspect='auto', label = sig.description,cmap="inferno" ) - ax.set_ylim([0,num_channels]) - ax.text(lower_lim+200, 45, sig.description, bbox={'facecolor': 'white', 'pad': 10},fontsize=fontsize-5) - ax.set_yticks([0,num_channels/2]) - ax.set_yticklabels(["0","0.5"]) - ax.set_ylabel("$\\rho$",size=fontsize) - ax.legend(loc="best",labelspacing=0.1,fontsize=fontsize,frameon=False) - ax.axvline(len(truth)-self.T_min_warn,color='r',linewidth=0.5) - plt.setp(ax.get_xticklabels(),visible=False) - plt.setp(ax.get_yticklabels(),fontsize=fontsize) + ax.imshow(sig_arr[:, :].T, aspect='auto', + label=sig.description, cmap="inferno") + ax.set_ylim([0, num_channels]) + ax.text(lower_lim+200, 45, sig.description, + bbox={'facecolor': 'white', 'pad': 10}, + fontsize=fontsize-5) + ax.set_yticks([0, num_channels/2]) + ax.set_yticklabels(["0", "0.5"]) + ax.set_ylabel("$\\rho$", size=fontsize) + ax.legend( + loc="best", + labelspacing=0.1, + fontsize=fontsize, + frameon=False) + ax.axvline( + len(truth) + - self.T_min_warn, + color='r', + linewidth=0.5) + plt.setp(ax.get_xticklabels(), visible=False) + plt.setp(ax.get_yticklabels(), fontsize=fontsize) f.subplots_adjust(hspace=0) - #print(sig) - #print('min: {}, max: {}'.format(np.min(sig_arr), np.max(sig_arr))) - ax = axarr[-1] - # ax.semilogy((-truth+0.0001),label='ground truth') - # ax.plot(-prediction+0.0001,'g',label='neural net prediction') - # ax.axhline(-P_thresh_opt,color='k',label='trigger threshold') - # nn = np.min(pred) - ax.plot(xx,truth,'g',label='target',linewidth=2) - # ax.axhline(0.4,linestyle="--",color='k',label='threshold') - ax.plot(xx,prediction,'b',label='RNN output',linewidth=2) - ax.axhline(P_thresh_opt,linestyle="--",color='k',label='threshold') - ax.set_ylim([-2,2]) - ax.set_yticks([-1,0,1]) + ax = axarr[-1] + # ax.semilogy((-truth+0.0001),label='ground truth') + # ax.plot(-prediction+0.0001,'g',label='neural net prediction') + # ax.axhline(-P_thresh_opt,color='k',label='trigger threshold') + # nn = np.min(pred) + ax.plot(xx, truth, 'g', label='target', linewidth=2) + # ax.axhline(0.4,linestyle="--",color='k',label='threshold') + ax.plot(xx, prediction, 'b', label='RNN output', linewidth=2) + ax.axhline(P_thresh_opt, linestyle="--", color='k', + label='threshold') + ax.set_ylim([-2, 2]) + ax.set_yticks([-1, 0, 1]) # if len(truth)-T_max_warn >= 0: - # ax.axvline(len(truth)-T_max_warn,color='r')#,label='max warning time') - ax.axvline(len(truth)-self.T_min_warn,color='r',linewidth=0.5)#,label='min warning time') - ax.set_xlabel('T [ms]',size=fontsize) + # ax.axvline(len(truth)-T_max_warn,color='r')#,label='max + # warning time') + # ,label='min warning time') + ax.axvline( + len(truth) + - self.T_min_warn, + color='r', + linewidth=0.5) + ax.set_xlabel('T [ms]', size=fontsize) # ax.axvline(2400) - ax.legend(loc = (0.5,0.7),fontsize=fontsize-5,labelspacing=0.1,frameon=False) - plt.setp(ax.get_yticklabels(),fontsize=fontsize) - plt.setp(ax.get_xticklabels(),fontsize=fontsize) + ax.legend( + loc=( + 0.5, + 0.7), + fontsize=fontsize-5, + labelspacing=0.1, + frameon=False) + plt.setp(ax.get_yticklabels(), fontsize=fontsize) + plt.setp(ax.get_xticklabels(), fontsize=fontsize) # plt.xlim(0,200) - plt.xlim([lower_lim,len(truth)]) + plt.xlim([lower_lim, len(truth)]) # plt.savefig("{}.png".format(num),dpi=200,bbox_inches="tight") if save_fig: - plt.savefig('sig_fig_{}{}.png'.format(shot.number,extra_filename),bbox_inches='tight') - np.savez('sig_{}{}.npz'.format(shot.number,extra_filename),shot=shot,T_min_warn=self.T_min_warn,T_max_warn=self.T_max_warn,prediction=prediction,truth=truth,use_signals=use_signals,P_thresh=P_thresh_opt) - #plt.show() + plt.savefig( + 'sig_fig_{}{}.png'.format( + shot.number, + extra_filename), + bbox_inches='tight') + np.savez( + 'sig_{}{}.npz'.format( + shot.number, + extra_filename), + shot=shot, + T_min_warn=self.T_min_warn, + T_max_warn=self.T_max_warn, + prediction=prediction, + truth=truth, + use_signals=use_signals, + P_thresh=P_thresh_opt) + # plt.show() else: print("Shot hasn't been processed") - - - def plot_shot_old(self,shot,save_fig=True,normalize=True,truth=None,prediction=None,P_thresh_opt=None,prediction_type='',extra_filename=''): + def plot_shot_old( + self, + shot, + save_fig=True, + normalize=True, + truth=None, + prediction=None, + P_thresh_opt=None, + prediction_type='', + extra_filename=''): if self.normalizer is None and normalize: if self.conf is not None: - self.saved_conf['paths']['normalizer_path'] = self.conf['paths']['normalizer_path'] + self.saved_conf['paths']['normalizer_path'] = ( + self.conf['paths']['normalizer_path']) nn = Normalizer(self.saved_conf) nn.train() self.normalizer = nn @@ -750,100 +920,155 @@ def plot_shot_old(self,shot,save_fig=True,normalize=True,truth=None,prediction=N if(shot.previously_saved(self.shots_dir)): shot.restore(self.shots_dir) - t_disrupt = shot.t_disrupt - is_disruptive = shot.is_disruptive + # t_disrupt = shot.t_disrupt + # is_disruptive = shot.is_disruptive if normalize: self.normalizer.apply(shot) use_signals = self.saved_conf['paths']['use_signals'] - f,axarr = plt.subplots(len(use_signals)+1,1,sharex=True,figsize=(13,13))#, squeeze=False) + f, axarr = plt.subplots(len(use_signals)+1, 1, sharex=True, + figsize=(13, 13)) plt.title(prediction_type) - #all files must agree on T_warning due to output of truth vs. normalized shot ttd. + # all files must agree on T_warning due to output of truth vs. + # normalized shot ttd. assert(np.all(shot.ttd.flatten() == truth.flatten())) - for i,sig in enumerate(use_signals): + for i, sig in enumerate(use_signals): num_channels = sig.num_channels ax = axarr[i] sig_arr = shot.signals_dict[sig] if num_channels == 1: - ax.plot(sig_arr[:,0],label = sig.description) + ax.plot(sig_arr[:, 0], label=sig.description) else: - ax.imshow(sig_arr[:,:].T, aspect='auto', label = sig.description + " (profile)") - ax.set_ylim([0,num_channels]) - ax.legend(loc='best',fontsize=8) - plt.setp(ax.get_xticklabels(),visible=False) - plt.setp(ax.get_yticklabels(),fontsize=7) + ax.imshow(sig_arr[:, :].T, aspect='auto', + label=sig.description + " (profile)") + ax.set_ylim([0, num_channels]) + ax.legend(loc='best', fontsize=8) + plt.setp(ax.get_xticklabels(), visible=False) + plt.setp(ax.get_yticklabels(), fontsize=7) f.subplots_adjust(hspace=0) - #print(sig) - #print('min: {}, max: {}'.format(np.min(sig_arr), np.max(sig_arr))) - ax = axarr[-1] + # print(sig) + # print('min: {}, max: {}'.format(np.min(sig_arr), + # np.max(sig_arr))) + ax = axarr[-1] if self.pred_ttd: - ax.semilogy((-truth+0.0001),label='ground truth') - ax.plot(-prediction+0.0001,'g',label='neural net prediction') - ax.axhline(-P_thresh_opt,color='k',label='trigger threshold') + ax.semilogy((-truth+0.0001), label='ground truth') + ax.plot(-prediction+0.0001, 'g', label='neural net prediction') + ax.axhline(-P_thresh_opt, color='k', label='trigger threshold') else: - ax.plot((truth+0.001),label='ground truth') - ax.plot(prediction,'g',label='neural net prediction') - ax.axhline(P_thresh_opt,color='k',label='trigger threshold') - #ax.set_ylim([1e-5,1.1e0]) - ax.set_ylim([-2,2]) + ax.plot((truth+0.001), label='ground truth') + ax.plot(prediction, 'g', label='neural net prediction') + ax.axhline(P_thresh_opt, color='k', label='trigger threshold') + # ax.set_ylim([1e-5,1.1e0]) + ax.set_ylim([-2, 2]) if len(truth)-self.T_max_warn >= 0: - ax.axvline(len(truth)-self.T_max_warn,color='r',label='min warning time') - ax.axvline(len(truth)-self.T_min_warn,color='r',label='max warning time') + ax.axvline( + len(truth)-self.T_max_warn, + color='r', + label='min warning time') + ax.axvline( + len(truth) + - self.T_min_warn, + color='r', + label='max warning time') ax.set_xlabel('T [ms]') - #ax.legend(loc = 'lower left',fontsize=10) - plt.setp(ax.get_yticklabels(),fontsize=7) - # ax.grid() + # ax.legend(loc = 'lower left',fontsize=10) + plt.setp(ax.get_yticklabels(), fontsize=7) + # ax.grid() if save_fig: - plt.savefig('sig_fig_{}{}.png'.format(shot.number,extra_filename),bbox_inches='tight') - np.savez('sig_{}{}.npz'.format(shot.number,extra_filename),shot=shot,T_min_warn=self.T_min_warn,T_max_warn=self.T_max_warn,prediction=prediction,truth=truth,use_signals=use_signals,P_thresh=P_thresh_opt) + plt.savefig( + 'sig_fig_{}{}.png'.format( + shot.number, + extra_filename), + bbox_inches='tight') + np.savez( + 'sig_{}{}.npz'.format( + shot.number, + extra_filename), + shot=shot, + T_min_warn=self.T_min_warn, + T_max_warn=self.T_max_warn, + prediction=prediction, + truth=truth, + use_signals=use_signals, + P_thresh=P_thresh_opt) plt.close() else: print("Shot hasn't been processed") - - def tradeoff_plot(self,accuracy_range,missed_range,fp_range,early_alarm_range,save_figure=False,plot_string='',linestyle="-"): - fontsize=15 + def tradeoff_plot( + self, + accuracy_range, + missed_range, + fp_range, + early_alarm_range, + save_figure=False, + plot_string='', + linestyle="-"): + fontsize = 15 plt.figure() P_thresh_range = self.get_p_thresh_range() # semilogx(P_thresh_range,accuracy_range,label="accuracy") if self.pred_ttd: - plt.semilogx(abs(P_thresh_range[::-1]),missed_range,'r',label="missed",linestyle=linestyle) - plt.plot(abs(P_thresh_range[::-1]),fp_range,'k',label="false positives",linestyle=linestyle) + plt.semilogx(abs(P_thresh_range[::-1]), + missed_range, + 'r', + label="missed", + linestyle=linestyle) + plt.plot(abs(P_thresh_range[::-1]), + fp_range, + 'k', + label="false positives", + linestyle=linestyle) else: - plt.plot(P_thresh_range,missed_range,'r',label="missed",linestyle=linestyle) - plt.plot(P_thresh_range,fp_range,'k',label="false positives",linestyle=linestyle) + plt.plot( + P_thresh_range, + missed_range, + 'r', + label="missed", + linestyle=linestyle) + plt.plot( + P_thresh_range, + fp_range, + 'k', + label="false positives", + linestyle=linestyle) # plot(P_thresh_range,early_alarm_range,'c',label="early alarms") - plt.legend(loc=(1.0,.6)) - plt.xlabel('Alarm threshold',size=fontsize) + plt.legend(loc=(1.0, .6)) + plt.xlabel('Alarm threshold', size=fontsize) plt.grid() - title_str = 'metrics{}'.format(plot_string.replace('_',' ')) + title_str = 'metrics{}'.format(plot_string.replace('_', ' ')) plt.title(title_str) if save_figure: - plt.savefig(title_str + '.png',bbox_inches='tight') + plt.savefig(title_str + '.png', bbox_inches='tight') plt.close('all') - plt.plot(fp_range,1-missed_range,'-b',linestyle=linestyle) + plt.plot(fp_range, 1-missed_range, '-b', linestyle=linestyle) ax = plt.gca() - plt.xlabel('FP rate',size=fontsize) - plt.ylabel('TP rate',size=fontsize) - major_ticks = np.arange(0,1.01,0.2) - minor_ticks = np.arange(0,1.01,0.05) + plt.xlabel('FP rate', size=fontsize) + plt.ylabel('TP rate', size=fontsize) + major_ticks = np.arange(0, 1.01, 0.2) + minor_ticks = np.arange(0, 1.01, 0.05) ax.set_xticks(major_ticks) ax.set_yticks(major_ticks) - ax.set_xticks(minor_ticks,minor=True) - ax.set_yticks(minor_ticks,minor=True) - plt.setp(plt.gca().get_yticklabels(),fontsize=fontsize) - plt.setp(plt.gca().get_xticklabels(),fontsize=fontsize) + ax.set_xticks(minor_ticks, minor=True) + ax.set_yticks(minor_ticks, minor=True) + plt.setp(plt.gca().get_yticklabels(), fontsize=fontsize) + plt.setp(plt.gca().get_xticklabels(), fontsize=fontsize) ax.grid(which='both') - ax.grid(which='major',alpha=0.5) - ax.grid(which='minor',alpha=0.3) - plt.xlim([0,1]) - plt.ylim([0,1]) + ax.grid(which='major', alpha=0.5) + ax.grid(which='minor', alpha=0.3) + plt.xlim([0, 1]) + plt.ylim([0, 1]) if save_figure: - plt.savefig(title_str + '_roc.png',bbox_inches='tight',dpi=200) - print('ROC area ({}) is {}'.format(plot_string,self.roc_from_missed_fp(missed_range,fp_range))) - return P_thresh_range,missed_range,fp_range - - def get_pred_truth_disr_by_shot(self,shot): + plt.savefig(title_str + '_roc.png', bbox_inches='tight', dpi=200) + print( + 'ROC area ({}) is {}'.format( + plot_string, + self.roc_from_missed_fp( + missed_range, + fp_range))) + return P_thresh_range, missed_range, fp_range + + def get_pred_truth_disr_by_shot(self, shot): if shot in self.shot_list_test: mode = 'test' elif shot in self.shot_list_train: @@ -866,50 +1091,49 @@ def get_pred_truth_disr_by_shot(self,shot): p = pred[i] is_disr = is_disruptive[i] shot = shot_list.shots[i] - return p,t,is_disr + return p, t, is_disr - def save_shot(self,shot,P_thresh_opt = 0,extra_filename=''): + def save_shot(self, shot, P_thresh_opt=0, extra_filename=''): if self.normalizer is None: if self.conf is not None: - self.saved_conf['paths']['normalizer_path'] = self.conf['paths']['normalizer_path'] + self.saved_conf['paths']['normalizer_path'] = ( + self.conf['paths']['normalizer_path']) nn = Normalizer(self.saved_conf) nn.train() self.normalizer = nn self.normalizer.set_inference_mode(True) - + shot.restore(self.shots_dir) - t_disrupt = shot.t_disrupt - is_disruptive = shot.is_disruptive + # t_disrupt = shot.t_disrupt + # is_disruptive = shot.is_disruptive self.normalizer.apply(shot) - - - pred,truth,is_disr = self.get_pred_truth_disr_by_shot(shot) + pred, truth, is_disr = self.get_pred_truth_disr_by_shot(shot) use_signals = self.saved_conf['paths']['use_signals'] - np.savez('sig_{}{}.npz'.format(shot.number,extra_filename),shot=shot,T_min_warn=self.T_min_warn,T_max_warn=self.T_max_warn,prediction=pred,truth=truth,use_signals=use_signals,P_thresh=P_thresh_opt) + np.savez('sig_{}{}.npz'.format(shot.number, extra_filename), + shot=shot, T_min_warn=self.T_min_warn, + T_max_warn=self.T_max_warn, prediction=pred, + truth=truth, use_signals=use_signals, P_thresh=P_thresh_opt) - def get_roc_area_by_mode(self,mode='test'): + def get_roc_area_by_mode(self, mode='test'): if mode == 'test': pred = self.pred_test truth = self.truth_test is_disruptive = self.disruptive_test - shot_list = self.shot_list_test + # shot_list = self.shot_list_test else: pred = self.pred_train truth = self.truth_train is_disruptive = self.disruptive_train - shot_list = self.shot_list_train - return self.get_roc_area(pred,truth,is_disruptive) - - def get_roc_area(self,all_preds,all_truths,all_disruptive): - correct_range, accuracy_range, fp_range,missed_range,early_alarm_range = \ - self.get_metrics_vs_p_thresh_custom(all_preds,all_truths,all_disruptive) - - return self.roc_from_missed_fp(missed_range,fp_range) + # shot_list = self.shot_list_train + return self.get_roc_area(pred, truth, is_disruptive) - def roc_from_missed_fp(self,missed_range,fp_range): - #print(fp_range) - #print(missed_range) - return -np.trapz(1-missed_range,x=fp_range) + def get_roc_area(self, all_preds, all_truths, all_disruptive): + (correct_range, accuracy_range, fp_range, missed_range, + early_alarm_range) = self.get_metrics_vs_p_thresh_custom( + all_preds, all_truths, all_disruptive) + return self.roc_from_missed_fp(missed_range, fp_range) + def roc_from_missed_fp(self, missed_range, fp_range): + return -np.trapz(1 - missed_range, x=fp_range) diff --git a/plasma/utils/processing.py b/plasma/utils/processing.py index 7f8ab81f..d6995bcb 100644 --- a/plasma/utils/processing.py +++ b/plasma/utils/processing.py @@ -12,26 +12,29 @@ import itertools import numpy as np -from scipy.interpolate import UnivariateSpline -import sys - -#interpolate in a way that doesn't use future information. -#It simply finds the latest time point in the original array -#that is less than or equal than the time point in question -#and interpolates to there. -def time_sensitive_interp(x,t,t_new): - indices = np.maximum(0,np.searchsorted(t,t_new,side='right')-1) +# from scipy.interpolate import UnivariateSpline + +# interpolate in a way that doesn't use future information. +# It simply finds the latest time point in the original array +# that is less than or equal than the time point in question +# and interpolates to there. + + +def time_sensitive_interp(x, t, t_new): + indices = np.maximum(0, np.searchsorted(t, t_new, side='right')-1) return x[indices] -def resample_signal(t,sig,tmin,tmax,dt,precision_str='float32'): + +def resample_signal(t, sig, tmin, tmax, dt, precision_str='float32'): order = np.argsort(t) t = t[order] - sig = sig[order,:] + sig = sig[order, :] sig_width = sig.shape[1] - tt = np.arange(tmin,tmax,dt,dtype=precision_str) - sig_interp = np.zeros((len(tt),sig_width),dtype=precision_str) + tt = np.arange(tmin, tmax, dt, dtype=precision_str) + sig_interp = np.zeros((len(tt), sig_width), dtype=precision_str) for i in range(sig_width): - sig_interp[:,i] = time_sensitive_interp(sig[:,i],t,tt) #make sure to not use future information + # make sure to not use future information + sig_interp[:, i] = time_sensitive_interp(sig[:, i], t, tt) # f = UnivariateSpline(t,sig[:,i],s=0,k=1,ext=0) # sig_interp[:,i] = f(tt) if(np.any(np.isnan(sig_interp))): @@ -41,59 +44,67 @@ def resample_signal(t,sig,tmin,tmax,dt,precision_str='float32'): idx = np.where(t[1:] - t[:-1] <= 0)[0][0] print(t[idx-10:idx+10]) - return tt,sig_interp + return tt, sig_interp + -def cut_signal(t,sig,tmin,tmax): +def cut_signal(t, sig, tmin, tmax): mask = np.logical_and(t >= tmin, t <= tmax) - return t[mask],sig[mask,:] + return t[mask], sig[mask, :] + -def cut_and_resample_signal(t,sig,tmin,tmax,dt,precision_str): - t,sig = cut_signal(t,sig,tmin,tmax) - return resample_signal(t,sig,tmin,tmax,dt,precision_str) +def cut_and_resample_signal(t, sig, tmin, tmax, dt, precision_str): + t, sig = cut_signal(t, sig, tmin, tmax) + return resample_signal(t, sig, tmin, tmax, dt, precision_str) -def get_individual_shot_file(prepath,shot_num,ext='.txt'): - return prepath + str(shot_num) + ext -def append_to_filename(path,to_append): +def get_individual_shot_file(prepath, shot_num, ext='.txt'): + return prepath + str(shot_num) + ext + + +def append_to_filename(path, to_append): ending_idx = path.rfind('.') new_path = path[:ending_idx] + to_append + path[ending_idx:] return new_path -def train_test_split(x,frac,do_shuffle=False): - if not isinstance(x,np.ndarray): - return train_test_split_robust(x,frac,do_shuffle) + +def train_test_split(x, frac, do_shuffle=False): + if not isinstance(x, np.ndarray): + return train_test_split_robust(x, frac, do_shuffle) mask = np.array(range(len(x))) < frac*len(x) if do_shuffle: np.random.shuffle(mask) - return x[mask],x[~mask] + return x[mask], x[~mask] -def train_test_split_robust(x,frac,do_shuffle=False): + +def train_test_split_robust(x, frac, do_shuffle=False): mask = np.array(range(len(x))) < frac*len(x) if do_shuffle: np.random.shuffle(mask) train = [] test = [] - for (i,_x) in enumerate(x): + for (i, _x) in enumerate(x): if mask[i]: train.append(_x) else: test.append(_x) - return train,test + return train, test + -def train_test_split_all(x,frac,do_shuffle=True): +def train_test_split_all(x, frac, do_shuffle=True): groups = [] length = len(x[0]) mask = np.array(range(length)) < frac*length if do_shuffle: np.random.shuffle(mask) for item in x: - groups.append((item[mask],item[~mask])) + groups.append((item[mask], item[~mask])) return groups def concatenate_sublists(superlist): return list(itertools.chain.from_iterable(superlist)) + def get_signal_slices(signals_superlist): indices_superlist = [] signals_so_far = 0 diff --git a/plasma/utils/state_reset.py b/plasma/utils/state_reset.py index 3d872447..172023bf 100644 --- a/plasma/utils/state_reset.py +++ b/plasma/utils/state_reset.py @@ -1,32 +1,35 @@ from __future__ import print_function import keras.backend as K -import numpy as np + def get_states(model): - all_states = [] - for layer in model.layers: - if hasattr(layer,"states"): - layer_states = [] - for state in layer.states: - #print(K.get_value(state)[0][0:3]) - layer_states.append(K.get_value(state)) - all_states.append(layer_states) - # print(all_states) - return all_states + all_states = [] + for layer in model.layers: + if hasattr(layer, "states"): + layer_states = [] + for state in layer.states: + # print(K.get_value(state)[0][0:3]) + layer_states.append(K.get_value(state)) + all_states.append(layer_states) + # print(all_states) + return all_states + def set_states(model, all_states): - i = 0 - for layer in model.layers: - if hasattr(layer,"states"): - layer.reset_states(all_states[i]) - i += 1 + i = 0 + for layer in model.layers: + if hasattr(layer, "states"): + layer.reset_states(all_states[i]) + i += 1 + def reset_states(model, batches_to_reset): - old_states = get_states(model) - model.reset_states() - new_states = get_states(model) - for i,layer_states in enumerate(new_states): - for j,within_layer_state in enumerate(layer_states): - assert(len(batches_to_reset) == within_layer_state.shape[0]) - within_layer_state[~batches_to_reset,:] = old_states[i][j][~batches_to_reset,:] - set_states(model,new_states) + old_states = get_states(model) + model.reset_states() + new_states = get_states(model) + for i, layer_states in enumerate(new_states): + for j, within_layer_state in enumerate(layer_states): + assert(len(batches_to_reset) == within_layer_state.shape[0]) + within_layer_state[~batches_to_reset, + :] = old_states[i][j][~batches_to_reset, :] + set_states(model, new_states) From 492f52f598541eac43090a97f7d310606b869be9 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 24 Sep 2019 16:29:16 -0500 Subject: [PATCH 093/272] Fix remaining flake8 warnings and errors --- examples/mpi_augment_learn.py | 2 +- examples/simple_augmentation.py | 17 +++++++++-------- plasma/__init__.py | 22 +++++++++++----------- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/examples/mpi_augment_learn.py b/examples/mpi_augment_learn.py index 727d559b..d826bdfd 100644 --- a/examples/mpi_augment_learn.py +++ b/examples/mpi_augment_learn.py @@ -1,6 +1,6 @@ from plasma.models.mpi_runner import ( mpi_train, mpi_make_predictions_and_evaluate - ) +) from mpi4py import MPI from plasma.preprocessor.preprocess import guarantee_preprocessed from plasma.preprocessor.augment import Augmentator diff --git a/examples/simple_augmentation.py b/examples/simple_augmentation.py index 8be07b85..ea89e42f 100644 --- a/examples/simple_augmentation.py +++ b/examples/simple_augmentation.py @@ -1,6 +1,6 @@ from plasma.models.mpi_runner import ( - mpi_make_predictions_and_evaluate - ) + mpi_make_predictions, mpi_make_predictions_and_evaluate +) from mpi4py import MPI from plasma.preprocessor.preprocess import guarantee_preprocessed from plasma.preprocessor.augment import ByShotAugmentator @@ -115,7 +115,7 @@ def create_shot_list_tmp(original_shot, time_points, sigs=None): assert(new_shot.augmentation_fn is None) new_shot.augmentation_fn = partial( hide_signal_data, t=t, sigs_to_hide=sigs) - #new_shot.number = original_shot.number + # new_shot.number = original_shot.number shot_list_tmp.append(new_shot) return shot_list_tmp, t_range @@ -126,7 +126,7 @@ def get_importance_measure( custom_path, metric, time_points=10, - sig=None): + sigs=None): shot_list_tmp, t_range = create_shot_list_tmp( original_shot, time_points, sigs) y_prime, y_gold, disruptive = mpi_make_predictions( @@ -138,8 +138,8 @@ def get_importance_measure( def difference_metric(y_prime, y_prime_orig): idx = np.argmax(y_prime_orig) - return (np.max(y_prime_orig) - y_prime[idx]) / \ - (np.max(y_prime_orig) - np.min(y_prime_orig)) + return ((np.max(y_prime_orig) - y_prime[idx]) + / (np.max(y_prime_orig) - np.min(y_prime_orig))) def get_importance_measure_given_y_prime(y_prime, metric): @@ -176,8 +176,9 @@ def get_importance_measure_given_y_prime(y_prime, metric): # for sigs_to_hide in [[s] for s in use_signals[:-3]] + # [use_signals[-3:-1]] + [use_signals[-1]]: -for sigs_to_hide in [[s] for s in use_signals[:-3]] + [[s] - for s in use_signals[-3:-1]] + [use_signals[-3:-1]]: # + [use_signals[-1]]: +for sigs_to_hide in ([[s] for s in use_signals[:-3]] + + [[s] for s in use_signals[-3:-1]] + + [use_signals[-3:-1]]): for shot in shot_list_test: shot.augmentation_fn = partial( hide_signal_data, t=0, sigs_to_hide=sigs_to_hide) diff --git a/plasma/__init__.py b/plasma/__init__.py index 23d79fd9..0e21193f 100644 --- a/plasma/__init__.py +++ b/plasma/__init__.py @@ -1,15 +1,15 @@ -#from plasma.conf import * -#from plasma.jet_signals import * +# from plasma.conf import * +# from plasma.jet_signals import * -#from plasma.model.builder import * -#from plasma.model.runner import * -#from plasma.model.targets import * +# from plasma.model.builder import * +# from plasma.model.runner import * +# from plasma.model.targets import * -#from plasma.preprocessor.load import * -#from plasma.preprocessor.normalize import * -#from plasma.preprocessor.preprocess import * +# from plasma.preprocessor.load import * +# from plasma.preprocessor.normalize import * +# from plasma.preprocessor.preprocess import * -#from plasma.primitives.shots import * +# from plasma.primitives.shots import * -#from plasma.utils.preprocessing import * -#from plasma.utils.performance_analysis_utils import * +# from plasma.utils.preprocessing import * +# from plasma.utils.performance_analysis_utils import * From 946449be56ad368acde807a9b2c790b93c002f3f Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 26 Sep 2019 09:56:17 -0500 Subject: [PATCH 094/272] Add PEP 8 style violation to test Travis CI build message --- plasma/preprocessor/preprocess.py | 1 - 1 file changed, 1 deletion(-) diff --git a/plasma/preprocessor/preprocess.py b/plasma/preprocessor/preprocess.py index 17a5699f..4b0e0b69 100644 --- a/plasma/preprocessor/preprocess.py +++ b/plasma/preprocessor/preprocess.py @@ -21,7 +21,6 @@ from plasma.primitives.shots import ShotList from plasma.utils.downloading import mkdirdepth - class Preprocessor(object): def __init__(self, conf): From 0c2dbd6154ed6da0560a03529bbab1e4affe1530 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 26 Sep 2019 10:10:01 -0500 Subject: [PATCH 095/272] Try fixing missing Slack notifications from Travis CI Also add diagnostic/progress message for flake8 that is only posted when flake8 returns 0 (no style warnings or errors) --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a983ca04..d16ec7c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ matrix: - stage: python linter env: install: pip install flake8 - script: python -m flake8 + script: python -m flake8 && echo "Finished linting Python files with flake8" addons: apt: @@ -55,6 +55,7 @@ notifications: on_success: change on_failure: always slack: - secure: NBjGYpIF2VO/GvhbC7XVPIfi0WLGFEuVhi51UbZjfHg4IOv1UuCF0fImi8GN2UZZBFRnZcbtPVffZnyMUyJI0Krw0M1ropkjA4YDaJx1JUGnYuQLxNmX+jbQmN61Usjg5MQOgcRnAn1bdMN1ttWInqkKejpV5buCjZbt8SZbDePfXz4U3No68P/pRsDTVXSy0xLTtRacuEITJjwxfjp6phbmR3qs127MZMRbVYDC2HA6KsoJW6YSKF1vFyHqnFMl7GSavxYw/XQpqFLJkGKXfnNgPZV6qAbVk5+bzyytAbbwLGvgbuFpnJsGPvAyebV8wVYaIPg7OeK+Sm1A3q0jt774NnqFp2AZ+pSKrxbxIkygDM0zoWLm4i3pt6ToJ6dKcqSbCKbELnSY5NphcyYuiJ8uhLVpAR0Y+vp+fOvhb8td/nH2AWkxFpp6xwOHPXvtodBsyPMkiaeKoVElYBfbfOhDSYH2KafADECTX75S63A9KleeNZh0DSImfFdQPaN3GFLEL8Z9UFABzTkM9eYKP9pyEP82Wh/JGDaqbARazEgNzy8rwghomsEguV247XHdOx32PSd0att541gsRFZ5uyMuVFKzI0jiLijekibY2I1c5b6dDeuK4O8uBiq4FTS+bM55Rj4Job01kdxSCKw3RwOh0amzITRTQvEWKpTAekg= + rooms: + - secure: "NBjGYpIF2VO/GvhbC7XVPIfi0WLGFEuVhi51UbZjfHg4IOv1UuCF0fImi8GN2UZZBFRnZcbtPVffZnyMUyJI0Krw0M1ropkjA4YDaJx1JUGnYuQLxNmX+jbQmN61Usjg5MQOgcRnAn1bdMN1ttWInqkKejpV5buCjZbt8SZbDePfXz4U3No68P/pRsDTVXSy0xLTtRacuEITJjwxfjp6phbmR3qs127MZMRbVYDC2HA6KsoJW6YSKF1vFyHqnFMl7GSavxYw/XQpqFLJkGKXfnNgPZV6qAbVk5+bzyytAbbwLGvgbuFpnJsGPvAyebV8wVYaIPg7OeK+Sm1A3q0jt774NnqFp2AZ+pSKrxbxIkygDM0zoWLm4i3pt6ToJ6dKcqSbCKbELnSY5NphcyYuiJ8uhLVpAR0Y+vp+fOvhb8td/nH2AWkxFpp6xwOHPXvtodBsyPMkiaeKoVElYBfbfOhDSYH2KafADECTX75S63A9KleeNZh0DSImfFdQPaN3GFLEL8Z9UFABzTkM9eYKP9pyEP82Wh/JGDaqbARazEgNzy8rwghomsEguV247XHdOx32PSd0att541gsRFZ5uyMuVFKzI0jiLijekibY2I1c5b6dDeuK4O8uBiq4FTS+bM55Rj4Job01kdxSCKw3RwOh0amzITRTQvEWKpTAekg=" on_success: always on_failure: always From 038012d96319cb15060add584b1e7e50c1b14c32 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 26 Sep 2019 10:14:20 -0500 Subject: [PATCH 096/272] Travis CI app in Slack is working again--- fix intentional style error --- plasma/preprocessor/preprocess.py | 1 + 1 file changed, 1 insertion(+) diff --git a/plasma/preprocessor/preprocess.py b/plasma/preprocessor/preprocess.py index 4b0e0b69..17a5699f 100644 --- a/plasma/preprocessor/preprocess.py +++ b/plasma/preprocessor/preprocess.py @@ -21,6 +21,7 @@ from plasma.primitives.shots import ShotList from plasma.utils.downloading import mkdirdepth + class Preprocessor(object): def __init__(self, conf): From a00bc965230d72e3bde5763f115e5b75d49639ec Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 26 Sep 2019 10:15:26 -0500 Subject: [PATCH 097/272] Update Travis CI shield on README.md Migrated repo from open-source travis-ci.org to unified travis-ci.com via Travis CI's beta migration tool. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5ec11e38..8b8a9eca 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # FRNN -[![Build Status](https://travis-ci.org/PPPLDeepLearning/plasma-python.svg?branch=master)](https://travis-ci.org/PPPLDeepLearning/plasma-python) +[![Build Status](https://travis-ci.com/PPPLDeepLearning/plasma-python.svg?branch=master)](https://travis-ci.com/PPPLDeepLearning/plasma-python) [![Build Status](https://jenkins.princeton.edu/buildStatus/icon?job=FRNM/PPPL)](https://jenkins.princeton.edu/job/FRNM/job/PPPL/) ## Package description From eed497597e8d4b3b34c2ac1cc950f35872f7fe44 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 3 Oct 2019 17:17:27 -0500 Subject: [PATCH 098/272] Address PyYAML deprecation warning with yaml.load() default See https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation about the CVE about RCE (similar to Pickle) --- examples/submit_batch_job.py | 4 ++-- examples/tune_hyperparams.py | 2 +- jenkins-ci/run_jenkins.py | 2 +- plasma/conf.py | 1 + plasma/conf_parser.py | 2 +- plasma/primitives/hyperparameters.py | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/submit_batch_job.py b/examples/submit_batch_job.py index facc2214..789b6751 100644 --- a/examples/submit_batch_job.py +++ b/examples/submit_batch_job.py @@ -31,7 +31,7 @@ def copy_conf_file( pathdst = os.path.join(save_path, conf_name) with open(pathsrc, 'r') as yaml_file: - conf = yaml.load(yaml_file) + conf = yaml.load(yaml_file, Loader=yaml.SafeLoader) # make sure all files like checkpoints and normalization are done locally conf['training']['hyperparam_tuning'] = True with open(pathdst, 'w') as outfile: @@ -40,7 +40,7 @@ def copy_conf_file( def get_conf(template_path, conf_name): with open(os.path.join(template_path, conf_name), 'r') as yaml_file: - conf = yaml.load(yaml_file) + conf = yaml.load(yaml_file, Loader=yaml.SafeLoader) return conf diff --git a/examples/tune_hyperparams.py b/examples/tune_hyperparams.py index 6d8f5d8f..7c6d7d34 100644 --- a/examples/tune_hyperparams.py +++ b/examples/tune_hyperparams.py @@ -108,7 +108,7 @@ def generate_conf_file( conf_name="conf.yaml"): assert(template_path != save_path) with open(os.path.join(template_path, conf_name), 'r') as yaml_file: - conf = yaml.load(yaml_file) + conf = yaml.load(yaml_file, Loader=yaml.SafeLoader) for tunable in tunables: tunable.assign_to_conf(conf, save_path) # rely on early stopping to terminate training diff --git a/jenkins-ci/run_jenkins.py b/jenkins-ci/run_jenkins.py index 282b3900..b5ae4714 100644 --- a/jenkins-ci/run_jenkins.py +++ b/jenkins-ci/run_jenkins.py @@ -23,7 +23,7 @@ def generate_conf_file( conf_name="conf.yaml"): assert(template_path != save_path) with open(os.path.join(template_path, conf_name), 'r') as yaml_file: - conf = yaml.load(yaml_file) + conf = yaml.load(yaml_file, Loader=yaml.SafeLoader) conf['training']['num_epochs'] = 2 conf['paths']['data'] = test_configuration[1] if test_configuration[1] == "Python3": diff --git a/plasma/conf.py b/plasma/conf.py index 84b6ce49..69aafd88 100644 --- a/plasma/conf.py +++ b/plasma/conf.py @@ -2,6 +2,7 @@ import os import errno +# TODO(KGF): this conf.py feels like an unnecessary level of indirection if os.path.exists(os.path.join(os.path.abspath(os.path.dirname(__file__)), '../examples/conf.yaml')): conf = parameters(os.path.join(os.path.abspath(os.path.dirname(__file__)), diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index 050d48cc..f94068ae 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -16,7 +16,7 @@ def parameters(input_file): TTDTarget, TTDInvTarget, TTDLinearTarget ) with open(input_file, 'r') as yaml_file: - params = yaml.load(yaml_file) + params = yaml.load(yaml_file, Loader=yaml.SafeLoader) params['user_name'] = getpass.getuser() output_path = params['fs_path'] + "/" + params['user_name'] diff --git a/plasma/primitives/hyperparameters.py b/plasma/primitives/hyperparameters.py index 9bed2a78..10808654 100644 --- a/plasma/primitives/hyperparameters.py +++ b/plasma/primitives/hyperparameters.py @@ -105,7 +105,7 @@ def __init__(self, path, conf_name="conf.yaml"): self.raw_logs_path = path[:-1] + ".out" self.changed_path = os.path.join(path, "changed_params.out") with open(os.path.join(self.path, conf_name), 'r') as yaml_file: - conf = yaml.load(yaml_file) + conf = yaml.load(yaml_file, Loader=yaml.SafeLoader) self.name_to_monitor = conf['callbacks']['monitor'] self.load_data() self.get_changed() From 6563d669ec5c8626c0a0587857361b104e94b977 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 3 Oct 2019 17:19:56 -0500 Subject: [PATCH 099/272] Ignore 2x more types of local Emacs config files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index d7ce66f1..139f148f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ # Emacs *~ \#*\# +.auctex-auto/ +.dir-locals.el # Generated by test plot_*.html From 1b88161c2b76d119aa50ef74538c0846a64ae1d4 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Fri, 4 Oct 2019 16:19:17 -0500 Subject: [PATCH 100/272] Explicitly add allow_pickle=False (default) to all np.load() calls Will change to True only for the calls that may possibly load trusted Pickles, not only generic .npz files. https://docs.scipy.org/doc/numpy/reference/generated/numpy.load.html Changed in version 1.16.3: Made default False in response to CVE-2019-6446. Reported by @Wouter-VDP via Slack on 2019-08-08. --- examples/notebooks/Signal Influence.ipynb | 66 +++++++---------------- plasma/conf_parser.py | 21 ++++---- plasma/models/mpi_runner.py | 2 +- plasma/models/shallow_runner.py | 2 +- plasma/preprocessor/normalize.py | 6 +-- plasma/preprocessor/preprocess.py | 17 +++--- plasma/primitives/shots.py | 2 +- plasma/utils/performance.py | 3 +- 8 files changed, 45 insertions(+), 74 deletions(-) diff --git a/examples/notebooks/Signal Influence.ipynb b/examples/notebooks/Signal Influence.ipynb index 6e865638..f5f70897 100644 --- a/examples/notebooks/Signal Influence.ipynb +++ b/examples/notebooks/Signal Influence.ipynb @@ -26,20 +26,16 @@ { "cell_type": "code", "execution_count": 6, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ - "dat = np.load('./temp_data/signal_influence/signal_influence_results_155191_2017-12-01-01-42-33.npz')" + "dat = np.load('./temp_data/signal_influence/signal_influence_results_155191_2017-12-01-01-42-33.npz', allow_pickle=False)" ] }, { "cell_type": "code", "execution_count": 7, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -60,9 +56,7 @@ { "cell_type": "code", "execution_count": 8, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -102,9 +96,7 @@ { "cell_type": "code", "execution_count": 9, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -139,9 +131,7 @@ { "cell_type": "code", "execution_count": 17, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -211,9 +201,7 @@ { "cell_type": "code", "execution_count": 15, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -315,15 +303,13 @@ }, "outputs": [], "source": [ - "dat = np.load('./temp_data/signal_influence/signal_influence_results_2017-11-30-20-13-39.npz')" + "dat = np.load('./temp_data/signal_influence/signal_influence_results_2017-11-30-20-13-39.npz', allow_pickle=False)" ] }, { "cell_type": "code", "execution_count": 4, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "y_gold = dat['y_gold']\n", @@ -333,9 +319,7 @@ { "cell_type": "code", "execution_count": 97, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -362,9 +346,7 @@ { "cell_type": "code", "execution_count": 18, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "t_range = np.linspace(0,len(y_gold[0]),10,dtype=np.int)" @@ -373,9 +355,7 @@ { "cell_type": "code", "execution_count": 19, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -395,9 +375,7 @@ { "cell_type": "code", "execution_count": 95, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "def difference_metric(y_prime,y_prime_orig):\n", @@ -422,9 +400,7 @@ { "cell_type": "code", "execution_count": 96, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "differences = get_importance_measure(y_prime,difference_metric)" @@ -433,9 +409,7 @@ { "cell_type": "code", "execution_count": 24, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -464,21 +438,21 @@ ], "metadata": { "kernelspec": { - "display_name": "Python [conda root]", + "display_name": "Python 3", "language": "python", - "name": "conda-root-py" + "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12" + "pygments_lexer": "ipython3", + "version": "3.7.4" }, "latex_envs": { "LaTeX_envs_menu_present": true, diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index f94068ae..2e20a1a0 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -23,14 +23,15 @@ def parameters(input_file): base_path = output_path params['paths']['base_path'] = base_path - params['paths']['signal_prepath'] = base_path + \ - params['paths']['signal_prepath'] - params['paths']['shot_list_dir'] = base_path + \ - params['paths']['shot_list_dir'] + params['paths']['signal_prepath'] = ( + base_path + params['paths']['signal_prepath']) + params['paths']['shot_list_dir'] = ( + base_path + params['paths']['shot_list_dir']) params['paths']['output_path'] = output_path h = get_unique_signal_hash(sig.all_signals.values()) - params['paths']['global_normalizer_path'] = output_path + \ - '/normalization/normalization_signal_group_{}.npz'.format(h) + params['paths']['global_normalizer_path'] = ( + output_path + + '/normalization/normalization_signal_group_{}.npz'.format(h)) if params['training']['hyperparam_tuning']: # params['paths']['saved_shotlist_path'] = # './normalization/shot_lists.npz' @@ -44,12 +45,12 @@ def parameters(input_file): # '/normalization/shot_lists.npz' params['paths']['normalizer_path'] = ( params['paths']['global_normalizer_path']) - params['paths']['model_save_path'] = ( - output_path + '/model_checkpoints/') + params['paths']['model_save_path'] = (output_path + + '/model_checkpoints/') params['paths']['csvlog_save_path'] = output_path + '/csv_logs/' params['paths']['results_prepath'] = output_path + '/results/' - params['paths']['tensorboard_save_path'] = output_path + \ - params['paths']['tensorboard_save_path'] + params['paths']['tensorboard_save_path'] = ( + output_path + params['paths']['tensorboard_save_path']) params['paths']['saved_shotlist_path'] = ( params['paths']['base_path'] + '/processed_shotlists/' + params['paths']['data'] diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index e092d8b7..e7664131 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -670,7 +670,7 @@ def save_shotlists(conf, shot_list_train, shot_list_validate, shot_list_test): def load_shotlists(conf): path = get_shot_list_path(conf) - data = np.load(path) + data = np.load(path, allow_pickle=False) shot_list_train = data['shot_list_train'][()] shot_list_validate = data['shot_list_validate'][()] shot_list_test = data['shot_list_test'][()] diff --git a/plasma/models/shallow_runner.py b/plasma/models/shallow_runner.py index c4fa1552..b5dabd0e 100644 --- a/plasma/models/shallow_runner.py +++ b/plasma/models/shallow_runner.py @@ -137,7 +137,7 @@ def process(self, shot): # print(X.shape, Y.shape) else: try: - dat = np.load(save_path) + dat = np.load(save_path, allow_pickle=False) # X, Y, disr = dat["X"], dat["Y"], dat["disr"][()] X = dat["X"] except BaseException: diff --git a/plasma/preprocessor/normalize.py b/plasma/preprocessor/normalize.py index 3f181d0d..6a89c023 100644 --- a/plasma/preprocessor/normalize.py +++ b/plasma/preprocessor/normalize.py @@ -185,7 +185,7 @@ def previously_saved_stats(self): if not os.path.isfile(self.path): return False, set([]) else: - dat = np.load(self.path, encoding="latin1") + dat = np.load(self.path, encoding="latin1", allow_pickle=False) machines = dat['machines'][()] ret = all( [m in machines for m in self.conf['paths']['all_machines']]) @@ -292,7 +292,7 @@ def save_stats(self): def load_stats(self): assert self.previously_saved_stats()[0], "stats not saved before" - dat = np.load(self.path, encoding="latin1") + dat = np.load(self.path, encoding="latin1", allow_pickle=False) self.means = dat['means'][()] self.stds = dat['stds'][()] self.num_processed = dat['num_processed'][()] @@ -448,7 +448,7 @@ def save_stats(self): def load_stats(self): assert(self.previously_saved_stats()[0]) - dat = np.load(self.path, encoding="latin1") + dat = np.load(self.path, encoding="latin1", allow_pickle=False) self.minimums = dat['minimums'][()] self.maximums = dat['maximums'][()] self.num_processed = dat['num_processed'][()] diff --git a/plasma/preprocessor/preprocess.py b/plasma/preprocessor/preprocess.py index 17a5699f..1d93775f 100644 --- a/plasma/preprocessor/preprocess.py +++ b/plasma/preprocessor/preprocess.py @@ -149,7 +149,7 @@ def get_shot_list_path(self): def load_shotlists(self): path = self.get_shot_list_path() - data = np.load(path, encoding="latin1") + data = np.load(path, encoding="latin1", allow_pickle=False) shot_list_train = data['shot_list_train'][()] shot_list_validate = data['shot_list_validate'][()] shot_list_test = data['shot_list_test'][()] @@ -159,18 +159,13 @@ def load_shotlists(self): return ShotList(shot_list_train), ShotList( shot_list_validate), ShotList(shot_list_test) - def save_shotlists( - self, - shot_list_train, - shot_list_validate, - shot_list_test): + def save_shotlists(self, shot_list_train, shot_list_validate, + shot_list_test): path = self.get_shot_list_path() mkdirdepth(path) - np.savez( - path, - shot_list_train=shot_list_train, - shot_list_validate=shot_list_validate, - shot_list_test=shot_list_test) + np.savez(path, shot_list_train=shot_list_train, + shot_list_validate=shot_list_validate, + shot_list_test=shot_list_test) def apply_bleed_in(conf, shot_list_train, shot_list_validate, shot_list_test): diff --git a/plasma/primitives/shots.py b/plasma/primitives/shots.py index 3b96e9fc..ea4ae29c 100644 --- a/plasma/primitives/shots.py +++ b/plasma/primitives/shots.py @@ -498,7 +498,7 @@ def get_save_path(self, prepath): def restore(self, prepath, light=False): assert self.previously_saved(prepath), 'shot was never saved' save_path = self.get_save_path(prepath) - dat = np.load(save_path, encoding="latin1") + dat = np.load(save_path, encoding="latin1", allow_pickle=False) self.valid = dat['valid'][()] self.is_disruptive = dat['is_disruptive'][()] diff --git a/plasma/utils/performance.py b/plasma/utils/performance.py index 5df6a108..80c3c7f5 100644 --- a/plasma/utils/performance.py +++ b/plasma/utils/performance.py @@ -364,7 +364,8 @@ def get_accuracy_and_fp_rate_from_stats( def load_ith_file(self): results_files = os.listdir(self.results_dir) print(results_files) - dat = np.load(self.results_dir + results_files[self.i]) + dat = np.load(self.results_dir + results_files[self.i], + allow_pickle=False) print("Loading results file {}".format( self.results_dir + results_files[self.i])) From cc71f3eef914e28e5253f97ead242b43c07bf9da Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Fri, 4 Oct 2019 19:44:42 -0500 Subject: [PATCH 101/272] Clean up style of preprocess.py --- plasma/preprocessor/preprocess.py | 59 ++++++++++++------------------- 1 file changed, 22 insertions(+), 37 deletions(-) diff --git a/plasma/preprocessor/preprocess.py b/plasma/preprocessor/preprocess.py index 1d93775f..66fc804f 100644 --- a/plasma/preprocessor/preprocess.py +++ b/plasma/preprocessor/preprocess.py @@ -29,13 +29,9 @@ def __init__(self, conf): def clean_shot_lists(self): shot_list_dir = self.conf['paths']['shot_list_dir'] - paths = [ - os.path.join( - shot_list_dir, - f) for f in listdir(shot_list_dir) if os.path.isfile( - os.path.join( - shot_list_dir, - f))] + paths = [os.path.join(shot_list_dir, f) for f in + listdir(shot_list_dir) if + os.path.isfile(os.path.join(shot_list_dir, f))] for path in paths: self.clean_shot_list(path) @@ -87,16 +83,15 @@ def preprocess_from_files(self, shot_files, use_shots): # empty used_shots = ShotList() - use_cores = max(1, mp.cpu_count()-2) + # TODO(KGF): generalize the follwowing line to perform well on + # architecutres other than CPUs, e.g. KNLs + # min( , max(1,mp.cpu_count()-2) ) + use_cores = max(1, mp.cpu_count() - 2) pool = mp.Pool(use_cores) - print('running in parallel on {} processes'.format(pool._processes)) + print('Running in parallel on {} processes'.format(pool._processes)) start_time = time.time() - for ( - i, - shot) in enumerate( - pool.imap_unordered( - self.preprocess_single_file, - shot_list_picked)): + for (i, shot) in enumerate(pool.imap_unordered( + self.preprocess_single_file, shot_list_picked)): # for (i,shot) in # enumerate(map(self.preprocess_single_file,shot_list_picked)): sys.stdout.write('\r{}/{}'.format(i, len(shot_list_picked))) @@ -105,19 +100,15 @@ def preprocess_from_files(self, shot_files, use_shots): pool.close() pool.join() print('Finished Preprocessing {} files in {} seconds'.format( - len(shot_list_picked), time.time()-start_time)) - print( - 'Omitted {} shots of {} total.'.format( - len(shot_list_picked) - - len(used_shots), - len(shot_list_picked))) + len(shot_list_picked), time.time() - start_time)) + print('Omitted {} shots of {} total.'.format( + len(shot_list_picked) - len(used_shots), len(shot_list_picked))) print('{}/{} disruptive shots'.format(used_shots.num_disruptive(), len(used_shots))) if len(used_shots) == 0: - print( - "WARNING: All shots were omitted, please ensure raw data " - " is complete and available at {}.".format( - self.conf['paths']['signal_prepath'])) + print("WARNING: All shots were omitted, please ensure raw data " + " is complete and available at {}.".format( + self.conf['paths']['signal_prepath'])) return used_shots def preprocess_single_file(self, shot): @@ -127,7 +118,6 @@ def preprocess_single_file(self, shot): if recompute or not shot.previously_saved(processed_prepath): shot.preprocess(self.conf) shot.save(processed_prepath) - else: try: shot.restore(processed_prepath, light=True) @@ -135,9 +125,8 @@ def preprocess_single_file(self, shot): except BaseException: shot.preprocess(self.conf) shot.save(processed_prepath) - sys.stdout.write( - '\r{} exists but corrupted, resaved.'.format( - shot.number)) + sys.stdout.write('\r{} exists but corrupted, resaved.'.format( + shot.number)) shot.make_light() return shot @@ -262,14 +251,10 @@ def guarantee_preprocessed(conf): pp.save_shotlists(shot_list_train, shot_list_validate, shot_list_test) shot_list_train, shot_list_validate, shot_list_test = apply_bleed_in( conf, shot_list_train, shot_list_validate, shot_list_test) - print( - 'validate: {} shots, {} disruptive'.format( - len(shot_list_validate), - shot_list_validate.num_disruptive())) - print( - 'training: {} shots, {} disruptive'.format( - len(shot_list_train), - shot_list_train.num_disruptive())) + print('validate: {} shots, {} disruptive'.format( + len(shot_list_validate), shot_list_validate.num_disruptive())) + print('training: {} shots, {} disruptive'.format( + len(shot_list_train), shot_list_train.num_disruptive())) print('testing: {} shots, {} disruptive'.format( len(shot_list_test), shot_list_test.num_disruptive())) print("...done") From 67c5622b3c7c0e0547da9efed93f23af4677fda8 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 8 Oct 2019 19:33:02 -0400 Subject: [PATCH 102/272] Start switching np.load(...,allow_pickle=True) as needed Even though these 2x calls load .npz files, they contain NumPy object arrays of ShotList class objects that are implicitly serialized by Pickle via np.savez() calls. Also, fix bug with sorted(ShotList) being converted to a List instead of a ShotList. --- plasma/preprocessor/preprocess.py | 5 +++-- plasma/primitives/shots.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/plasma/preprocessor/preprocess.py b/plasma/preprocessor/preprocess.py index 66fc804f..9861f4a4 100644 --- a/plasma/preprocessor/preprocess.py +++ b/plasma/preprocessor/preprocess.py @@ -138,7 +138,7 @@ def get_shot_list_path(self): def load_shotlists(self): path = self.get_shot_list_path() - data = np.load(path, encoding="latin1", allow_pickle=False) + data = np.load(path, encoding="latin1", allow_pickle=True) shot_list_train = data['shot_list_train'][()] shot_list_validate = data['shot_list_validate'][()] shot_list_test = data['shot_list_test'][()] @@ -239,7 +239,8 @@ def guarantee_preprocessed(conf): else: print("preprocessing all shots", end='') pp.clean_shot_lists() - shot_list = sorted(pp.preprocess_all()) + shot_list = pp.preprocess_all() + shot_list.sort() shot_list_train, shot_list_test = shot_list.split_train_test(conf) # num_shots = len(shot_list_train) + len(shot_list_test) validation_frac = conf['training']['validation_frac'] diff --git a/plasma/primitives/shots.py b/plasma/primitives/shots.py index ea4ae29c..cafb76ed 100644 --- a/plasma/primitives/shots.py +++ b/plasma/primitives/shots.py @@ -498,7 +498,7 @@ def get_save_path(self, prepath): def restore(self, prepath, light=False): assert self.previously_saved(prepath), 'shot was never saved' save_path = self.get_save_path(prepath) - dat = np.load(save_path, encoding="latin1", allow_pickle=False) + dat = np.load(save_path, encoding="latin1", allow_pickle=True) self.valid = dat['valid'][()] self.is_disruptive = dat['is_disruptive'][()] From aae1536fd967c94d3b95ef170e5af4fc60fb62c8 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 8 Oct 2019 19:38:48 -0400 Subject: [PATCH 103/272] Re-order matplotlib.use() calls (before plt import) Addressing warning: This call to matplotlib.use() has no effect because the backend has already been chosen; matplotlib.use() must be called *before* pylab, matplotlib.pyplot, or matplotlib.backends is imported for the first time. --- plasma/models/runner.py | 2 +- plasma/models/shallow_runner.py | 2 +- plasma/utils/performance.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plasma/models/runner.py b/plasma/models/runner.py index 2eb0b58b..d579d9c7 100644 --- a/plasma/models/runner.py +++ b/plasma/models/runner.py @@ -10,9 +10,9 @@ from hyperopt import hp, STATUS_OK import numpy as np import sys -import matplotlib.pyplot as plt import matplotlib matplotlib.use('Agg') +import matplotlib.pyplot as plt # if sys.version_info[0] < 3: # from itertools import imap diff --git a/plasma/models/shallow_runner.py b/plasma/models/shallow_runner.py index b5dabd0e..cc7ea2e6 100644 --- a/plasma/models/shallow_runner.py +++ b/plasma/models/shallow_runner.py @@ -20,9 +20,9 @@ import datetime import time import numpy as np -# import matplotlib.pyplot as plt import matplotlib matplotlib.use('Agg') +# import matplotlib.pyplot as plt # import sys # if sys.version_info[0] < 3: diff --git a/plasma/utils/performance.py b/plasma/utils/performance.py index 80c3c7f5..efe26b50 100644 --- a/plasma/utils/performance.py +++ b/plasma/utils/performance.py @@ -5,10 +5,10 @@ import numpy as np from pprint import pprint import os -import matplotlib.pyplot as plt -from matplotlib import rc import matplotlib matplotlib.use('Agg') # for machines that don't have a display +import matplotlib.pyplot as plt +from matplotlib import rc rc('font', **{'family': 'serif', 'sans-serif': ['Times']}) rc('text', usetex=True) From 52d497190747911d2c7607c685cec4eaf8180154 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 8 Oct 2019 18:45:04 -0500 Subject: [PATCH 104/272] Suppress flake8 warnings from the change in the parent commit --- plasma/models/runner.py | 2 +- plasma/utils/performance.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plasma/models/runner.py b/plasma/models/runner.py index d579d9c7..5d09b33e 100644 --- a/plasma/models/runner.py +++ b/plasma/models/runner.py @@ -12,7 +12,7 @@ import sys import matplotlib matplotlib.use('Agg') -import matplotlib.pyplot as plt +import matplotlib.pyplot as plt # noqa # if sys.version_info[0] < 3: # from itertools import imap diff --git a/plasma/utils/performance.py b/plasma/utils/performance.py index efe26b50..ff6acef7 100644 --- a/plasma/utils/performance.py +++ b/plasma/utils/performance.py @@ -7,8 +7,8 @@ import os import matplotlib matplotlib.use('Agg') # for machines that don't have a display -import matplotlib.pyplot as plt -from matplotlib import rc +import matplotlib.pyplot as plt # noqa +from matplotlib import rc # noqa rc('font', **{'family': 'serif', 'sans-serif': ['Times']}) rc('text', usetex=True) From f34c9b11eccc79e23bc12b0a5b22fa7701152810 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 9 Oct 2019 10:47:59 -0400 Subject: [PATCH 105/272] Let normalize.py load pickles with np.load() --- plasma/preprocessor/normalize.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plasma/preprocessor/normalize.py b/plasma/preprocessor/normalize.py index 6a89c023..eba693a0 100644 --- a/plasma/preprocessor/normalize.py +++ b/plasma/preprocessor/normalize.py @@ -185,7 +185,7 @@ def previously_saved_stats(self): if not os.path.isfile(self.path): return False, set([]) else: - dat = np.load(self.path, encoding="latin1", allow_pickle=False) + dat = np.load(self.path, encoding="latin1", allow_pickle=True) machines = dat['machines'][()] ret = all( [m in machines for m in self.conf['paths']['all_machines']]) @@ -292,7 +292,7 @@ def save_stats(self): def load_stats(self): assert self.previously_saved_stats()[0], "stats not saved before" - dat = np.load(self.path, encoding="latin1", allow_pickle=False) + dat = np.load(self.path, encoding="latin1", allow_pickle=True) self.means = dat['means'][()] self.stds = dat['stds'][()] self.num_processed = dat['num_processed'][()] @@ -448,7 +448,7 @@ def save_stats(self): def load_stats(self): assert(self.previously_saved_stats()[0]) - dat = np.load(self.path, encoding="latin1", allow_pickle=False) + dat = np.load(self.path, encoding="latin1", allow_pickle=True) self.minimums = dat['minimums'][()] self.maximums = dat['maximums'][()] self.num_processed = dat['num_processed'][()] From dd2c8a635b631062b2363e49d29a2af7011f71d0 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 9 Oct 2019 17:27:40 -0500 Subject: [PATCH 106/272] Fix flake8 style warnings for all files except torch_runner.py --- plasma/conf_parser.py | 16 ++++++++-------- plasma/models/builder.py | 4 +++- plasma/models/mpi_runner.py | 4 ++-- plasma/models/shallow_runner.py | 1 - plasma/preprocessor/normalize.py | 9 +++++---- plasma/utils/downloading.py | 15 ++++++++++----- plasma/utils/performance.py | 16 +++++----------- 7 files changed, 33 insertions(+), 32 deletions(-) diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index 2a1b1972..6bbe9bb3 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -96,10 +96,10 @@ def parameters(input_file): sig.jet, params['paths']['shot_list_dir'], ['ILW_unint_late.txt', 'ILW_clear_late.txt'], 'Late jet iter like wall data') - jet_iterlike_wall_full = ShotListFiles( - sig.jet, params['paths']['shot_list_dir'], - ['ILW_unint_full.txt', 'ILW_clear_full.txt'], - 'Full jet iter like wall data') + # jet_iterlike_wall_full = ShotListFiles( + # sig.jet, params['paths']['shot_list_dir'], + # ['ILW_unint_full.txt', 'ILW_clear_full.txt'], + # 'Full jet iter like wall data') jenkins_jet_carbon_wall = ShotListFiles( sig.jet, params['paths']['shot_list_dir'], @@ -169,12 +169,12 @@ def parameters(input_file): params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = { - 'etemp_profile': etemp_profile} + 'etemp_profile': sig.etemp_profile} elif params['paths']['data'] == 'jet_data_dens_profile': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [jet_iterlike_wall] params['paths']['use_signals_dict'] = { - 'edens_profile': edens_profile} + 'edens_profile': sig.edens_profile} elif params['paths']['data'] == 'jet_carbon_data': params['paths']['shot_files'] = [jet_carbon_wall] params['paths']['shot_files_test'] = [] @@ -302,13 +302,13 @@ def parameters(input_file): params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { - 'etemp_profile': etemp_profile} # fully_defined_signals_0D + 'etemp_profile': sig.etemp_profile} # fully_defined_signals_0D elif params['paths']['data'] == 'd3d_data_dens_profile': # jet data but with fully defined signals params['paths']['shot_files'] = [d3d_full] params['paths']['shot_files_test'] = [] params['paths']['use_signals_dict'] = { - 'edens_profile': edens_profile} # fully_defined_signals_0D + 'edens_profile': sig.edens_profile} # fully_defined_signals_0D # cross-machine elif params['paths']['data'] == 'jet_to_d3d_data': diff --git a/plasma/models/builder.py b/plasma/models/builder.py index f7782362..dc49e0ef 100644 --- a/plasma/models/builder.py +++ b/plasma/models/builder.py @@ -82,6 +82,7 @@ def build_model(self, predict, custom_batch_size=None): conf = self.conf model_conf = conf['model'] rnn_size = model_conf['rnn_size'] + use_bidirectional = model_conf['use_bidirectional'] rnn_type = model_conf['rnn_type'] regularization = model_conf['regularization'] dense_regularization = model_conf['dense_regularization'] @@ -353,7 +354,8 @@ def get_all_saved_files(self): self.ensure_save_directory() unique_id = self.get_unique_id() path = self.conf['paths']['model_save_path'] - filenames = [name for name in os.listdir(path) if os.path.isfile(os.path.join(path, name))] + filenames = [name for name in os.listdir(path) + if os.path.isfile(os.path.join(path, name))] epochs = [] for file in filenames: curr_id, epoch = self.extract_id_and_epoch_from_filename(file) diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 3760fb49..00bf4704 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -786,7 +786,7 @@ def mpi_make_predictions_and_evaluate_multiple_times(conf, shot_list, loader, def mpi_train(conf, shot_list_train, shot_list_validate, loader, - callbacks_list=None shot_list_test=None): + callbacks_list=None, shot_list_test=None): loader.set_inference_mode(False) conf['num_workers'] = comm.Get_size() @@ -949,7 +949,7 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, round(e)), epoch_logs) print_unique("end epoch {} 0".format(e)) - stop_training = comm.bcast(stop_training,root=0) + stop_training = comm.bcast(stop_training, root=0) print_unique("end epoch {} 1".format(e)) if stop_training: print("Stopping training due to early stopping") diff --git a/plasma/models/shallow_runner.py b/plasma/models/shallow_runner.py index 63e0fe44..aaabd2b9 100644 --- a/plasma/models/shallow_runner.py +++ b/plasma/models/shallow_runner.py @@ -21,7 +21,6 @@ import time import numpy as np from copy import deepcopy -from keras.utils.generic_utils import Progbar # import matplotlib.pyplot as plt import matplotlib diff --git a/plasma/preprocessor/normalize.py b/plasma/preprocessor/normalize.py index 47eb20e2..09f488a2 100644 --- a/plasma/preprocessor/normalize.py +++ b/plasma/preprocessor/normalize.py @@ -153,12 +153,13 @@ def cut_end_of_shot(self, shot): # only cut shots during training if not self.inference_mode and cut_shot_ends: T_min_warn = self.conf['data']['T_min_warn'] - if shot.ttd.shape[0] - T_min_warn <= max(self.conf['model']['length'],0): - print("not cutting shot since length of shot after cutting by T_min_warn would be shorter than RNN length") + if shot.ttd.shape[0] - T_min_warn <= max( + self.conf['model']['length'], 0): + print("not cutting shot; length of shot after cutting by ", + "T_min_warn would be shorter than RNN length") return for key in shot.signals_dict: - shot.signals_dict[key] = shot.signals_dict[key][:-T_min_warn, - :] + shot.signals_dict[key] = shot.signals_dict[key][:-T_min_warn,:] # noqa shot.ttd = shot.ttd[:-T_min_warn] # def apply_mask(self,shot): diff --git a/plasma/utils/downloading.py b/plasma/utils/downloading.py index d4d66463..27c9fb93 100644 --- a/plasma/utils/downloading.py +++ b/plasma/utils/downloading.py @@ -33,9 +33,11 @@ def general_object_hash(o): - """Makes a hash from a dictionary, list, tuple or set to any level, that contains - only other hashable types (including any lists, tuples, sets, and - dictionaries). Relies on dill for serialization""" + """ + Makes a hash from a dictionary, list, tuple or set to any level, that + contains only other hashable types (including any lists, tuples, sets, and + dictionaries). Relies on dill for serialization +""" if isinstance(o, (set, tuple, list)): return tuple([general_object_hash(e) for e in o]) @@ -51,9 +53,12 @@ def general_object_hash(o): def myhash(x): - return int(hashlib.md5((dill.dumps(x).decode('unicode_escape')).encode('utf-8')).hexdigest(),16) + return int(hashlib.md5((dill.dumps(x).decode('unicode_escape')).encode( + 'utf-8')).hexdigest(), 16) # return int(hashlib.md5((dill.dumps(x))).hexdigest(),16) - # return int(hashlib.md5((dill.dumps(x))))#.decode('unicode_escape')).encode('utf-8')).hexdigest(),16) + # return + # int(hashlib.md5((dill.dumps(x))))#.decode('unicode_escape')).encode( + # 'utf-8')).hexdigest(),16) def get_missing_value_array(): diff --git a/plasma/utils/performance.py b/plasma/utils/performance.py index 25ac904e..3dd52474 100644 --- a/plasma/utils/performance.py +++ b/plasma/utils/performance.py @@ -14,16 +14,8 @@ class PerformanceAnalyzer(): - def __init__( - self, - results_dir=None, - shots_dir=None, - i=0, - T_min_warn=None, - T_max_warn=None, - verbose=False, - pred_ttd=False, - conf=None): + def __init__(self, results_dir=None, shots_dir=None, i=0, T_min_warn=None, + T_max_warn=None, verbose=False, pred_ttd=False, conf=None): self.T_min_warn = T_min_warn self.T_max_warn = T_max_warn dt = conf['data']['dt'] @@ -35,7 +27,9 @@ def __init__( if T_max_warn is None: self.T_max_warn = T_max_warn_def if self.T_max_warn < self.T_min_warn: - print("T max warn is too small: need to increase artificially.") #computation of statistics is only correct if T_max_warn is larger than T_min_warn + # computation of statistics is only correct if T_max_warn is larger + # than T_min_warn + print("T max warn is too small: need to increase artificially.") self.T_max_warn = self.T_min_warn + 1 self.verbose = verbose self.results_dir = results_dir From 3f9ea6d02647af0963028f498e40b14ba322c1ef Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 9 Oct 2019 17:31:26 -0500 Subject: [PATCH 107/272] Apply autopep8 to torch_runner.py --- plasma/models/torch_runner.py | 399 +++++++++++++++++++++------------- 1 file changed, 253 insertions(+), 146 deletions(-) diff --git a/plasma/models/torch_runner.py b/plasma/models/torch_runner.py index 3d7885ba..f85cae87 100644 --- a/plasma/models/torch_runner.py +++ b/plasma/models/torch_runner.py @@ -1,60 +1,87 @@ +import keras.callbacks as cbks +from keras.utils.generic_utils import Progbar +from torch.nn.utils import weight_norm +import torch.optim as opt +from torch.autograd import Variable +import torch.nn as nn +import torch +import hashlib +from plasma.utils.downloading import makedirs_process_safe +from plasma.utils.state_reset import reset_states +from plasma.utils.evaluation import * +from plasma.utils.performance import PerformanceAnalyzer +from plasma.models.loader import Loader, ProcessGenerator +from plasma.conf import conf +from sklearn.neural_network import MLPClassifier +from xgboost import XGBClassifier +import pathos.multiprocessing as mp +from functools import partial +import os +import datetime +import time +import sys +import numpy as np +import matplotlib.pyplot as plt from __future__ import print_function import matplotlib matplotlib.use('Agg') -import matplotlib.pyplot as plt -import numpy as np -import sys if sys.version_info[0] < 3: from itertools import imap -#leading to import errors: +# leading to import errors: #from hyperopt import hp, STATUS_OK #from hyperas.distributions import conditional -import time -import datetime -import os -from functools import partial -import pathos.multiprocessing as mp -from xgboost import XGBClassifier -from sklearn.neural_network import MLPClassifier - -from plasma.conf import conf -from plasma.models.loader import Loader, ProcessGenerator -from plasma.utils.performance import PerformanceAnalyzer -from plasma.utils.evaluation import * -from plasma.utils.state_reset import reset_states -from plasma.utils.downloading import makedirs_process_safe - -from keras.utils.generic_utils import Progbar - -import hashlib - -import torch -import torch.nn as nn -from torch.autograd import Variable -import torch.optim as opt -from torch.nn.utils import weight_norm model_filename = 'torch_model.pt' + class FTCN(nn.Module): - def __init__(self,n_scalars,n_profiles,profile_size,layer_sizes_spatial, - kernel_size_spatial,linear_size,output_size, - num_channels_tcn,kernel_size_temporal,dropout=0.1): + def __init__( + self, + n_scalars, + n_profiles, + profile_size, + layer_sizes_spatial, + kernel_size_spatial, + linear_size, + output_size, + num_channels_tcn, + kernel_size_temporal, + dropout=0.1): super(FTCN, self).__init__() - self.lin = InputBlock(n_scalars, n_profiles,profile_size, layer_sizes_spatial, kernel_size_spatial, linear_size, dropout) - self.input_layer = TimeDistributed(self.lin,batch_first=True) - self.tcn = TCN(linear_size, output_size, num_channels_tcn , kernel_size_temporal, dropout) - self.model = nn.Sequential(self.input_layer,self.tcn) - - def forward(self,x): + self.lin = InputBlock( + n_scalars, + n_profiles, + profile_size, + layer_sizes_spatial, + kernel_size_spatial, + linear_size, + dropout) + self.input_layer = TimeDistributed(self.lin, batch_first=True) + self.tcn = TCN( + linear_size, + output_size, + num_channels_tcn, + kernel_size_temporal, + dropout) + self.model = nn.Sequential(self.input_layer, self.tcn) + + def forward(self, x): return self.model(x) class InputBlock(nn.Module): - def __init__(self, n_scalars, n_profiles,profile_size, layer_sizes, kernel_size, linear_size, dropout=0.2): + def __init__( + self, + n_scalars, + n_profiles, + profile_size, + layer_sizes, + kernel_size, + linear_size, + dropout=0.2): super(InputBlock, self).__init__() self.pooling_size = 2 self.n_scalars = n_scalars @@ -66,25 +93,39 @@ def __init__(self, n_scalars, n_profiles,profile_size, layer_sizes, kernel_size, self.conv_output_size = 0 else: self.layers = [] - for (i,layer_size) in enumerate(layer_sizes): + for (i, layer_size) in enumerate(layer_sizes): if i == 0: input_size = n_profiles else: input_size = layer_sizes[i-1] - self.layers.append(weight_norm(nn.Conv1d(input_size, layer_size, kernel_size))) + self.layers.append( + weight_norm( + nn.Conv1d( + input_size, + layer_size, + kernel_size))) self.layers.append(nn.ReLU()) - self.conv_output_size = calculate_conv_output_size(self.conv_output_size,0,1,1,kernel_size) + self.conv_output_size = calculate_conv_output_size( + self.conv_output_size, 0, 1, 1, kernel_size) self.layers.append(nn.MaxPool1d(kernel_size=self.pooling_size)) - self.conv_output_size = calculate_conv_output_size(self.conv_output_size,0,1,self.pooling_size,self.pooling_size) + self.conv_output_size = calculate_conv_output_size( + self.conv_output_size, 0, 1, self.pooling_size, self.pooling_size) self.layers.append(nn.Dropout2d(dropout)) self.net = nn.Sequential(*self.layers) self.conv_output_size = self.conv_output_size*layer_sizes[-1] self.linear_layers = [] - - print("Final feature size = {}".format(self.n_scalars + self.conv_output_size)) - self.linear_layers.append(nn.Linear(self.conv_output_size+self.n_scalars,linear_size)) + + print( + "Final feature size = {}".format( + self.n_scalars + + self.conv_output_size)) + self.linear_layers.append( + nn.Linear( + self.conv_output_size + + self.n_scalars, + linear_size)) self.linear_layers.append(nn.ReLU()) - self.linear_layers.append(nn.Linear(linear_size,linear_size)) + self.linear_layers.append(nn.Linear(linear_size, linear_size)) self.linear_layers.append(nn.ReLU()) print("Final output size = {}".format(linear_size)) self.linear_net = nn.Sequential(*self.linear_layers) @@ -97,28 +138,30 @@ def __init__(self, n_scalars, n_profiles,profile_size, layer_sizes, kernel_size, def forward(self, x): if self.n_profiles == 0: - full_features = x#x_scalars + full_features = x # x_scalars else: if self.n_scalars == 0: x_profiles = x else: - x_scalars = x[:,:self.n_scalars] - x_profiles = x[:,self.n_scalars:] - x_profiles = x_profiles.contiguous().view(x.size(0),self.n_profiles,self.profile_size) - profile_features = self.net(x_profiles).view(x.size(0),-1) + x_scalars = x[:, :self.n_scalars] + x_profiles = x[:, self.n_scalars:] + x_profiles = x_profiles.contiguous().view( + x.size(0), self.n_profiles, self.profile_size) + profile_features = self.net(x_profiles).view(x.size(0), -1) if self.n_scalars == 0: full_features = profile_features else: - full_features = torch.cat([x_scalars,profile_features],dim=1) - + full_features = torch.cat([x_scalars, profile_features], dim=1) + out = self.linear_net(full_features) # out = self.net(x) # res = x if self.downsample is None else self.downsample(x) return out -def calculate_conv_output_size(L_in,padding,dilation,stride,kernel_size): - return int(np.floor((L_in + 2*padding - dilation*(kernel_size-1) - 1)*1.0/stride + 1)) +def calculate_conv_output_size(L_in, padding, dilation, stride, kernel_size): + return int(np.floor((L_in + 2*padding - dilation + * (kernel_size-1) - 1)*1.0/stride + 1)) class Chomp1d(nn.Module): @@ -131,23 +174,51 @@ def forward(self, x): class TemporalBlock(nn.Module): - def __init__(self, n_inputs, n_outputs, kernel_size, stride, dilation, padding, dropout=0.2): + def __init__( + self, + n_inputs, + n_outputs, + kernel_size, + stride, + dilation, + padding, + dropout=0.2): super(TemporalBlock, self).__init__() - self.conv1 = weight_norm(nn.Conv1d(n_inputs, n_outputs, kernel_size, - stride=stride, padding=padding, dilation=dilation)) + self.conv1 = weight_norm( + nn.Conv1d( + n_inputs, + n_outputs, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation)) self.chomp1 = Chomp1d(padding) self.relu1 = nn.ReLU() self.dropout1 = nn.Dropout2d(dropout) - self.conv2 = weight_norm(nn.Conv1d(n_outputs, n_outputs, kernel_size, - stride=stride, padding=padding, dilation=dilation)) + self.conv2 = weight_norm( + nn.Conv1d( + n_outputs, + n_outputs, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation)) self.chomp2 = Chomp1d(padding) self.relu2 = nn.ReLU() self.dropout2 = nn.Dropout2d(dropout) - self.net = nn.Sequential(self.conv1, self.chomp1, self.relu1, self.dropout1, - self.conv2, self.chomp2, self.relu2, self.dropout2) - self.downsample = nn.Conv1d(n_inputs, n_outputs, 1) if n_inputs != n_outputs else None + self.net = nn.Sequential( + self.conv1, + self.chomp1, + self.relu1, + self.dropout1, + self.conv2, + self.chomp2, + self.relu2, + self.dropout2) + self.downsample = nn.Conv1d( + n_inputs, n_outputs, 1) if n_inputs != n_outputs else None self.relu = nn.ReLU() self.init_weights() @@ -162,7 +233,9 @@ def forward(self, x): res = x if self.downsample is None else self.downsample(x) return self.relu(out + res) -#dimensions are batch,channels,length +# dimensions are batch,channels,length + + class TemporalConvNet(nn.Module): def __init__(self, num_inputs, num_channels, kernel_size=2, dropout=0.2): super(TemporalConvNet, self).__init__() @@ -172,32 +245,45 @@ def __init__(self, num_inputs, num_channels, kernel_size=2, dropout=0.2): dilation_size = 2 ** i in_channels = num_inputs if i == 0 else num_channels[i-1] out_channels = num_channels[i] - layers += [TemporalBlock(in_channels, out_channels, kernel_size, stride=1, dilation=dilation_size, - padding=(kernel_size-1) * dilation_size, dropout=dropout)] + layers += [TemporalBlock(in_channels, + out_channels, + kernel_size, + stride=1, + dilation=dilation_size, + padding=(kernel_size-1) * dilation_size, + dropout=dropout)] self.network = nn.Sequential(*layers) def forward(self, x): return self.network(x) - - + + class TCN(nn.Module): - def __init__(self, input_size, output_size, num_channels, kernel_size, dropout): + def __init__( + self, + input_size, + output_size, + num_channels, + kernel_size, + dropout): super(TCN, self).__init__() - self.tcn = TemporalConvNet(input_size, num_channels, kernel_size, dropout=dropout) + self.tcn = TemporalConvNet( + input_size, + num_channels, + kernel_size, + dropout=dropout) self.linear = nn.Linear(num_channels[-1], output_size) # self.sig = nn.Sigmoid() def forward(self, x): # x needs to have dimension (N, C, L) in order to be passed into CNN output = self.tcn(x.transpose(1, 2)).transpose(1, 2) - output = self.linear(output)#.transpose(1,2)).transpose(1,2) + output = self.linear(output) # .transpose(1,2)).transpose(1,2) return output # return self.sig(output) - - # def train(model,data_gen,lr=0.001,iters = 100): # log_step = int(round(iters*0.1)) # optimizer = opt.Adam(model.parameters(),lr = lr) @@ -206,7 +292,7 @@ def forward(self, x): # count = 0 # loss_fn = nn.MSELoss(size_average=False) # for i in range(iters): -# x_,y_,mask_ = data_gen() +# x_,y_,mask_ = data_gen() # # print(y) # x, y, mask = Variable(torch.from_numpy(x_).float()), Variable(torch.from_numpy(y_).float()),Variable(torch.from_numpy(mask_).byte()) # # print(y) @@ -231,12 +317,6 @@ def forward(self, x): # count = 0 - - - - - - class TimeDistributed(nn.Module): def __init__(self, module, batch_first=False): super(TimeDistributed, self).__init__() @@ -249,22 +329,21 @@ def forward(self, x): return self.module(x) # Squash samples and timesteps into a single axis - x_reshape = x.contiguous().view(-1, x.size(-1)) # (samples * timesteps, input_size) + # (samples * timesteps, input_size) + x_reshape = x.contiguous().view(-1, x.size(-1)) y = self.module(x_reshape) # We have to reshape Y if self.batch_first: - y = y.contiguous().view(x.size(0), -1, y.size(-1)) # (samples, timesteps, output_size) + # (samples, timesteps, output_size) + y = y.contiguous().view(x.size(0), -1, y.size(-1)) else: - y = y.view(-1, x.size(1), y.size(-1)) # (timesteps, samples, output_size) + # (timesteps, samples, output_size) + y = y.view(-1, x.size(1), y.size(-1)) return y -import keras.callbacks as cbks - - - def build_torch_model(conf): dropout = conf['model']['dropout_prob'] @@ -277,25 +356,35 @@ def build_torch_model(conf): output_size = 1 # intermediate_dim = 15 - layer_sizes_spatial = [6,3,3]#[40,20,20] + layer_sizes_spatial = [6, 3, 3] # [40,20,20] kernel_size_spatial = 3 linear_size = 5 - num_channels_tcn = [10,5,3,3]#[3]*5 - kernel_size_temporal = 3 #3 - model = FTCN(n_scalars,n_profiles,profile_size,layer_sizes_spatial, - kernel_size_spatial,linear_size,output_size,num_channels_tcn, - kernel_size_temporal,dropout) + num_channels_tcn = [10, 5, 3, 3] # [3]*5 + kernel_size_temporal = 3 # 3 + model = FTCN( + n_scalars, + n_profiles, + profile_size, + layer_sizes_spatial, + kernel_size_spatial, + linear_size, + output_size, + num_channels_tcn, + kernel_size_temporal, + dropout) return model + def get_signal_dimensions(conf): - #make sure all 1D indices are contiguous in the end! + # make sure all 1D indices are contiguous in the end! use_signals = conf['paths']['use_signals'] n_scalars = 0 n_profiles = 0 profile_size = 0 - is_1D_region = use_signals[0].num_channels > 1#do we have any 1D indices? + # do we have any 1D indices? + is_1D_region = use_signals[0].num_channels > 1 for sig in use_signals: num_channels = sig.num_channels if num_channels > 1: @@ -303,23 +392,25 @@ def get_signal_dimensions(conf): n_profiles += 1 is_1D_region = True else: - assert(not is_1D_region), "make sure all use_signals are ordered such that 1D signals come last!" + assert( + not is_1D_region), "make sure all use_signals are ordered such that 1D signals come last!" assert(num_channels == 1) n_scalars += 1 is_1D_region = False - return n_scalars,n_profiles,profile_size + return n_scalars, n_profiles, profile_size -def apply_model_to_np(model,x): - # return model(Variable(torch.from_numpy(x).float()).unsqueeze(0)).squeeze(0).data.numpy() - return model(Variable(torch.from_numpy(x).float())).data.numpy() +def apply_model_to_np(model, x): + # return + # model(Variable(torch.from_numpy(x).float()).unsqueeze(0)).squeeze(0).data.numpy() + return model(Variable(torch.from_numpy(x).float())).data.numpy() -def make_predictions(conf,shot_list,loader,custom_path=None): +def make_predictions(conf, shot_list, loader, custom_path=None): generator = loader.inference_batch_generator_full_shot(shot_list) inference_model = build_torch_model(conf) - if custom_path == None: + if custom_path is None: model_path = get_model_path(conf) else: model_path = custom_path @@ -331,15 +422,15 @@ def make_predictions(conf,shot_list,loader,custom_path=None): disruptive = [] num_shots = len(shot_list) - pbar = Progbar(num_shots) + pbar = Progbar(num_shots) while True: - x,y,mask,disr,lengths,num_so_far,num_total = next(generator) + x, y, mask, disr, lengths, num_so_far, num_total = next(generator) #x, y, mask = Variable(torch.from_numpy(x_).float()), Variable(torch.from_numpy(y_).float()),Variable(torch.from_numpy(mask_).byte()) - output = apply_model_to_np(inference_model,x) + output = apply_model_to_np(inference_model, x) for batch_idx in range(x.shape[0]): curr_length = lengths[batch_idx] - y_prime += [output[batch_idx,:curr_length,0]] - y_gold += [y[batch_idx,:curr_length,0]] + y_prime += [output[batch_idx, :curr_length, 0]] + y_gold += [y[batch_idx, :curr_length, 0]] disruptive += [disr[batch_idx]] pbar.add(1.0) if len(disruptive) >= num_shots: @@ -347,68 +438,84 @@ def make_predictions(conf,shot_list,loader,custom_path=None): y_gold = y_gold[:num_shots] disruptive = disruptive[:num_shots] break - return y_prime,y_gold,disruptive + return y_prime, y_gold, disruptive + -def make_predictions_and_evaluate_gpu(conf,shot_list,loader,custom_path = None): - y_prime,y_gold,disruptive = make_predictions(conf,shot_list,loader,custom_path) +def make_predictions_and_evaluate_gpu( + conf, shot_list, loader, custom_path=None): + y_prime, y_gold, disruptive = make_predictions( + conf, shot_list, loader, custom_path) analyzer = PerformanceAnalyzer(conf=conf) - roc_area = analyzer.get_roc_area(y_prime,y_gold,disruptive) - loss = get_loss_from_list(y_prime,y_gold,conf['data']['target']) - return y_prime,y_gold,disruptive,roc_area,loss + roc_area = analyzer.get_roc_area(y_prime, y_gold, disruptive) + loss = get_loss_from_list(y_prime, y_gold, conf['data']['target']) + return y_prime, y_gold, disruptive, roc_area, loss def get_model_path(conf): - return conf['paths']['model_save_path'] + 'torch/' + model_filename #save_prepath + model_filename + return conf['paths']['model_save_path'] + 'torch/' + \ + model_filename # save_prepath + model_filename -def train_epoch(model,data_gen,optimizer,loss_fn): +def train_epoch(model, data_gen, optimizer, loss_fn): loss = 0 total_loss = 0 num_so_far = 0 - x_,y_,mask_,num_so_far_start,num_total = next(data_gen) + x_, y_, mask_, num_so_far_start, num_total = next(data_gen) num_so_far = num_so_far_start step = 0 while True: - # print(y) - x, y, mask = Variable(torch.from_numpy(x_).float()), Variable(torch.from_numpy(y_).float()),Variable(torch.from_numpy(mask_).byte()) + # print(y) + x, y, mask = Variable( + torch.from_numpy(x_).float()), Variable( + torch.from_numpy(y_).float()), Variable( + torch.from_numpy(mask_).byte()) # print(y) optimizer.zero_grad() # output = model(x.unsqueeze(0)).squeeze(0) - output = model(x)#.unsqueeze(0)).squeeze(0) - output_masked = torch.masked_select(output,mask) - y_masked = torch.masked_select(y,mask) + output = model(x) # .unsqueeze(0)).squeeze(0) + output_masked = torch.masked_select(output, mask) + y_masked = torch.masked_select(y, mask) # print(y.shape,output.shape) - loss = loss_fn(output_masked,y_masked) + loss = loss_fn(output_masked, y_masked) total_loss += loss.data[0] # count += output.size(0) # if args.clip > 0: - # torch.nn.utils.clip_grad_norm(model.parameters(), args.clip) + # torch.nn.utils.clip_grad_norm(model.parameters(), args.clip) loss.backward() optimizer.step() step += 1 - print("[{}] [{}/{}] loss: {:.3f}, ave_loss: {:.3f}".format(step,num_so_far-num_so_far_start,num_total,loss.data[0],total_loss/step)) + print("[{}] [{}/{}] loss: {:.3f}, ave_loss: {:.3f}".format(step, + num_so_far-num_so_far_start, num_total, loss.data[0], total_loss/step)) if num_so_far-num_so_far_start >= num_total: break - x_,y_,mask_,num_so_far,num_total = next(data_gen) - return step,loss.data[0],total_loss,num_so_far,1.0*num_so_far/num_total + x_, y_, mask_, num_so_far, num_total = next(data_gen) + return step, loss.data[0], total_loss, num_so_far, 1.0*num_so_far/num_total -def train(conf,shot_list_train,shot_list_validate,loader): +def train(conf, shot_list_train, shot_list_validate, loader): np.random.seed(1) #data_gen = ProcessGenerator(partial(loader.training_batch_generator_full_shot_partial_reset,shot_list=shot_list_train)()) - data_gen = partial(loader.training_batch_generator_full_shot_partial_reset,shot_list=shot_list_train)() - - print('validate: {} shots, {} disruptive'.format(len(shot_list_validate),shot_list_validate.num_disruptive())) - print('training: {} shots, {} disruptive'.format(len(shot_list_train),shot_list_train.num_disruptive())) + data_gen = partial( + loader.training_batch_generator_full_shot_partial_reset, + shot_list=shot_list_train)() + + print( + 'validate: {} shots, {} disruptive'.format( + len(shot_list_validate), + shot_list_validate.num_disruptive())) + print( + 'training: {} shots, {} disruptive'.format( + len(shot_list_train), + shot_list_train.num_disruptive())) loader.set_inference_mode(False) - train_model = build_torch_model(conf) + train_model = build_torch_model(conf) - #load the latest epoch we did. Returns -1 if none exist yet + # load the latest epoch we did. Returns -1 if none exist yet # e = specific_builder.load_model_weights(train_model) num_epochs = conf['training']['num_epochs'] @@ -433,15 +540,14 @@ def train(conf,shot_list_train,shot_list_validate,loader): print('{} epochs left to go'.format(num_epochs - 1 - e)) - if conf['callbacks']['mode'] == 'max': best_so_far = -np.inf cmp_fn = max else: best_so_far = np.inf cmp_fn = min - optimizer = opt.Adam(train_model.parameters(),lr = lr) - scheduler = opt.lr_scheduler.ExponentialLR(optimizer,lr_decay) + optimizer = opt.Adam(train_model.parameters(), lr=lr) + scheduler = opt.lr_scheduler.ExponentialLR(optimizer, lr_decay) train_model.train() not_updated = 0 total_loss = 0 @@ -451,16 +557,18 @@ def train(conf,shot_list_train,shot_list_validate,loader): makedirs_process_safe(os.path.dirname(model_path)) while e < num_epochs-1: scheduler.step() - print('\nEpoch {}/{}'.format(e,num_epochs)) - (step,ave_loss,curr_loss,num_so_far,effective_epochs) = train_epoch(train_model,data_gen,optimizer,loss_fn) + print('\nEpoch {}/{}'.format(e, num_epochs)) + (step, ave_loss, curr_loss, num_so_far, effective_epochs) = train_epoch( + train_model, data_gen, optimizer, loss_fn) e = effective_epochs - loader.verbose=False #True during the first iteration - # if task_index == 0: - # specific_builder.save_model_weights(train_model,int(round(e))) - torch.save(train_model.state_dict(),model_path) - _,_,_,roc_area,loss = make_predictions_and_evaluate_gpu(conf,shot_list_validate,loader) + loader.verbose = False # True during the first iteration + # if task_index == 0: + # specific_builder.save_model_weights(train_model,int(round(e))) + torch.save(train_model.state_dict(), model_path) + _, _, _, roc_area, loss = make_predictions_and_evaluate_gpu( + conf, shot_list_validate, loader) - best_so_far = cmp_fn(roc_area,best_so_far) + best_so_far = cmp_fn(roc_area, best_so_far) stop_training = False print('=========Summary======== for epoch{}'.format(step)) @@ -468,7 +576,7 @@ def train(conf,shot_list_train,shot_list_validate,loader): print('Validation Loss: {:.3e}'.format(loss)) print('Validation ROC: {:.4f}'.format(roc_area)) - if best_so_far != roc_area: #only save model weights if quantity we are tracking is improving + if best_so_far != roc_area: # only save model weights if quantity we are tracking is improving print("No improvement, still saving model") not_updated += 1 else: @@ -477,4 +585,3 @@ def train(conf,shot_list_train,shot_list_validate,loader): if not_updated > patience: print("Stopping training due to early stopping") break - From e14a6780b6060ffa474411696b140283630246b3 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 9 Oct 2019 18:39:39 -0500 Subject: [PATCH 108/272] Manually fix PEP 8 issues in torch_runner.py --- plasma/models/torch_runner.py | 178 +++++++++++++--------------------- 1 file changed, 65 insertions(+), 113 deletions(-) diff --git a/plasma/models/torch_runner.py b/plasma/models/torch_runner.py index f85cae87..6ffae00d 100644 --- a/plasma/models/torch_runner.py +++ b/plasma/models/torch_runner.py @@ -1,71 +1,32 @@ -import keras.callbacks as cbks +from __future__ import print_function from keras.utils.generic_utils import Progbar from torch.nn.utils import weight_norm import torch.optim as opt from torch.autograd import Variable import torch.nn as nn import torch -import hashlib from plasma.utils.downloading import makedirs_process_safe -from plasma.utils.state_reset import reset_states -from plasma.utils.evaluation import * from plasma.utils.performance import PerformanceAnalyzer -from plasma.models.loader import Loader, ProcessGenerator -from plasma.conf import conf -from sklearn.neural_network import MLPClassifier -from xgboost import XGBClassifier -import pathos.multiprocessing as mp +from plasma.utils.evaluation import get_loss_from_list from functools import partial import os -import datetime -import time -import sys import numpy as np -import matplotlib.pyplot as plt -from __future__ import print_function -import matplotlib -matplotlib.use('Agg') - -if sys.version_info[0] < 3: - from itertools import imap - -# leading to import errors: -#from hyperopt import hp, STATUS_OK -#from hyperas.distributions import conditional - model_filename = 'torch_model.pt' class FTCN(nn.Module): - def __init__( - self, - n_scalars, - n_profiles, - profile_size, - layer_sizes_spatial, - kernel_size_spatial, - linear_size, - output_size, - num_channels_tcn, - kernel_size_temporal, - dropout=0.1): + def __init__(self, n_scalars, n_profiles, profile_size, + layer_sizes_spatial, kernel_size_spatial, + linear_size, output_size, + num_channels_tcn, kernel_size_temporal, dropout=0.1): super(FTCN, self).__init__() - self.lin = InputBlock( - n_scalars, - n_profiles, - profile_size, - layer_sizes_spatial, - kernel_size_spatial, - linear_size, - dropout) + self.lin = InputBlock(n_scalars, n_profiles, profile_size, + layer_sizes_spatial, kernel_size_spatial, + linear_size, dropout) self.input_layer = TimeDistributed(self.lin, batch_first=True) - self.tcn = TCN( - linear_size, - output_size, - num_channels_tcn, - kernel_size_temporal, - dropout) + self.tcn = TCN(linear_size, output_size, num_channels_tcn, + kernel_size_temporal, dropout) self.model = nn.Sequential(self.input_layer, self.tcn) def forward(self, x): @@ -73,15 +34,8 @@ def forward(self, x): class InputBlock(nn.Module): - def __init__( - self, - n_scalars, - n_profiles, - profile_size, - layer_sizes, - kernel_size, - linear_size, - dropout=0.2): + def __init__(self, n_scalars, n_profiles, profile_size, layer_sizes, + kernel_size, linear_size, dropout=0.2): super(InputBlock, self).__init__() self.pooling_size = 2 self.n_scalars = n_scalars @@ -98,18 +52,15 @@ def __init__( input_size = n_profiles else: input_size = layer_sizes[i-1] - self.layers.append( - weight_norm( - nn.Conv1d( - input_size, - layer_size, - kernel_size))) + self.layers.append(weight_norm( + nn.Conv1d(input_size, layer_size, kernel_size))) self.layers.append(nn.ReLU()) self.conv_output_size = calculate_conv_output_size( self.conv_output_size, 0, 1, 1, kernel_size) self.layers.append(nn.MaxPool1d(kernel_size=self.pooling_size)) self.conv_output_size = calculate_conv_output_size( - self.conv_output_size, 0, 1, self.pooling_size, self.pooling_size) + self.conv_output_size, 0, 1, self.pooling_size, + self.pooling_size) self.layers.append(nn.Dropout2d(dropout)) self.net = nn.Sequential(*self.layers) self.conv_output_size = self.conv_output_size*layer_sizes[-1] @@ -294,7 +245,9 @@ def forward(self, x): # for i in range(iters): # x_,y_,mask_ = data_gen() # # print(y) -# x, y, mask = Variable(torch.from_numpy(x_).float()), Variable(torch.from_numpy(y_).float()),Variable(torch.from_numpy(mask_).byte()) +# x, y, mask = Variable(torch.from_numpy(x_).float()), +# Variable(torch.from_numpy(y_).float()),Variable(torch.from_numpy(mask_) +# . byte()) # # print(y) # optimizer.zero_grad() # # output = model(x.unsqueeze(0)).squeeze(0) @@ -312,7 +265,8 @@ def forward(self, x): # optimizer.step() # if i > 0 and i % log_step == 0: # cur_loss = total_loss / count -# print("Epoch {:2d} | lr {:.5f} | loss {:.5f}".format(0,lr, cur_loss)) +# print("Epoch {:2d} | lr {:.5f} | loss {:.5f}".format(0,lr, +# # cur_loss)) # total_loss = 0.0 # count = 0 @@ -351,8 +305,8 @@ def build_torch_model(conf): # lin = nn.Linear(input_size,intermediate_dim) n_scalars, n_profiles, profile_size = get_signal_dimensions(conf) - dim = n_scalars+n_profiles*profile_size - input_size = dim + # dim = n_scalars + n_profiles*profile_size + # input_size = dim output_size = 1 # intermediate_dim = 15 @@ -362,17 +316,9 @@ def build_torch_model(conf): num_channels_tcn = [10, 5, 3, 3] # [3]*5 kernel_size_temporal = 3 # 3 - model = FTCN( - n_scalars, - n_profiles, - profile_size, - layer_sizes_spatial, - kernel_size_spatial, - linear_size, - output_size, - num_channels_tcn, - kernel_size_temporal, - dropout) + model = FTCN(n_scalars, n_profiles, profile_size, layer_sizes_spatial, + kernel_size_spatial, linear_size, output_size, + num_channels_tcn, kernel_size_temporal, dropout) return model @@ -392,9 +338,10 @@ def get_signal_dimensions(conf): n_profiles += 1 is_1D_region = True else: - assert( - not is_1D_region), "make sure all use_signals are ordered such that 1D signals come last!" - assert(num_channels == 1) + assert not is_1D_region, ( + "make sure all use_signals are ordered such that ", + "1D signals come last!") + assert num_channels == 1 n_scalars += 1 is_1D_region = False return n_scalars, n_profiles, profile_size @@ -402,7 +349,8 @@ def get_signal_dimensions(conf): def apply_model_to_np(model, x): # return - # model(Variable(torch.from_numpy(x).float()).unsqueeze(0)).squeeze(0).data.numpy() + # model(Variable(torch.from_numpy(x).float()).unsqueeze(0)).squeeze( + # 0).data.numpy() return model(Variable(torch.from_numpy(x).float())).data.numpy() @@ -415,7 +363,7 @@ def make_predictions(conf, shot_list, loader, custom_path=None): else: model_path = custom_path inference_model.load_state_dict(torch.load(model_path)) - #shot_list = shot_list.random_sublist(10) + # shot_list = shot_list.random_sublist(10) y_prime = [] y_gold = [] @@ -425,7 +373,9 @@ def make_predictions(conf, shot_list, loader, custom_path=None): pbar = Progbar(num_shots) while True: x, y, mask, disr, lengths, num_so_far, num_total = next(generator) - #x, y, mask = Variable(torch.from_numpy(x_).float()), Variable(torch.from_numpy(y_).float()),Variable(torch.from_numpy(mask_).byte()) + # x, y, mask = Variable(torch.from_numpy(x_).float()), + # Variable(torch.from_numpy(y_).float()), + # Variable(torch.from_numpy(mask_).byte()) output = apply_model_to_np(inference_model, x) for batch_idx in range(x.shape[0]): curr_length = lengths[batch_idx] @@ -441,8 +391,8 @@ def make_predictions(conf, shot_list, loader, custom_path=None): return y_prime, y_gold, disruptive -def make_predictions_and_evaluate_gpu( - conf, shot_list, loader, custom_path=None): +def make_predictions_and_evaluate_gpu(conf, shot_list, loader, + custom_path=None): y_prime, y_gold, disruptive = make_predictions( conf, shot_list, loader, custom_path) analyzer = PerformanceAnalyzer(conf=conf) @@ -452,8 +402,8 @@ def make_predictions_and_evaluate_gpu( def get_model_path(conf): - return conf['paths']['model_save_path'] + 'torch/' + \ - model_filename # save_prepath + model_filename + return (conf['paths']['model_save_path'] + 'torch/' + + model_filename) # save_prepath + model_filename def train_epoch(model, data_gen, optimizer, loss_fn): @@ -485,8 +435,9 @@ def train_epoch(model, data_gen, optimizer, loss_fn): loss.backward() optimizer.step() step += 1 - print("[{}] [{}/{}] loss: {:.3f}, ave_loss: {:.3f}".format(step, - num_so_far-num_so_far_start, num_total, loss.data[0], total_loss/step)) + print("[{}] [{}/{}] loss: {:.3f}, ave_loss: {:.3f}".format( + step, num_so_far - num_so_far_start, num_total, loss.data[0], + total_loss/step)) if num_so_far-num_so_far_start >= num_total: break x_, y_, mask_, num_so_far, num_total = next(data_gen) @@ -497,19 +448,16 @@ def train(conf, shot_list_train, shot_list_validate, loader): np.random.seed(1) - #data_gen = ProcessGenerator(partial(loader.training_batch_generator_full_shot_partial_reset,shot_list=shot_list_train)()) + # data_gen = ProcessGenerator(partial( + # loader.training_batch_generator_full_shot_partial_reset,shot_list + # = shot_list_train)()) data_gen = partial( loader.training_batch_generator_full_shot_partial_reset, shot_list=shot_list_train)() - - print( - 'validate: {} shots, {} disruptive'.format( - len(shot_list_validate), - shot_list_validate.num_disruptive())) - print( - 'training: {} shots, {} disruptive'.format( - len(shot_list_train), - shot_list_train.num_disruptive())) + print('validate: {} shots, {} disruptive'.format( + len(shot_list_validate), shot_list_validate.num_disruptive())) + print('training: {} shots, {} disruptive'.format( + len(shot_list_train), shot_list_train.num_disruptive())) loader.set_inference_mode(False) @@ -521,16 +469,18 @@ def train(conf, shot_list_train, shot_list_validate, loader): num_epochs = conf['training']['num_epochs'] patience = conf['callbacks']['patience'] lr_decay = conf['model']['lr_decay'] - batch_size = conf['training']['batch_size'] + # batch_size = conf['training']['batch_size'] lr = conf['model']['lr'] - clipnorm = conf['model']['clipnorm'] + # clipnorm = conf['model']['clipnorm'] e = 0 # warmup_steps = conf['model']['warmup_steps'] # num_batches_minimum = conf['training']['num_batches_minimum'] # if 'adam' in conf['model']['optimizer']: # optimizer = MPIAdam(lr=lr) - # elif conf['model']['optimizer'] == 'sgd' or conf['model']['optimizer'] == 'tf_sgd': + # elif conf['model']['optimizer'] == 'sgd' or conf['model']['optimizer'] == + # 'tf_sgd': + # # optimizer = MPISGD(lr=lr) # elif 'momentum_sgd' in conf['model']['optimizer']: # optimizer = MPIMomentumSGD(lr=lr) @@ -550,16 +500,17 @@ def train(conf, shot_list_train, shot_list_validate, loader): scheduler = opt.lr_scheduler.ExponentialLR(optimizer, lr_decay) train_model.train() not_updated = 0 - total_loss = 0 - count = 0 + # total_loss = 0 + # count = 0 loss_fn = nn.MSELoss(size_average=True) model_path = get_model_path(conf) makedirs_process_safe(os.path.dirname(model_path)) - while e < num_epochs-1: + while e < num_epochs - 1: scheduler.step() print('\nEpoch {}/{}'.format(e, num_epochs)) - (step, ave_loss, curr_loss, num_so_far, effective_epochs) = train_epoch( - train_model, data_gen, optimizer, loss_fn) + (step, ave_loss, curr_loss, num_so_far, + effective_epochs) = train_epoch(train_model, data_gen, optimizer, + loss_fn) e = effective_epochs loader.verbose = False # True during the first iteration # if task_index == 0: @@ -570,13 +521,14 @@ def train(conf, shot_list_train, shot_list_validate, loader): best_so_far = cmp_fn(roc_area, best_so_far) - stop_training = False + # stop_training = False print('=========Summary======== for epoch{}'.format(step)) print('Training Loss numpy: {:.3e}'.format(ave_loss)) print('Validation Loss: {:.3e}'.format(loss)) print('Validation ROC: {:.4f}'.format(roc_area)) - if best_so_far != roc_area: # only save model weights if quantity we are tracking is improving + # only save model weights if the quantity we are tracking is improving + if best_so_far != roc_area: print("No improvement, still saving model") not_updated += 1 else: From 98cbac75122c49888a8384bba3e452e0855bc2c6 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 9 Oct 2019 19:02:21 -0500 Subject: [PATCH 109/272] Start using saner line breaks around fn defs and calls reduce # of lines by passing multiple arguments per line than what autopep8 does --- plasma/models/loader.py | 29 +--- plasma/models/mpi_runner.py | 39 ++--- plasma/models/shallow_runner.py | 25 +-- plasma/models/torch_runner.py | 88 +++-------- plasma/primitives/data.py | 80 +++------- plasma/utils/performance.py | 267 ++++++++++---------------------- 6 files changed, 153 insertions(+), 375 deletions(-) diff --git a/plasma/models/loader.py b/plasma/models/loader.py index 8ed8806d..39e92d27 100644 --- a/plasma/models/loader.py +++ b/plasma/models/loader.py @@ -107,13 +107,8 @@ def training_batch_generator(self, shot_list): num_so_far, num_total epoch += 1 - def fill_training_buffer( - self, - Xbuff, - Ybuff, - end_indices, - shot, - is_first_fill=False): + def fill_training_buffer(self, Xbuff, Ybuff, end_indices, shot, + is_first_fill=False): sig, res = self.get_signal_result_from_shot(shot) length = self.conf['model']['length'] if is_first_fill: # cut signal to random position @@ -388,20 +383,16 @@ def fill_batch_queue(self, shot_list, queue): def training_batch_generator_process(self, shot_list): queue = mp.Queue() - proc = mp.Process( - target=self.fill_batch_queue, args=( - shot_list, queue)) + proc = mp.Process(target=self.fill_batch_queue, + args=(shot_list, queue)) proc.start() while True: yield queue.get(True) proc.join() queue.close() - def load_as_X_y_list( - self, - shot_list, - verbose=False, - prediction_mode=False): + def load_as_X_y_list(self, shot_list, verbose=False, + prediction_mode=False): """ The method turns a ShotList into a set of equal-sized patches which contain a number of examples that is a multiple of the batch size. @@ -696,12 +687,8 @@ def arange_patches(self, sig_patches, res_patches): y_list.append(y) return X_list, y_list - def arange_patches_single( - self, - sig_patches, - res_patches, - prediction_mode=False, - custom_batch_size=None): + def arange_patches_single(self, sig_patches, res_patches, + prediction_mode=False, custom_batch_size=None): if prediction_mode: num_timesteps = self.conf['model']['pred_length'] batch_size = self.conf['model']['pred_batch_size'] diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 00bf4704..01949c44 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -54,8 +54,8 @@ if backend == 'tf' or backend == 'tensorflow': if NUM_GPUS > 1: - os.environ['CUDA_VISIBLE_DEVICES'] = '{}'.format( - MY_GPU) # ,mode=NanGuardMode' + os.environ['CUDA_VISIBLE_DEVICES'] = '{}'.format(MY_GPU) + # ,mode=NanGuardMode' os.environ['KERAS_BACKEND'] = 'tensorflow' import tensorflow as tf from keras.backend.tensorflow_backend import set_session @@ -749,8 +749,8 @@ def mpi_make_predictions(conf, shot_list, loader, custom_path=None): return y_prime_global, y_gold_global, disruptive_global -def mpi_make_predictions_and_evaluate( - conf, shot_list, loader, custom_path=None): +def mpi_make_predictions_and_evaluate(conf, shot_list, loader, + custom_path=None): y_prime, y_gold, disruptive = mpi_make_predictions( conf, shot_list, loader, custom_path) analyzer = PerformanceAnalyzer(conf=conf) @@ -823,17 +823,9 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, shot_list=shot_list_train) print("warmup {}".format(warmup_steps)) - mpi_model = MPIModel( - train_model, - optimizer, - comm, - batch_generator, - batch_size, - lr=lr, - warmup_steps=warmup_steps, - num_batches_minimum=num_batches_minimum, - conf=conf - ) + mpi_model = MPIModel(train_model, optimizer, comm, batch_generator, + batch_size, lr=lr, warmup_steps=warmup_steps, + num_batches_minimum=num_batches_minimum, conf=conf) mpi_model.compile( conf['model']['optimizer'], clipnorm, @@ -940,13 +932,11 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, # tensorboard if backend != 'theano': - val_generator = partial( - loader.training_batch_generator, - shot_list=shot_list_validate)() + val_generator = partial(loader.training_batch_generator, + shot_list=shot_list_validate)() val_steps = 1 - tensorboard.on_epoch_end( - val_generator, val_steps, int( - round(e)), epoch_logs) + tensorboard.on_epoch_end(val_generator, val_steps, + int(round(e)), epoch_logs) print_unique("end epoch {} 0".format(e)) stop_training = comm.bcast(stop_training, root=0) @@ -972,11 +962,8 @@ def get_stop_training(callbacks): class TensorBoard(object): - def __init__(self, log_dir='./logs', - histogram_freq=0, - validation_steps=0, - write_graph=True, - write_grads=False): + def __init__(self, log_dir='./logs', histogram_freq=0, validation_steps=0, + write_graph=True, write_grads=False): if K.backend() != 'tensorflow': raise RuntimeError('TensorBoard callback only works ' 'with the TensorFlow backend.') diff --git a/plasma/models/shallow_runner.py b/plasma/models/shallow_runner.py index aaabd2b9..fc775414 100644 --- a/plasma/models/shallow_runner.py +++ b/plasma/models/shallow_runner.py @@ -71,12 +71,8 @@ def get_sample_probs(self, shot_list, num_samples): return val, val return sample_prob_d, sample_prob_nd - def load_shots( - self, - shot_list, - is_inference=False, - as_list=False, - num_samples=np.Inf): + def load_shots(self, shot_list, is_inference=False, as_list=False, + num_samples=np.Inf): X = [] Y = [] Disr = [] @@ -153,7 +149,6 @@ def process(self, shot): return X, Y, disr def get_X(self, shot): - use_signals = self.loader.conf['paths']['use_signals'] sig_sample = shot.signals_dict[use_signals[0]] if len(shot.ttd.shape) == 1: @@ -181,13 +176,8 @@ def get_Y(self, shot): offset = self.timesteps - 1 return np.round(shot.ttd[offset:, 0]).astype(np.int) - def load_shot( - self, - shot, - is_inference=False, - sample_prob_d=1.0, - sample_prob_nd=1.0): - + def load_shot(self, shot, is_inference=False, sample_prob_d=1.0, + sample_prob_nd=1.0): X, Y, disr = self.process(shot) # cut shot ends if we are supposed to @@ -200,10 +190,9 @@ def load_shot( if disr: sample_prob = sample_prob_d if sample_prob < 1.0: - indices = np.sort( - np.random.choice(np.array(range(len(Y))), - int(round(sample_prob*len(Y))), - replace=False)) + indices = np.sort(np.random.choice(np.array(range(len(Y))), + int(round(sample_prob*len(Y))), + replace=False)) X = X[indices] Y = Y[indices] return X, Y, disr diff --git a/plasma/models/torch_runner.py b/plasma/models/torch_runner.py index 6ffae00d..5c85e134 100644 --- a/plasma/models/torch_runner.py +++ b/plasma/models/torch_runner.py @@ -66,10 +66,8 @@ def __init__(self, n_scalars, n_profiles, profile_size, layer_sizes, self.conv_output_size = self.conv_output_size*layer_sizes[-1] self.linear_layers = [] - print( - "Final feature size = {}".format( - self.n_scalars - + self.conv_output_size)) + print("Final feature size = {}".format(self.n_scalars + + self.conv_output_size)) self.linear_layers.append( nn.Linear( self.conv_output_size @@ -111,8 +109,8 @@ def forward(self, x): def calculate_conv_output_size(L_in, padding, dilation, stride, kernel_size): - return int(np.floor((L_in + 2*padding - dilation - * (kernel_size-1) - 1)*1.0/stride + 1)) + return int(np.floor( + (L_in + 2*padding - dilation * (kernel_size-1) - 1)*1.0/stride + 1)) class Chomp1d(nn.Module): @@ -125,51 +123,28 @@ def forward(self, x): class TemporalBlock(nn.Module): - def __init__( - self, - n_inputs, - n_outputs, - kernel_size, - stride, - dilation, - padding, - dropout=0.2): + def __init__(self, n_inputs, n_outputs, kernel_size, stride, dilation, + padding, dropout=0.2): super(TemporalBlock, self).__init__() - self.conv1 = weight_norm( - nn.Conv1d( - n_inputs, - n_outputs, - kernel_size, - stride=stride, - padding=padding, - dilation=dilation)) + self.conv1 = weight_norm(nn.Conv1d( + n_inputs, n_outputs, kernel_size, stride=stride, padding=padding, + dilation=dilation)) self.chomp1 = Chomp1d(padding) self.relu1 = nn.ReLU() self.dropout1 = nn.Dropout2d(dropout) - self.conv2 = weight_norm( - nn.Conv1d( - n_outputs, - n_outputs, - kernel_size, - stride=stride, - padding=padding, - dilation=dilation)) + self.conv2 = weight_norm(nn.Conv1d( + n_outputs, n_outputs, kernel_size, stride=stride, padding=padding, + dilation=dilation)) self.chomp2 = Chomp1d(padding) self.relu2 = nn.ReLU() self.dropout2 = nn.Dropout2d(dropout) - self.net = nn.Sequential( - self.conv1, - self.chomp1, - self.relu1, - self.dropout1, - self.conv2, - self.chomp2, - self.relu2, - self.dropout2) - self.downsample = nn.Conv1d( - n_inputs, n_outputs, 1) if n_inputs != n_outputs else None + self.net = nn.Sequential(self.conv1, self.chomp1, self.relu1, + self.dropout1, self.conv2, self.chomp2, + self.relu2, self.dropout2) + self.downsample = (nn.Conv1d(n_inputs, n_outputs, 1) + if n_inputs != n_outputs else None) self.relu = nn.ReLU() self.init_weights() @@ -196,11 +171,8 @@ def __init__(self, num_inputs, num_channels, kernel_size=2, dropout=0.2): dilation_size = 2 ** i in_channels = num_inputs if i == 0 else num_channels[i-1] out_channels = num_channels[i] - layers += [TemporalBlock(in_channels, - out_channels, - kernel_size, - stride=1, - dilation=dilation_size, + layers += [TemporalBlock(in_channels, out_channels, kernel_size, + stride=1, dilation=dilation_size, padding=(kernel_size-1) * dilation_size, dropout=dropout)] @@ -211,19 +183,11 @@ def forward(self, x): class TCN(nn.Module): - def __init__( - self, - input_size, - output_size, - num_channels, - kernel_size, - dropout): + def __init__(self, input_size, output_size, num_channels, kernel_size, + dropout): super(TCN, self).__init__() - self.tcn = TemporalConvNet( - input_size, - num_channels, - kernel_size, - dropout=dropout) + self.tcn = TemporalConvNet(input_size, num_channels, kernel_size, + dropout=dropout) self.linear = nn.Linear(num_channels[-1], output_size) # self.sig = nn.Sigmoid() @@ -278,7 +242,6 @@ def __init__(self, module, batch_first=False): self.batch_first = batch_first def forward(self, x): - if len(x.size()) <= 2: return self.module(x) @@ -301,8 +264,7 @@ def forward(self, x): def build_torch_model(conf): dropout = conf['model']['dropout_prob'] -# dim = 10 - + # dim = 10 # lin = nn.Linear(input_size,intermediate_dim) n_scalars, n_profiles, profile_size = get_signal_dimensions(conf) # dim = n_scalars + n_profiles*profile_size @@ -445,9 +407,7 @@ def train_epoch(model, data_gen, optimizer, loss_fn): def train(conf, shot_list_train, shot_list_validate, loader): - np.random.seed(1) - # data_gen = ProcessGenerator(partial( # loader.training_batch_generator_full_shot_partial_reset,shot_list # = shot_list_train)()) diff --git a/plasma/primitives/data.py b/plasma/primitives/data.py index 922cce5a..4710ff77 100644 --- a/plasma/primitives/data.py +++ b/plasma/primitives/data.py @@ -222,30 +222,13 @@ def __repr__(self): class ProfileSignal(Signal): - def __init__( - self, - description, - paths, - machines, - tex_label=None, - causal_shifts=None, - mapping_range=( - 0, - 1), - num_channels=32, - data_avail_tolerances=None, - is_strictly_positive=False, - mapping_paths=None): - super( - ProfileSignal, - self).__init__( - description, - paths, - machines, - tex_label, - causal_shifts, - is_ip=False, - data_avail_tolerances=data_avail_tolerances, + def __init__(self, description, paths, machines, tex_label=None, + causal_shifts=None, mapping_range=(0, 1), num_channels=32, + data_avail_tolerances=None, is_strictly_positive=False, + mapping_paths=None): + super(ProfileSignal, self).__init__( + description, paths, machines, tex_label, causal_shifts, + is_ip=False, data_avail_tolerances=data_avail_tolerances, is_strictly_positive=is_strictly_positive, mapping_paths=mapping_paths) self.mapping_range = mapping_range @@ -261,10 +244,8 @@ def load_data(self, prepath, shot, dtype='float32'): # time is stored twice, once for mapping and once for signal T = data.shape[0]//2 mapping = data[:T, 1:] - remapping = np.linspace( - self.mapping_range[0], - self.mapping_range[1], - self.num_channels) + remapping = np.linspace(self.mapping_range[0], self.mapping_range[1], + self.num_channels) t = data[:T, 0] sig = data[T:, 1:] if sig.shape[1] < 2: @@ -289,8 +270,8 @@ def load_data(self, prepath, shot, dtype='float32'): _, order = np.unique(mapping[i, :], return_index=True) if sig[i, order].shape[0] > 2: # ext = 0 is extrapolation, ext = 3 is boundary value. - f = UnivariateSpline( - mapping[i, order], sig[i, order], s=0, k=1, ext=3) + f = UnivariateSpline(mapping[i, order], sig[i, order], s=0, + k=1, ext=3) sig_interp[i, :] = f(remapping) else: print('Signal {}, shot {} '.format(self.description, @@ -302,8 +283,8 @@ def load_data(self, prepath, shot, dtype='float32'): return t, sig_interp, True def fetch_data(self, machine, shot_num, c): - time, data, mapping, success = self.fetch_data_basic( - machine, shot_num, c) + time, data, mapping, success = self.fetch_data_basic(machine, shot_num, + c) path = self.get_path(machine) mapping_path = self.get_mapping_path(machine) @@ -335,26 +316,12 @@ def fetch_data(self, machine, shot_num, c): class ChannelSignal(Signal): - def __init__( - self, - description, - paths, - machines, - tex_label=None, - causal_shifts=None, - data_avail_tolerances=None, - is_strictly_positive=False, - mapping_paths=None): - super( - ChannelSignal, - self).__init__( - description, - paths, - machines, - tex_label, - causal_shifts, - is_ip=False, - data_avail_tolerances=data_avail_tolerances, + def __init__(self, description, paths, machines, tex_label=None, + causal_shifts=None, data_avail_tolerances=None, + is_strictly_positive=False, mapping_paths=None): + super(ChannelSignal, self).__init__( + description, paths, machines, tex_label, causal_shifts, + is_ip=False, data_avail_tolerances=data_avail_tolerances, is_strictly_positive=is_strictly_positive, mapping_paths=mapping_paths) nums, new_paths = self.get_channel_nums(paths) @@ -407,13 +374,8 @@ def get_file_path(self, prepath, machine, shot_number): class Machine(object): - def __init__( - self, - name, - server, - fetch_data_fn, - max_cores=8, - current_threshold=0): + def __init__(self, name, server, fetch_data_fn, max_cores=8, + current_threshold=0): self.name = name self.server = server self.max_cores = max_cores diff --git a/plasma/utils/performance.py b/plasma/utils/performance.py index 3dd52474..b88774ec 100644 --- a/plasma/utils/performance.py +++ b/plasma/utils/performance.py @@ -67,8 +67,8 @@ def get_metrics_vs_p_thresh(self, mode): return self.get_metrics_vs_p_thresh_custom( all_preds, all_truths, all_disruptive) - def get_metrics_vs_p_thresh_custom( - self, all_preds, all_truths, all_disruptive): + def get_metrics_vs_p_thresh_custom(self, all_preds, all_truths, + all_disruptive): return self.get_metrics_vs_p_thresh_fast( all_preds, all_truths, all_disruptive) P_thresh_range = self.get_p_thresh_range() @@ -113,8 +113,8 @@ def get_p_thresh_range(self): # print(np.unique(self.p_thresh_range)) return self.p_thresh_range - def get_metrics_vs_p_thresh_fast( - self, all_preds, all_truths, all_disruptive): + def get_metrics_vs_p_thresh_fast(self, all_preds, all_truths, + all_disruptive): all_disruptive = np.array(all_disruptive) if self.pred_train is not None: p_thresh_range = self.get_p_thresh_range() @@ -225,9 +225,8 @@ def get_threshold_arrays(self, preds, truths, disruptives): return (np.array(d_early_thresholds), np.array(d_correct_thresholds), np.array(d_late_thresholds), np.array(nd_thresholds)) - def summarize_shot_prediction_stats_by_mode( - self, P_thresh, mode, verbose=False): - + def summarize_shot_prediction_stats_by_mode(self, P_thresh, mode, + verbose=False): if mode == 'train': all_preds = self.pred_train all_truths = self.truth_train @@ -241,20 +240,13 @@ def summarize_shot_prediction_stats_by_mode( return self.summarize_shot_prediction_stats( P_thresh, all_preds, all_truths, all_disruptive, verbose) - def summarize_shot_prediction_stats( - self, - P_thresh, - all_preds, - all_truths, - all_disruptive, - verbose=False): + def summarize_shot_prediction_stats(self, P_thresh, all_preds, all_truths, + all_disruptive, verbose=False): TPs, FPs, FNs, TNs, earlies, lates = (0, 0, 0, 0, 0, 0) - for i in range(len(all_preds)): preds = all_preds[i] truth = all_truths[i] is_disruptive = all_disruptive[i] - TP, FP, FN, TN, early, late = self.get_shot_prediction_stats( P_thresh, preds, truth, is_disruptive) TPs += TP @@ -334,8 +326,8 @@ def create_acceptable_region(self, truth, mode): acceptable[-acceptable_timesteps:] = True return acceptable - def get_accuracy_and_fp_rate_from_stats( - self, tp, fp, fn, tn, early, late, verbose=False): + def get_accuracy_and_fp_rate_from_stats(self, tp, fp, fn, tn, early, late, + verbose=False): total = tp + fp + fn + tn + early + late disr = early + late + tp + fn nondisr = fp + tn @@ -386,15 +378,15 @@ def load_ith_file(self): # normalized shot ttd. self.conf['data']['T_warning'] = self.saved_conf['data']['T_warning'] for mode in ['test', 'train']: - print( - '{}: loaded {} shot ({}) disruptive'.format( - mode, - self.get_num_shots(mode), - self.get_num_disruptive_shots(mode))) + print('{}: loaded {} shot ({}) disruptive'.format( + mode, self.get_num_shots(mode), + self.get_num_disruptive_shots(mode))) if self.verbose: self.print_conf() - # self.assert_same_lists(self.shot_list_test,self.truth_test,self.disruptive_test) - # self.assert_same_lists(self.shot_list_train,self.truth_train,self.disruptive_train) + # self.assert_same_lists(self.shot_list_test, self.truth_test, + # self.disruptive_test) + # self.assert_same_lists(self.shot_list_train, self.truth_train, + # self.disruptive_train) def assert_same_lists(self, shot_list, truth_arr, disr_arr): assert(len(shot_list) == len(truth_arr)) @@ -423,12 +415,8 @@ def get_num_disruptive_shots(self, mode): if mode == 'train': return sum(self.disruptive_train) - def hist_alarms( - self, - alarms, - title_str='alarms', - save_figure=False, - linestyle='-'): + def hist_alarms(self, alarms, title_str='alarms', save_figure=False, + linestyle='-'): fontsize = 15 T_min_warn = self.T_min_warn T_max_warn = self.T_max_warn @@ -448,7 +436,6 @@ def hist_alarms( plt.step(np.concatenate((alarms[::-1], alarms[[0]])), 1.0*np.arange(alarms.size+1)/(alarms.size), linestyle=linestyle, linewidth=1.5) - plt.gca().set_xscale('log') plt.axvline(T_min_warn, color='r', linewidth=0.5) # if T_max_warn < np.max(alarms): @@ -463,10 +450,8 @@ def hist_alarms( plt.setp(plt.gca().get_xticklabels(), fontsize=fontsize) plt.show() if save_figure: - plt.savefig( - 'accum_disruptions.png', - dpi=200, - bbox_inches='tight') + plt.savefig('accum_disruptions.png', dpi=200, + bbox_inches='tight') else: print(title_str + ": No alarms!") @@ -640,12 +625,8 @@ def get_prediction_type(self, TP, FP, FN, TN, early, late): elif late: return 'late' - def plot_individual_shot( - self, - P_thresh_opt, - shot_num, - normalize=True, - plot_signals=True): + def plot_individual_shot(self, P_thresh_opt, shot_num, normalize=True, + plot_signals=True): success = False for mode in ['test', 'train']: if mode == 'test': @@ -663,28 +644,20 @@ def plot_individual_shot( t = truth[i] p = pred[i] is_disr = is_disruptive[i] - TP, FP, FN, TN, early, late = ( self.get_shot_prediction_stats(P_thresh_opt, p, t, is_disr)) prediction_type = self.get_prediction_type(TP, FP, FN, TN, early, late) print(prediction_type) - self.plot_shot( - shot, - True, - normalize, - t, - p, - P_thresh_opt, - prediction_type, - extra_filename='_indiv') + self.plot_shot(shot, True, normalize, t, p, P_thresh_opt, + prediction_type, extra_filename='_indiv') success = True if not success: print("Shot {} not found".format(shot_num)) - def get_prediction_type_for_individual_shot( - self, P_thresh, shot, mode='test'): + def get_prediction_type_for_individual_shot(self, P_thresh, shot, + mode='test'): p, t, is_disr = self.get_pred_truth_disr_by_shot(shot) TP, FP, FN, TN, early, late = self.get_shot_prediction_stats( @@ -692,15 +665,9 @@ def get_prediction_type_for_individual_shot( prediction_type = self.get_prediction_type(TP, FP, FN, TN, early, late) return prediction_type - def example_plots( - self, - P_thresh_opt, - mode='test', - types_to_plot=['FP'], - max_plot=5, - normalize=True, - plot_signals=True, - extra_filename=''): + def example_plots(self, P_thresh_opt, mode='test', types_to_plot=['FP'], + max_plot=5, normalize=True, plot_signals=True, + extra_filename=''): if mode == 'test': pred = self.pred_test truth = self.truth_test @@ -731,51 +698,30 @@ def example_plots( if (('any' in types_to_plot or prediction_type in types_to_plot) and plotted < max_plot): if plot_signals: - self.plot_shot( - shot, - True, - normalize, - t, - p, - P_thresh_opt, - prediction_type, - extra_filename=extra_filename) + self.plot_shot(shot, True, normalize, t, p, P_thresh_opt, + prediction_type, + extra_filename=extra_filename) else: plt.figure() plt.semilogy((t+0.001)[::-1], label='ground truth') plt.plot(p[::-1], 'g', label='neural net prediction') - plt.axvline( - self.T_min_warn, - color='r', - label='max warning time') - plt.axvline( - self.T_max_warn, - color='r', - label='min warning time') - plt.axhline( - P_thresh_opt, - color='k', - label='trigger threshold') + plt.axvline(self.T_min_warn, color='r', + label='max warning time') + plt.axvline(self.T_max_warn, color='r', + label='min warning time') + plt.axhline(P_thresh_opt, color='k', + label='trigger threshold') plt.xlabel('TTD [ms]') plt.legend(loc=(1.0, 0.6)) plt.ylim([1e-7, 1.1e0]) plt.grid() - plt.savefig( - 'fig_{}.png'.format( - shot.number), - bbox_inches='tight') + plt.savefig('fig_{}.png'.format(shot.number), + bbox_inches='tight') plotted += 1 - def plot_shot( - self, - shot, - save_fig=True, - normalize=True, - truth=None, - prediction=None, - P_thresh_opt=None, - prediction_type='', - extra_filename=''): + def plot_shot(self, shot, save_fig=True, normalize=True, truth=None, + prediction=None, P_thresh_opt=None, prediction_type='', + extra_filename=''): if self.normalizer is None and normalize: if self.conf is not None: self.saved_conf['paths']['normalizer_path'] = ( @@ -832,16 +778,10 @@ def plot_shot( ax.set_yticks([0, num_channels/2]) ax.set_yticklabels(["0", "0.5"]) ax.set_ylabel("$\\rho$", size=fontsize) - ax.legend( - loc="best", - labelspacing=0.1, - fontsize=fontsize, - frameon=False) - ax.axvline( - len(truth) - - self.T_min_warn, - color='r', - linewidth=0.5) + ax.legend(loc="best", labelspacing=0.1, fontsize=fontsize, + frameon=False) + ax.axvline(len(truth) - self.T_min_warn, color='r', + linewidth=0.5) plt.setp(ax.get_xticklabels(), visible=False) plt.setp(ax.get_yticklabels(), fontsize=fontsize) f.subplots_adjust(hspace=0) @@ -881,36 +821,22 @@ def plot_shot( plt.xlim([lower_lim, len(truth)]) # plt.savefig("{}.png".format(num),dpi=200,bbox_inches="tight") if save_fig: - plt.savefig( - 'sig_fig_{}{}.png'.format( - shot.number, - extra_filename), - bbox_inches='tight') - np.savez( - 'sig_{}{}.npz'.format( - shot.number, - extra_filename), - shot=shot, - T_min_warn=self.T_min_warn, - T_max_warn=self.T_max_warn, - prediction=prediction, - truth=truth, - use_signals=use_signals, - P_thresh=P_thresh_opt) + plt.savefig('sig_fig_{}{}.png'.format(shot.number, + extra_filename), + bbox_inches='tight') + np.savez('sig_{}{}.npz'.format(shot.number, + extra_filename), + shot=shot, T_min_warn=self.T_min_warn, + T_max_warn=self.T_max_warn, prediction=prediction, + truth=truth, use_signals=use_signals, + P_thresh=P_thresh_opt) # plt.show() else: print("Shot hasn't been processed") - def plot_shot_old( - self, - shot, - save_fig=True, - normalize=True, - truth=None, - prediction=None, - P_thresh_opt=None, - prediction_type='', - extra_filename=''): + def plot_shot_old(self, shot, save_fig=True, normalize=True, truth=None, + prediction=None, P_thresh_opt=None, prediction_type='', + extra_filename=''): if self.normalizer is None and normalize: if self.conf is not None: self.saved_conf['paths']['normalizer_path'] = ( @@ -977,63 +903,34 @@ def plot_shot_old( plt.setp(ax.get_yticklabels(), fontsize=7) # ax.grid() if save_fig: - plt.savefig( - 'sig_fig_{}{}.png'.format( - shot.number, - extra_filename), - bbox_inches='tight') - np.savez( - 'sig_{}{}.npz'.format( - shot.number, - extra_filename), - shot=shot, - T_min_warn=self.T_min_warn, - T_max_warn=self.T_max_warn, - prediction=prediction, - truth=truth, - use_signals=use_signals, - P_thresh=P_thresh_opt) + plt.savefig('sig_fig_{}{}.png'.format( + shot.number, extra_filename), bbox_inches='tight') + np.savez('sig_{}{}.npz'.format(shot.number, extra_filename), + shot=shot, T_min_warn=self.T_min_warn, + T_max_warn=self.T_max_warn, prediction=prediction, + truth=truth, use_signals=use_signals, + P_thresh=P_thresh_opt) plt.close() else: print("Shot hasn't been processed") - def tradeoff_plot( - self, - accuracy_range, - missed_range, - fp_range, - early_alarm_range, - save_figure=False, - plot_string='', - linestyle="-"): + def tradeoff_plot(self, accuracy_range, missed_range, fp_range, + early_alarm_range, save_figure=False, plot_string='', + linestyle="-"): fontsize = 15 plt.figure() P_thresh_range = self.get_p_thresh_range() # semilogx(P_thresh_range,accuracy_range,label="accuracy") if self.pred_ttd: - plt.semilogx(abs(P_thresh_range[::-1]), - missed_range, - 'r', - label="missed", - linestyle=linestyle) - plt.plot(abs(P_thresh_range[::-1]), - fp_range, - 'k', - label="false positives", - linestyle=linestyle) + plt.semilogx(abs(P_thresh_range[::-1]), missed_range, 'r', + label="missed", linestyle=linestyle) + plt.plot(abs(P_thresh_range[::-1]), fp_range, 'k', + label="false positives", linestyle=linestyle) else: - plt.plot( - P_thresh_range, - missed_range, - 'r', - label="missed", - linestyle=linestyle) - plt.plot( - P_thresh_range, - fp_range, - 'k', - label="false positives", - linestyle=linestyle) + plt.plot(P_thresh_range, missed_range, 'r', label="missed", + linestyle=linestyle) + plt.plot(P_thresh_range, fp_range, 'k', label="false positives", + linestyle=linestyle) # plot(P_thresh_range,early_alarm_range,'c',label="early alarms") plt.legend(loc=(1.0, .6)) plt.xlabel('Alarm threshold', size=fontsize) @@ -1062,12 +959,8 @@ def tradeoff_plot( plt.ylim([0, 1]) if save_figure: plt.savefig(title_str + '_roc.png', bbox_inches='tight', dpi=200) - print( - 'ROC area ({}) is {}'.format( - plot_string, - self.roc_from_missed_fp( - missed_range, - fp_range))) + print('ROC area ({}) is {}'.format( + plot_string, self.roc_from_missed_fp(missed_range, fp_range))) return P_thresh_range, missed_range, fp_range def get_pred_truth_disr_by_shot(self, shot): From 6a30ddcd7bb56be138b2da9715027fa4d0124ce7 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 10 Oct 2019 10:08:07 -0500 Subject: [PATCH 110/272] Re-remove bidirectional RNN Repeat of 76a566ecfade9cebcbfcc3c103c8c011b336856e 735e1c26a2e9269b14e35dfc31db1baac5d08c44 46f2df9e11057d053821775a5bf3d68b875728e7 --- examples/conf.yaml | 1 - plasma/models/builder.py | 39 ++++++++++----------------------------- 2 files changed, 10 insertions(+), 30 deletions(-) diff --git a/examples/conf.yaml b/examples/conf.yaml index 3b0abce8..b9116a5c 100644 --- a/examples/conf.yaml +++ b/examples/conf.yaml @@ -55,7 +55,6 @@ data: model: loss_scale_factor: 1.0 - use_bidirectional: false use_batch_norm: false torch: False shallow: True diff --git a/plasma/models/builder.py b/plasma/models/builder.py index dc49e0ef..d7b2c0a8 100644 --- a/plasma/models/builder.py +++ b/plasma/models/builder.py @@ -5,7 +5,7 @@ Dense, Activation, Dropout, Lambda, Reshape, Flatten, Permute, # RepeatVector ) -from keras.layers import LSTM, SimpleRNN, Bidirectional, BatchNormalization +from keras.layers import LSTM, SimpleRNN, BatchNormalization from keras.layers.convolutional import Convolution1D from keras.layers.pooling import MaxPooling1D # from keras.utils.data_utils import get_file @@ -82,7 +82,6 @@ def build_model(self, predict, custom_batch_size=None): conf = self.conf model_conf = conf['model'] rnn_size = model_conf['rnn_size'] - use_bidirectional = model_conf['use_bidirectional'] rnn_type = model_conf['rnn_type'] regularization = model_conf['regularization'] dense_regularization = model_conf['dense_regularization'] @@ -249,33 +248,15 @@ def slicer_output_shape(input_shape, indices): # pre_rnn_model.summary() x_input = Input(batch_shape=batch_input_shape) x_in = TimeDistributed(pre_rnn_model)(x_input) - - if use_bidirectional: - for _ in range(model_conf['rnn_layers']): - x_in = Bidirectional( - rnn_model( - rnn_size, - return_sequences=return_sequences, - stateful=stateful, - kernel_regularizer=l2(regularization), - recurrent_regularizer=l2(regularization), - bias_regularizer=l2(regularization), - dropout=dropout_prob, - recurrent_dropout=dropout_prob))(x_in) - x_in = Dropout(dropout_prob)(x_in) - else: - for _ in range(model_conf['rnn_layers']): - x_in = rnn_model( - rnn_size, - return_sequences=return_sequences, - # batch_input_shape=batch_input_shape, - stateful=stateful, - kernel_regularizer=l2(regularization), - recurrent_regularizer=l2(regularization), - bias_regularizer=l2(regularization), - dropout=dropout_prob, - recurrent_dropout=dropout_prob)(x_in) - x_in = Dropout(dropout_prob)(x_in) + for _ in range(model_conf['rnn_layers']): + x_in = rnn_model( + rnn_size, return_sequences=return_sequences, + # batch_input_shape=batch_input_shape, + stateful=stateful, kernel_regularizer=l2(regularization), + recurrent_regularizer=l2(regularization), + bias_regularizer=l2(regularization), dropout=dropout_prob, + recurrent_dropout=dropout_prob)(x_in) + x_in = Dropout(dropout_prob)(x_in) if return_sequences: # x_out = TimeDistributed(Dense(100,activation='tanh')) (x_in) x_out = TimeDistributed( From 9fbe0f8c09bf793b477a9b1b557e2fa1cb177468 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 10 Oct 2019 15:04:32 -0500 Subject: [PATCH 111/272] Remove erroneous imports from runner.py of lr, tf Introduced by style changes in deddb520c861e6693811c90b718fcc843d395113 --- plasma/models/runner.py | 164 ++++++++++++++-------------------------- 1 file changed, 55 insertions(+), 109 deletions(-) diff --git a/plasma/models/runner.py b/plasma/models/runner.py index 0b18ef9f..f616f54a 100644 --- a/plasma/models/runner.py +++ b/plasma/models/runner.py @@ -1,5 +1,5 @@ from plasma.utils.state_reset import reset_states -from plasma.utils.evaluation import lr, tf, get_loss_from_list +from plasma.utils.evaluation import get_loss_from_list from plasma.utils.performance import PerformanceAnalyzer from plasma.models.loader import Loader, ProcessGenerator from plasma.conf import conf @@ -29,14 +29,10 @@ def train(conf, shot_list_train, shot_list_validate, loader, validation_losses = [] validation_roc = [] training_losses = [] - print( - 'validate: {} shots, {} disruptive'.format( - len(shot_list_validate), - shot_list_validate.num_disruptive())) - print( - 'training: {} shots, {} disruptive'.format( - len(shot_list_train), - shot_list_train.num_disruptive())) + print('validate: {} shots, {} disruptive'.format( + len(shot_list_validate), shot_list_validate.num_disruptive())) + print('training: {} shots, {} disruptive'.format( + len(shot_list_train), shot_list_train.num_disruptive())) if backend == 'tf' or backend == 'tensorflow': first_time = "tensorflow" not in sys.modules @@ -59,9 +55,8 @@ def train(conf, shot_list_train, shot_list_validate, loader, specific_builder = builder.ModelBuilder(conf) train_model = specific_builder.build_model(False) print('Compile model', end='') - train_model.compile( - optimizer=optimizer_class(), - loss=conf['data']['target'].loss) + train_model.compile(optimizer=optimizer_class(), + loss=conf['data']['target'].loss) print('...done') # load the latest epoch we did. Returns -1 if none exist yet @@ -87,7 +82,7 @@ def train(conf, shot_list_train, shot_list_validate, loader, best_so_far = np.inf cmp_fn = min - while e < num_epochs-1: + while e < (num_epochs - 1): e += 1 print('\nEpoch {}/{}'.format(e+1, num_epochs)) pbar = Progbar(len(shot_list_train)) @@ -114,10 +109,8 @@ def train(conf, shot_list_train, shot_list_validate, loader, if np.any(batches_to_reset): reset_states(train_model, batches_to_reset) if not is_warmup_period: - num_so_far = num_so_far_accum+num_so_far_curr - + num_so_far = num_so_far_accum + num_so_far_curr num_batches_current += 1 - loss = train_model.train_on_batch(batch_xs, batch_ys) training_losses_tmp.append(loss) pbar.add(num_so_far - num_so_far_old, @@ -127,7 +120,7 @@ def train(conf, shot_list_train, shot_list_validate, loader, _ = train_model.predict( batch_xs, batch_size=conf['training']['batch_size']) - e = e_start+1.0*num_so_far/num_total + e = e_start + 1.0*num_so_far/num_total sys.stdout.flush() ave_loss = np.mean(training_losses_tmp) training_losses.append(ave_loss) @@ -144,13 +137,13 @@ def train(conf, shot_list_train, shot_list_validate, loader, epoch_logs['val_roc'] = roc_area epoch_logs['val_loss'] = loss epoch_logs['train_loss'] = ave_loss - best_so_far = cmp_fn( - epoch_logs[conf['callbacks']['monitor']], best_so_far) + best_so_far = cmp_fn(epoch_logs[conf['callbacks']['monitor']], + best_so_far) # only save model weights if quantity we are tracking is improving if best_so_far != epoch_logs[conf['callbacks']['monitor']]: print("Not saving model weights") - specific_builder.delete_model_weights( - train_model, int(round(e))) + specific_builder.delete_model_weights(train_model, + int(round(e))) if conf['training']['ranking_difficulty_fac'] != 1.0: (_, _, _, roc_area_train, @@ -174,12 +167,8 @@ def train(conf, shot_list_train, shot_list_validate, loader, # plot_losses(conf,[training_losses],specific_builder,name='training') if conf['training']['validation_frac'] > 0.0: - plot_losses(conf, - [training_losses, - validation_losses, - validation_roc], - specific_builder, - name='training_validation_roc') + plot_losses(conf, [training_losses, validation_losses, validation_roc], + specific_builder, name='training_validation_roc') batch_iterator.__exit__() print('...done') @@ -190,30 +179,22 @@ def optimizer_class(): if conf['model']['optimizer'] == 'sgd': return SGD(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm']) elif conf['model']['optimizer'] == 'momentum_sgd': - return SGD( - lr=conf['model']['lr'], - clipnorm=conf['model']['clipnorm'], - decay=1e-6, - momentum=0.9) + return SGD(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm'], + decay=1e-6, momentum=0.9) elif conf['model']['optimizer'] == 'tf_momentum_sgd': - return TFOptimizer( - tf.train.MomentumOptimizer( - learning_rate=conf['model']['lr'], - momentum=0.9)) + return TFOptimizer(tf.train.MomentumOptimizer( + learning_rate=conf['model']['lr'], momentum=0.9)) elif conf['model']['optimizer'] == 'adam': return Adam(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm']) elif conf['model']['optimizer'] == 'tf_adam': - return TFOptimizer( - tf.train.AdamOptimizer( - learning_rate=conf['model']['lr'])) + return TFOptimizer(tf.train.AdamOptimizer( + learning_rate=conf['model']['lr'])) elif conf['model']['optimizer'] == 'rmsprop': - return RMSprop( - lr=conf['model']['lr'], - clipnorm=conf['model']['clipnorm']) + return RMSprop(lr=conf['model']['lr'], + clipnorm=conf['model']['clipnorm']) elif conf['model']['optimizer'] == 'nadam': - return Nadam( - lr=conf['model']['lr'], - clipnorm=conf['model']['clipnorm']) + return Nadam(lr=conf['model']['lr'], + clipnorm=conf['model']['clipnorm']) else: print("Optimizer not implemented yet") exit(1) @@ -232,9 +213,8 @@ def keras_fmin_fnct(self, space): specific_builder = builder.ModelBuilder(self.conf) train_model = specific_builder.hyper_build_model(space, False) - train_model.compile( - optimizer=optimizer_class(), - loss=conf['data']['target'].loss) + train_model.compile(optimizer=optimizer_class(), + loss=conf['data']['target'].loss) np.random.seed(1) validation_losses = [] @@ -267,17 +247,10 @@ def keras_fmin_fnct(self, space): X_list, y_list = self.loader.load_as_X_y_list(shot_sublist) for j, (X, y) in enumerate(zip(X_list, y_list)): history = builder.LossHistory() - train_model.fit( - X, - y, - batch_size=Loader.get_batch_size( - self.conf['training']['batch_size'], - prediction_mode=False), - epochs=1, - shuffle=False, - verbose=0, - validation_split=0.0, - callbacks=[history]) + train_model.fit(X, y, + batch_size=Loader.get_batch_size(self.conf['training']['batch_size'], prediction_mode=False), # noqa + epochs=1, shuffle=False, verbose=0, + validation_split=0.0, callbacks=[history]) train_model.reset_states() train_loss = np.mean(history.losses) training_losses_tmp.append(train_loss) @@ -288,12 +261,10 @@ def keras_fmin_fnct(self, space): sys.stdout.flush() training_losses.append(np.mean(training_losses_tmp)) specific_builder.save_model_weights(train_model, e) - _, _, _, roc_area, loss = make_predictions_and_evaluate_gpu( self.conf, shot_list_validate, self.loader) - print( - "Epoch: {}, loss: {}, validation_losses_size: {}".format( - e, loss, len(validation_losses))) + print("Epoch: {}, loss: {}, validation_losses_size: {}".format( + e, loss, len(validation_losses))) validation_losses.append(loss) validation_roc.append(roc_area) resulting_dict['loss'] = loss @@ -306,20 +277,13 @@ def keras_fmin_fnct(self, space): return resulting_dict def get_space(self): - return { - 'Dropout': hp.uniform('Dropout', 0, 1), - } + return {'Dropout': hp.uniform('Dropout', 0, 1), } def frnn_minimize(self, algo, max_evals, trials, rseed=1337): from hyperopt import fmin - - best_run = fmin(self.keras_fmin_fnct, - space=self.get_space(), - algo=algo, - max_evals=max_evals, - trials=trials, + best_run = fmin(self.keras_fmin_fnct, space=self.get_space(), + algo=algo, max_evals=max_evals, trials=trials, rstate=np.random.RandomState(rseed)) - best_model = None for trial in trials: vals = trial.get('misc').get('vals') @@ -373,20 +337,16 @@ def make_predictions(conf, shot_list, loader): disruptive = [] model = specific_builder.build_model(True) - model.compile( - optimizer=optimizer_class(), - loss=conf['data']['target'].loss) + model.compile(optimizer=optimizer_class(), + loss=conf['data']['target'].loss) specific_builder.load_model_weights(model) model_save_path = specific_builder.get_latest_save_path() start_time = time.time() pool = mp.Pool(use_cores) - fn = partial( - make_single_prediction, - builder=specific_builder, - loader=loader, - model_save_path=model_save_path) + fn = partial(make_single_prediction, builder=specific_builder, + loader=loader, model_save_path=model_save_path) print('running in parallel on {} processes'.format(pool._processes)) for (i, (y_p, y, is_disruptive)) in enumerate(pool.imap(fn, shot_list)): @@ -405,19 +365,16 @@ def make_predictions(conf, shot_list, loader): def make_single_prediction(shot, specific_builder, loader, model_save_path): loader.set_inference_mode(True) model = specific_builder.build_model(True) - model.compile( - optimizer=optimizer_class(), - loss=conf['data']['target'].loss) + model.compile(optimizer=optimizer_class(), + loss=conf['data']['target'].loss) model.load_weights(model_save_path) model.reset_states() X, y = loader.load_as_X_y(shot, prediction_mode=True) assert(X.shape[0] == y.shape[0]) y_p = model.predict( - X, - batch_size=Loader.get_batch_size(conf['training']['batch_size'], - prediction_mode=True), - verbose=0) + X, batch_size=Loader.get_batch_size(conf['training']['batch_size'], + prediction_mode=True), verbose=0) answer_dims = y_p.shape[-1] if conf['model']['return_sequences']: shot_length = y_p.shape[0]*y_p.shape[1] @@ -433,7 +390,6 @@ def make_single_prediction(shot, specific_builder, loader, model_save_path): def make_predictions_gpu(conf, shot_list, loader, custom_path=None): loader.set_inference_mode(True) - if backend == 'tf' or backend == 'tensorflow': first_time = "tensorflow" not in sys.modules if first_time: @@ -462,10 +418,8 @@ def make_predictions_gpu(conf, shot_list, loader, custom_path=None): model.reset_states() pbar = Progbar(len(shot_list)) - shot_sublists = shot_list.sublists( - conf['model']['pred_batch_size'], - do_shuffle=False, - equal_size=True) + shot_sublists = shot_list.sublists(conf['model']['pred_batch_size'], + do_shuffle=False, equal_size=True) for (i, shot_sublist) in enumerate(shot_sublists): X, y, shot_lengths, disr = loader.load_as_X_y_pred(shot_sublist) # load data and fit on data @@ -496,9 +450,8 @@ def make_predictions_and_evaluate_gpu( conf, shot_list, loader, custom_path) analyzer = PerformanceAnalyzer(conf=conf) roc_area = analyzer.get_roc_area(y_prime, y_gold, disruptive) - shot_list.set_weights( - analyzer.get_shot_difficulty( - y_prime, y_gold, disruptive)) + shot_list.set_weights(analyzer.get_shot_difficulty( + y_prime, y_gold, disruptive)) loss = get_loss_from_list(y_prime, y_gold, conf['data']['target']) return y_prime, y_gold, disruptive, roc_area, loss @@ -528,10 +481,8 @@ def make_evaluations_gpu(conf, shot_list, loader): batch_size = min(len(shot_list), conf['model']['pred_batch_size']) pbar = Progbar(len(shot_list)) - print( - 'evaluating {} shots using batchsize {}'.format( - len(shot_list), - batch_size)) + print('evaluating {} shots using batchsize {}'.format( + len(shot_list), batch_size)) shot_sublists = shot_list.sublists(batch_size, equal_size=False) all_metrics = [] @@ -540,21 +491,16 @@ def make_evaluations_gpu(conf, shot_list, loader): batch_size = len(shot_sublist) model = specific_builder.build_model( True, custom_batch_size=batch_size) - model.compile( - optimizer=optimizer_class(), - loss=conf['data']['target'].loss) + model.compile(optimizer=optimizer_class(), + loss=conf['data']['target'].loss) specific_builder.load_model_weights(model) model.reset_states() X, y, shot_lengths, disr = loader.load_as_X_y_pred( shot_sublist, custom_batch_size=batch_size) # load data and fit on data - all_metrics.append( - model.evaluate( - X, - y, - batch_size=batch_size, - verbose=False)) + all_metrics.append(model.evaluate(X, y, batch_size=batch_size, + verbose=False)) all_weights.append(batch_size) model.reset_states() From f13f95b997b28dc967c23893129508d0ccc828d7 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 10 Oct 2019 15:29:45 -0500 Subject: [PATCH 112/272] Fix missing tensorflow import and lr definition in runner.py --- plasma/models/mpi_runner.py | 117 ++++++++++++------------------------ plasma/models/runner.py | 10 ++- 2 files changed, 49 insertions(+), 78 deletions(-) diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 4a9b97d0..5cfab424 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -59,9 +59,8 @@ os.environ['KERAS_BACKEND'] = 'tensorflow' import tensorflow as tf from keras.backend.tensorflow_backend import set_session - gpu_options = tf.GPUOptions( - per_process_gpu_memory_fraction=0.95, - allow_growth=True) + gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.95, + allow_growth=True) config = tf.ConfigProto(gpu_options=gpu_options) set_session(tf.Session(config=config)) else: @@ -206,10 +205,8 @@ def __init__(self, model, optimizer, comm, batch_iterator, batch_size, self.num_replicas = self.num_workers else: self.num_replicas = num_replicas - self.lr = ( - lr/(1.0 + self.num_replicas/100.0) if (lr < self.max_lr) - else self.max_lr/(1.0+self.num_replicas/100.0) - ) + self.lr = (lr/(1.0 + self.num_replicas/100.0) if (lr < self.max_lr) + else self.max_lr/(1.0 + self.num_replicas/100.0)) def set_batch_iterator_func(self): if (self.conf is not None @@ -238,22 +235,16 @@ def compile(self, optimizer, clipnorm, loss='mse'): if optimizer == 'sgd': optimizer_class = SGD(lr=self.DUMMY_LR, clipnorm=clipnorm) elif optimizer == 'momentum_sgd': - optimizer_class = SGD( - lr=self.DUMMY_LR, - clipnorm=clipnorm, - decay=1e-6, - momentum=0.9) + optimizer_class = SGD(lr=self.DUMMY_LR, clipnorm=clipnorm, + decay=1e-6, momentum=0.9) elif optimizer == 'tf_momentum_sgd': - optimizer_class = TFOptimizer( - tf.train.MomentumOptimizer( - learning_rate=self.DUMMY_LR, - momentum=0.9)) + optimizer_class = TFOptimizer(tf.train.MomentumOptimizer( + learning_rate=self.DUMMY_LR, momentum=0.9)) elif optimizer == 'adam': optimizer_class = Adam(lr=self.DUMMY_LR, clipnorm=clipnorm) elif optimizer == 'tf_adam': - optimizer_class = TFOptimizer( - tf.train.AdamOptimizer( - learning_rate=self.DUMMY_LR)) + optimizer_class = TFOptimizer(tf.train.AdamOptimizer( + learning_rate=self.DUMMY_LR)) elif optimizer == 'rmsprop': optimizer_class = RMSprop(lr=self.DUMMY_LR, clipnorm=clipnorm) elif optimizer == 'nadam': @@ -372,9 +363,8 @@ def sync_deltas(self, deltas, num_replicas=None): global_deltas = [] # default is to reduce the deltas from all workers for delta in deltas: - global_deltas.append( - self.mpi_average_gradients( - delta, num_replicas)) + global_deltas.append(self.mpi_average_gradients( + delta, num_replicas)) return global_deltas def set_new_weights(self, deltas, num_replicas=None): @@ -442,11 +432,8 @@ def build_callbacks(self, conf, callbacks_list): datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")))] if "earlystop" in callbacks_list: - callbacks += [ - cbks.EarlyStopping( - patience=patience, - monitor=monitor, - mode=mode)] + callbacks += [cbks.EarlyStopping( + patience=patience, monitor=monitor, mode=mode)] if "lr_scheduler" in callbacks_list: pass @@ -496,11 +483,8 @@ def train_epoch(self): t1 = 0 t2 = 0 - while ( - self.num_so_far - - self.epoch - * num_total) < num_total or step < self.num_batches_minimum: - + while ((self.num_so_far - self.epoch * num_total) < num_total + or step < self.num_batches_minimum): try: (batch_xs, batch_ys, batches_to_reset, num_so_far_curr, num_total, is_warmup_period) = next(batch_iterator_func) @@ -531,9 +515,8 @@ def train_epoch(self): batch_xs, batch_ys, verbose) self.comm.Barrier() sys.stdout.flush() - print_unique( - 'Compilation finished in {:.2f}s'.format( - time.time()-t0_comp)) + print_unique('Compilation finished in {:.2f}s'.format( + time.time()-t0_comp)) t_start = time.time() sys.stdout.flush() @@ -548,7 +531,6 @@ def train_epoch(self): self.set_new_weights(deltas, num_replicas) t2 = time.time() write_str_0 = self.calculate_speed(t0, t1, t2, num_replicas) - curr_loss = self.mpi_average_scalars(1.0*loss, num_replicas) # if self.task_index == 0: # print(self.model.get_weights()[0][0][:4]) @@ -606,18 +588,13 @@ def calculate_speed(self, t0, t_after_deltas, t_after_update, num_replicas, frac_calculate = t_calculate/t_tot frac_sync = t_sync/t_tot - print_str = ( - '{:.2E} Examples/sec | {:.2E} sec/batch '.format(examples_per_sec, - t_tot) - + '[{:.1%} calc., {:.1%} synch.]'.format(frac_calculate, - frac_sync)) + print_str = ('{:.2E} Examples/sec | {:.2E} sec/batch '.format( + examples_per_sec, t_tot) + + '[{:.1%} calc., {:.1%} synch.]'.format( + frac_calculate, frac_sync)) print_str += '[batch = {} = {}*{}] [lr = {:.2E} = {:.2E}*{}]'.format( - effective_batch_size, - self.batch_size, - num_replicas, - self.get_effective_lr(num_replicas), - self.lr, - num_replicas) + effective_batch_size, self.batch_size, num_replicas, + self.get_effective_lr(num_replicas), self.lr, num_replicas) if verbose: print_unique(print_str) return print_str @@ -653,11 +630,9 @@ def get_shot_list_path(conf): def save_shotlists(conf, shot_list_train, shot_list_validate, shot_list_test): path = get_shot_list_path(conf) - np.savez( - path, - shot_list_train=shot_list_train, - shot_list_validate=shot_list_validate, - shot_list_test=shot_list_test) + np.savez(path, shot_list_train=shot_list_train, + shot_list_validate=shot_list_validate, + shot_list_test=shot_list_test) def load_shotlists(conf): @@ -696,11 +671,8 @@ def mpi_make_predictions(conf, shot_list, loader, custom_path=None): model.reset_states() if task_index == 0: pbar = Progbar(len(shot_list)) - shot_sublists = shot_list.sublists( - conf['model']['pred_batch_size'], - do_shuffle=False, - equal_size=True) - + shot_sublists = shot_list.sublists(conf['model']['pred_batch_size'], + do_shuffle=False, equal_size=True) y_prime_global = [] y_gold_global = [] disruptive_global = [] @@ -708,7 +680,6 @@ def mpi_make_predictions(conf, shot_list, loader, custom_path=None): loader.verbose = False for (i, shot_sublist) in enumerate(shot_sublists): - if i % num_workers == task_index: X, y, shot_lengths, disr = loader.load_as_X_y_pred(shot_sublist) @@ -756,8 +727,7 @@ def mpi_make_predictions_and_evaluate(conf, shot_list, loader, analyzer = PerformanceAnalyzer(conf=conf) roc_area = analyzer.get_roc_area(y_prime, y_gold, disruptive) shot_list.set_weights( - analyzer.get_shot_difficulty( - y_prime, y_gold, disruptive)) + analyzer.get_shot_difficulty(y_prime, y_gold, disruptive)) loss = get_loss_from_list(y_prime, y_gold, conf['data']['target']) return y_prime, y_gold, disruptive, roc_area, loss @@ -826,20 +796,15 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, mpi_model = MPIModel(train_model, optimizer, comm, batch_generator, batch_size, lr=lr, warmup_steps=warmup_steps, num_batches_minimum=num_batches_minimum, conf=conf) - mpi_model.compile( - conf['model']['optimizer'], - clipnorm, - conf['data']['target'].loss) - + mpi_model.compile(conf['model']['optimizer'], clipnorm, + conf['data']['target'].loss) tensorboard = None if backend != "theano" and task_index == 0: tensorboard_save_path = conf['paths']['tensorboard_save_path'] write_grads = conf['callbacks']['write_grads'] - tensorboard = TensorBoard( - log_dir=tensorboard_save_path, - histogram_freq=1, - write_graph=True, - write_grads=write_grads) + tensorboard = TensorBoard(log_dir=tensorboard_save_path, + histogram_freq=1, write_graph=True, + write_grads=write_grads) tensorboard.set_model(mpi_model.model) mpi_model.model.summary() @@ -905,8 +870,8 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, epoch_logs['val_roc'] = roc_area epoch_logs['val_loss'] = loss epoch_logs['train_loss'] = ave_loss - best_so_far = cmp_fn( - epoch_logs[conf['callbacks']['monitor']], best_so_far) + best_so_far = cmp_fn(epoch_logs[conf['callbacks']['monitor']], + best_so_far) if task_index == 0: print('=========Summary======== for epoch{}'.format(step)) @@ -992,9 +957,8 @@ def set_model(self, model): def is_indexed_slices(grad): return type(grad).__name__ == 'IndexedSlices' - grads = [ - grad.values if is_indexed_slices(grad) else grad - for grad in grads] + grads = [grad.values if is_indexed_slices(grad) else + grad for grad in grads] for grad in grads: tf.summary.histogram( '{}_grad'.format(mapped_weight_name), grad) @@ -1023,8 +987,7 @@ def on_epoch_end(self, val_generator, val_steps, epoch, logs=None): self.writer.add_summary(summary, epoch) self.writer.flush() - tensors = (self.model.inputs - + self.model.targets + tensors = (self.model.inputs + self.model.targets + self.model.sample_weights) if self.model.uses_learning_phase: diff --git a/plasma/models/runner.py b/plasma/models/runner.py index f616f54a..5b3f1d5a 100644 --- a/plasma/models/runner.py +++ b/plasma/models/runner.py @@ -87,6 +87,9 @@ def train(conf, shot_list_train, shot_list_validate, loader, print('\nEpoch {}/{}'.format(e+1, num_epochs)) pbar = Progbar(len(shot_list_train)) + # TODO(KGF): check this fix; lr, tf were undefined in neglected + # serial runner.py, since mpi_runner.py has been the main tool + lr = conf['model']['lr'] # decay learning rate each epoch: K.set_value(train_model.optimizer.lr, lr*lr_decay**(e)) @@ -175,6 +178,9 @@ def train(conf, shot_list_train, shot_list_validate, loader, def optimizer_class(): from keras.optimizers import SGD, Adam, RMSprop, Nadam, TFOptimizer + # TODO(KGF): check this fix; lr, tf were undefined in neglected + # serial runner.py, since mpi_runner.py has been the main tool + import tensorflow as tf if conf['model']['optimizer'] == 'sgd': return SGD(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm']) @@ -241,7 +247,9 @@ def keras_fmin_fnct(self, space): shot_list_train.shuffle() shot_sublists = shot_list_train.sublists(num_at_once)[:1] training_losses_tmp = [] - + # TODO(KGF): check this fix; lr, tf were undefined in neglected + # serial runner.py, since mpi_runner.py has been the main tool + lr = conf['model']['lr'] K.set_value(train_model.optimizer.lr, lr*lr_decay**(e)) for (i, shot_sublist) in enumerate(shot_sublists): X_list, y_list = self.loader.load_as_X_y_list(shot_sublist) From f10ec50938368128708ee500be4912dba2eaa00a Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 10 Oct 2019 16:43:39 -0400 Subject: [PATCH 113/272] Ignore output files from cluster job schedulers --- .gitignore | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.gitignore b/.gitignore index 139f148f..0125b4ab 100644 --- a/.gitignore +++ b/.gitignore @@ -95,3 +95,16 @@ ENV/ # Rope project settings .ropeproject + +# Job scheduler output +# Slurm +*.out + +# Cobalt +*.output +*.error +*.cobaltlog + +# PBS +# *.o* +# *.e* \ No newline at end of file From 3ec70373f6a741b37a88ed2a67fec4c7197f853b Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Mon, 14 Oct 2019 15:30:10 -0400 Subject: [PATCH 114/272] Install scikit-learn, not dummy sklearn, from pip See https://github.com/scikit-learn/scikit-learn/issues/8215#issue-201818378 --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5c8fec7f..8df361cd 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,9 @@ 'matplotlib==2.0.2', 'hyperopt', 'mpi4py', - 'xgboost'], + 'xgboost', + 'scikit-learn', + ], tests_require=[], classifiers=["Development Status :: 3 - Alpha", "Environment :: Console", From 32af980b231f9b52e31cbcabfb2148c8f2f31c84 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 15 Oct 2019 10:48:46 -0400 Subject: [PATCH 115/272] Return default conf.yaml to use shallow=False --- examples/conf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/conf.yaml b/examples/conf.yaml index b9116a5c..2b977191 100644 --- a/examples/conf.yaml +++ b/examples/conf.yaml @@ -57,7 +57,7 @@ model: loss_scale_factor: 1.0 use_batch_norm: false torch: False - shallow: True + shallow: False shallow_model: num_samples: 1000000 #1000000 #the number of samples to use for training type: "xgboost" #"xgboost" #"xgboost" #"random_forest" "xgboost" From 3942ddcbad7434c1651dedb4bff2754062190212 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 15 Oct 2019 10:06:02 -0500 Subject: [PATCH 116/272] Import joblib module directly, not through sklearn Starting with scikit-learn v0.21.0 (2019-05-09), Joblib is no longer vendored in scikit-learn, and becomes a dependency. Minimal supported version is joblib 0.11, however using version >= 0.13 is strongly recommended Addresses warning from shallow_runner.py: DeprecationWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+. --- plasma/models/shallow_runner.py | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/plasma/models/shallow_runner.py b/plasma/models/shallow_runner.py index 84e6121a..77fe074e 100644 --- a/plasma/models/shallow_runner.py +++ b/plasma/models/shallow_runner.py @@ -3,7 +3,7 @@ import keras.callbacks as cbks from sklearn.metrics import classification_report # accuracy_score, auc, confusion_matrix -from sklearn.externals import joblib +import joblib from sklearn.ensemble import RandomForestClassifier import hashlib from plasma.utils.downloading import makedirs_process_safe diff --git a/setup.py b/setup.py index 8df361cd..3f6b58ce 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ 'mpi4py', 'xgboost', 'scikit-learn', + 'joblib', ], tests_require=[], classifiers=["Development Status :: 3 - Alpha", From c410f2ae3888ab378d674d3f85b15e7a302b5718 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 15 Oct 2019 18:05:31 -0400 Subject: [PATCH 117/272] Fix shallow_runner.py; ttd is not imported from plasma.utils.evaluation --- plasma/models/shallow_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plasma/models/shallow_runner.py b/plasma/models/shallow_runner.py index 77fe074e..5d459a43 100644 --- a/plasma/models/shallow_runner.py +++ b/plasma/models/shallow_runner.py @@ -8,7 +8,7 @@ import hashlib from plasma.utils.downloading import makedirs_process_safe # from plasma.utils.state_reset import reset_states -from plasma.utils.evaluation import ttd, get_loss_from_list +from plasma.utils.evaluation import get_loss_from_list from plasma.utils.performance import PerformanceAnalyzer # from plasma.models.loader import Loader, ProcessGenerator # from plasma.conf import conf @@ -156,7 +156,7 @@ def get_X(self, shot): shot.ttd = np.expand_dims(shot.ttd, axis=1) length = sig_sample.shape[0] if length < self.timesteps: - print(ttd, shot, shot.number) + print(shot.ttd, shot.number) print("Shot must be at least as long as the RNN length.") exit(1) assert(len(sig_sample.shape) == len(shot.ttd.shape) == 2) From d7ad340a01200f0e66a57c4b883951d1911a57fe Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 15 Oct 2019 18:06:14 -0400 Subject: [PATCH 118/272] Fix bug introduced by #39 in mpi_runner.py In jdev branch, instead of broadcasting the class member mpi_model.model.stop_training directly, a conditional based on hasattr() was added to bind the value to a local variable "stop_training" for broadcast, which was undefined on MPI ranks other than the master rank 0. Set local variable to default False on all ranks --- plasma/models/mpi_runner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 5cfab424..db5e0d67 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -568,9 +568,9 @@ def estimate_remaining_time(self, time_so_far, work_so_far, work_total): def get_effective_lr(self, num_replicas): effective_lr = self.lr * num_replicas if effective_lr > self.max_lr: - print_unique('Warning: effective learning rate set to {}, '.format( - effective_lr) + 'larger than maximum {}. Clipping.'.format( - self.max_lr)) + print_unique( + 'Warning: effective learning rate set to {}, '.format(effective_lr) + + 'larger than maximum {}. Clipping.'.format(self.max_lr)) effective_lr = self.max_lr return effective_lr @@ -872,7 +872,7 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, epoch_logs['train_loss'] = ave_loss best_so_far = cmp_fn(epoch_logs[conf['callbacks']['monitor']], best_so_far) - + stop_training = False if task_index == 0: print('=========Summary======== for epoch{}'.format(step)) print('Training Loss numpy: {:.3e}'.format(ave_loss)) From f8b2cbb76511688f30dc4f01dc8eb8ac6bdf30b3 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 15 Oct 2019 18:14:34 -0400 Subject: [PATCH 119/272] Only build the master branch (and PRs) on Travis CI --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d16ec7c2..31c7f70b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,7 @@ language: python - +branches: + only: + - master os: - linux From 69f59c9c90a220234daf82f9b47e645d23f4ae44 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 15 Oct 2019 18:18:53 -0400 Subject: [PATCH 120/272] Undo formatting change --- plasma/models/mpi_runner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index db5e0d67..e3807709 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -568,9 +568,9 @@ def estimate_remaining_time(self, time_so_far, work_so_far, work_total): def get_effective_lr(self, num_replicas): effective_lr = self.lr * num_replicas if effective_lr > self.max_lr: - print_unique( - 'Warning: effective learning rate set to {}, '.format(effective_lr) - + 'larger than maximum {}. Clipping.'.format(self.max_lr)) + print_unique('Warning: effective learning rate set to {}, '.format( + effective_lr) + 'larger than maximum {}. Clipping.'.format( + self.max_lr)) effective_lr = self.max_lr return effective_lr From 24de29b5daedd47ece9ae2cb0e5c88f8ca0a4de0 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 16 Oct 2019 10:32:42 -0500 Subject: [PATCH 121/272] Update module versions, X11 info, OpenMPI BTL https://www.open-mpi.org/faq/?category=openfabrics#ib-btl NOTE: Prior versions of Open MPI used an sm BTL for shared memory. sm was effectively replaced with vader starting in Open MPI v3.0.0. Switching to "vader" BTL from "sm" prevents runtime error: # -------------------------------------------------------------------------- # As of version 3.0.0, the "sm" BTL is no longer available in Open MPI. # Efficient, high-speed same-node shared memory communication support in # Open MPI is available in the "vader" BTL. To use the vader BTL, you # can re-run your job with: # mpirun --mca btl vader,self,... your_mpi_application # -------------------------------------------------------------------------- --- docs/PrincetonUTutorial.md | 46 +++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/docs/PrincetonUTutorial.md b/docs/PrincetonUTutorial.md index b9a97565..7da0f44e 100644 --- a/docs/PrincetonUTutorial.md +++ b/docs/PrincetonUTutorial.md @@ -1,4 +1,5 @@ ## Tutorials +*Last updated 2019-10-16* ### Login to TigerGPU @@ -6,6 +7,7 @@ First, login to TigerGPU cluster headnode via ssh: ``` ssh -XC @tigergpu.princeton.edu ``` +Note, `-XC` is optional; it is only necessary if you are planning on performing remote visualization, e.g. the output `.png` files from the below [section](#Learning-curves-and-ROC-per-epoch). Trusted X11 forwarding can be used with `-Y` instead of `-X` and may prevent timeouts, but it disables X11 SECURITY extension controls. Compression `-C` reduces the bandwidth usage and may be useful on slow connections. ### Sample usage on TigerGPU @@ -15,21 +17,29 @@ git clone https://github.com/PPPLDeepLearning/plasma-python cd plasma-python ``` -After that, create an isolated Anaconda environment and load CUDA drivers: +After that, create an isolated Anaconda environment and load CUDA drivers, an MPI compiler, and the HDF5 library: ``` #cd plasma-python -module load anaconda3/4.4.0 +module load anaconda3 conda create --name my_env --file requirements-travis.txt source activate my_env -export OMPI_MCA_btl="tcp,self,sm" -module load cudatoolkit/8.0 -module load cudnn/cuda-8.0/6.0 -module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 -module load intel/17.0/64/17.0.5.239 +export OMPI_MCA_btl="tcp,self,vader" +# replace "vader" with "sm" for OpenMPI versions prior to 3.0.0 +module load cudatoolkit cudann +module load openmpi/cuda-8.0/intel-17.0/3.0.0/64 +module load intel +module load hdf5/intel-17.0/intel-mpi/1.10.0 +``` +As of the latest update of this document, the above modules correspond to the following versions on the TigerGPU system, given by `module list`: +``` +Currently Loaded Modulefiles: + 1) anaconda3/2019.3 4) openmpi/cuda-8.0/intel-17.0/3.0.0/64 7) hdf5/intel-17.0/intel-mpi/1.10.0 + 2) cudatoolkit/10.1 5) intel-mkl/2019.3/3/64 + 3) cudnn/cuda-9.2/7.6.3 6) intel/19.0/64/19.0.3.199 ``` -and install the `plasma-python` package: +Next, install the `plasma-python` package: ```bash #source activate my_env @@ -44,7 +54,7 @@ Common issue is Intel compiler mismatch in the `PATH` and what you use in the mo you should see something like this: ``` $ which mpicc -/usr/local/openmpi/cuda-8.0/2.1.0/intel170/x86_64/bin/mpicc +/usr/local/openmpi/cuda-8.0/3.0.0/intel170/x86_64/bin/mpicc ``` If you source activate the Anaconda environment after loading the openmpi, you would pick the MPI from Anaconda, which is not good and could lead to errors. @@ -93,20 +103,20 @@ For batch analysis, make sure to allocate 1 MPI process per GPU. Save the follow #SBATCH -c 4 #SBATCH --mem-per-cpu=0 -module load anaconda3/4.4.0 +module load anaconda3 source activate my_env -export OMPI_MCA_btl="tcp,self,sm" -module load cudatoolkit/8.0 -module load cudnn/cuda-8.0/6.0 -module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 -module load intel/17.0/64/17.0.4.196 +export OMPI_MCA_btl="tcp,self,vader" +module load cudatoolkit cudann +module load openmpi/cuda-8.0/intel-17.0/3.0.0/64 +module load intel +module load hdf5/intel-17.0/intel-mpi/1.10.0 srun python mpi_learn.py ``` where `X` is the number of nodes for distibuted training. -Submit the job with: +Submit the job with (assuming you are still in the `examples/` subdirectory): ```bash #cd examples sbatch slurm.cmd @@ -131,7 +141,7 @@ where the number of GPUs is X * 4. Then launch the application from the command line: ```bash -mpirun -npernode 4 python examples/mpi_learn.py +mpirun -npernode 4 python mpi_learn.py ``` ### Understanding the data @@ -205,7 +215,7 @@ python -m tensorflow.tensorboard --logdir /mnt/ Date: Wed, 16 Oct 2019 11:37:46 -0500 Subject: [PATCH 122/272] Add comments about the mpirun vs. srun behavior --- docs/PrincetonUTutorial.md | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/docs/PrincetonUTutorial.md b/docs/PrincetonUTutorial.md index 7da0f44e..37709662 100644 --- a/docs/PrincetonUTutorial.md +++ b/docs/PrincetonUTutorial.md @@ -1,5 +1,5 @@ ## Tutorials -*Last updated 2019-10-16* +*Last updated 2019-10-16.* ### Login to TigerGPU @@ -57,13 +57,13 @@ $ which mpicc /usr/local/openmpi/cuda-8.0/3.0.0/intel170/x86_64/bin/mpicc ``` -If you source activate the Anaconda environment after loading the openmpi, you would pick the MPI from Anaconda, which is not good and could lead to errors. +If you `source activate` the Anaconda environment **after** loading the OpenMPI library, your application would be built with the MPI library from Anaconda, which has worse performance on this cluster and could lead to errors. See [On Computing Well: Installing and Running ‘mpi4py’ on the Cluster](https://oncomputingwell.princeton.edu/2018/11/installing-and-running-mpi4py-on-the-cluster/) for a related discussion. #### Location of the data on Tigress -The JET and D3D datasets containing multi-modal time series of sensory measurements leading up to deleterious events called plasma disruptions are located on `/tigress/FRNN` filesystem on Princeton U clusters. -Fo convenience, create following symbolic links: +The JET and D3D datasets contain multi-modal time series of sensory measurements leading up to deleterious events called plasma disruptions. The datasets are located in the `/tigress/FRNN` project directory of the [GPFS](https://www.ibm.com/support/knowledgecenter/en/SSPT3X_3.0.0/com.ibm.swg.im.infosphere.biginsights.product.doc/doc/bi_gpfs_overview.html) filesystem on Princeton University clusters. +For convenience, create following symbolic links: ```bash cd /tigress/ ln -s /tigress/FRNN/shot_lists shot_lists @@ -76,14 +76,14 @@ ln -s /tigress/FRNN/signal_data signal_data cd examples/ python guarantee_preprocessed.py ``` -This will preprocess the data and save it in `/tigress//processed_shots`, `/tigress//processed_shotlists` and `/tigress//normalization` +This will preprocess the data and save rescaled copies of the signals in `/tigress//processed_shots`, `/tigress//processed_shotlists` and `/tigress//normalization` You would only have to run preprocessing once for each dataset. The dataset is specified in the config file `examples/conf.yaml`: ```yaml paths: data: jet_data_0D ``` -It take takes about 20 minutes to preprocess in parallel and can normally be done on the cluster headnode. +Preprocessing this dataset takes about 20 minutes to preprocess in parallel and can normally be done on the cluster headnode. #### Training and inference @@ -91,7 +91,7 @@ Use Slurm scheduler to perform batch or interactive analysis on TigerGPU cluster ##### Batch analysis -For batch analysis, make sure to allocate 1 MPI process per GPU. Save the following to slurm.cmd file (or make changes to the existing `examples/slurm.cmd`): +For batch analysis, make sure to allocate 1 MPI process per GPU. Save the following to `slurm.cmd` file (or make changes to the existing `examples/slurm.cmd`): ```bash #!/bin/bash @@ -114,7 +114,9 @@ module load hdf5/intel-17.0/intel-mpi/1.10.0 srun python mpi_learn.py ``` -where `X` is the number of nodes for distibuted training. +where `X` is the number of nodes for distibuted training and the total number of GPUs is `X * 4`. This configuration guarantees 1 MPI process per GPU, regardless of the value of `X`. + +Update the `num_gpus` value in `conf.yaml` to correspond to the total number of GPUs specified for your Slurm allocation. Submit the job with (assuming you are still in the `examples/` subdirectory): ```bash @@ -126,7 +128,11 @@ And monitor it's completion via: ```bash squeue -u ``` -Optionally, add an email notification option in the Slurm about the job completion. +Optionally, add an email notification option in the Slurm configuration about the job completion: +``` +#SBATCH --mail-user=@princeton.edu +#SBATCH --mail-type=ALL +``` ##### Interactive analysis @@ -136,13 +142,18 @@ The workflow is to request an interactive session: ```bash salloc -N [X] --ntasks-per-node=4 --ntasks-per-socket=2 --gres=gpu:4 -c 4 --mem-per-cpu=0 -t 0-6:00 ``` -where the number of GPUs is X * 4. - -Then launch the application from the command line: +Then, launch the application from the command line: ```bash -mpirun -npernode 4 python mpi_learn.py +mpirun -N 4 python mpi_learn.py ``` +where `-N` is a synonym for `-npernode` in OpenMPI. Do **not** use `srun` to launch the job inside an interactive session. +[//]: # (This option appears to be redundant given the salloc options; "mpirun python mpi_learn.py" appears to work just the same. HOWEVER, "srun python mpi_learn.py", "srun --ntasks-per-node python mpi_learn.py", etc. NEVER works--- it just hangs without any output. Why?) + +[//]: # (Consistent with https://www.open-mpi.org/faq/?category=slurm ?) + +[//]: # (certain output seems to be repeated by ntasks-per-node, e.g. echoing the conf.yaml. Expected?) + ### Understanding the data From c76ce9865347d5ae9c95073e2439d4ffbdc7e1f2 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 16 Oct 2019 12:13:56 -0500 Subject: [PATCH 123/272] Update PrincetonUTutorial.md - "conda activate" is preferred to "source activate since v4.4 (December 2017) https://www.anaconda.com/how-to-get-ready-for-the-release-of-conda-4-4/ - Replace "python -m tensorflow.tensorboard ..." with "python -m tensorboard.main" (when / which version changed this?) --- docs/PrincetonUTutorial.md | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/docs/PrincetonUTutorial.md b/docs/PrincetonUTutorial.md index 37709662..71402a17 100644 --- a/docs/PrincetonUTutorial.md +++ b/docs/PrincetonUTutorial.md @@ -22,7 +22,7 @@ After that, create an isolated Anaconda environment and load CUDA drivers, an MP #cd plasma-python module load anaconda3 conda create --name my_env --file requirements-travis.txt -source activate my_env +conda activate my_env export OMPI_MCA_btl="tcp,self,vader" # replace "vader" with "sm" for OpenMPI versions prior to 3.0.0 @@ -42,7 +42,7 @@ Currently Loaded Modulefiles: Next, install the `plasma-python` package: ```bash -#source activate my_env +#conda activate my_env python setup.py install ``` @@ -57,7 +57,7 @@ $ which mpicc /usr/local/openmpi/cuda-8.0/3.0.0/intel170/x86_64/bin/mpicc ``` -If you `source activate` the Anaconda environment **after** loading the OpenMPI library, your application would be built with the MPI library from Anaconda, which has worse performance on this cluster and could lead to errors. See [On Computing Well: Installing and Running ‘mpi4py’ on the Cluster](https://oncomputingwell.princeton.edu/2018/11/installing-and-running-mpi4py-on-the-cluster/) for a related discussion. +If you `conda activate` the Anaconda environment **after** loading the OpenMPI library, your application would be built with the MPI library from Anaconda, which has worse performance on this cluster and could lead to errors. See [On Computing Well: Installing and Running ‘mpi4py’ on the Cluster](https://oncomputingwell.princeton.edu/2018/11/installing-and-running-mpi4py-on-the-cluster/) for a related discussion. #### Location of the data on Tigress @@ -104,7 +104,7 @@ For batch analysis, make sure to allocate 1 MPI process per GPU. Save the follow #SBATCH --mem-per-cpu=0 module load anaconda3 -source activate my_env +conda activate my_env export OMPI_MCA_btl="tcp,self,vader" module load cudatoolkit cudann module load openmpi/cuda-8.0/intel-17.0/3.0.0/64 @@ -148,7 +148,10 @@ Then, launch the application from the command line: mpirun -N 4 python mpi_learn.py ``` where `-N` is a synonym for `-npernode` in OpenMPI. Do **not** use `srun` to launch the job inside an interactive session. -[//]: # (This option appears to be redundant given the salloc options; "mpirun python mpi_learn.py" appears to work just the same. HOWEVER, "srun python mpi_learn.py", "srun --ntasks-per-node python mpi_learn.py", etc. NEVER works--- it just hangs without any output. Why?) + +[//]: # (This option appears to be redundant given the salloc options; "mpirun python mpi_learn.py" appears to work just the same.) + +[//]: # (HOWEVER, "srun python mpi_learn.py", "srun --ntasks-per-node python mpi_learn.py", etc. NEVER works--- it just hangs without any output. Why?) [//]: # (Consistent with https://www.open-mpi.org/faq/?category=slurm ?) @@ -210,20 +213,23 @@ A regular FRNN run will produce several outputs and callbacks. Currently supports graph visualization, histograms of weights, activations and biases, and scalar variable summaries of losses and accuracies. -The summaries are written real time to `/tigress//Graph`. For MacOS, you can set up the `sshfs` mount of /tigress filesystem and view those summaries in your browser. +The summaries are written in real time to `/tigress//Graph`. For macOS, you can set up the `sshfs` mount of the `/tigress` filesystem and view those summaries in your browser. -For Mac, you could follow the instructions here: +To install SSHFS on a macOS system, you could follow the instructions here: https://github.com/osxfuse/osxfuse/wiki/SSHFS +Or use [Homebrew](https://brew.sh/), `brew cask install osxfuse; brew install sshfs`. Note, to install and/or use `osxfuse` you may need to enable its kernel extension in: System Preferences → Security & Privacy → General then do something like: ``` -sshfs -o allow_other,defer_permissions netid@tigergpu.princeton.edu:/tigress/netid/ /mnt// +sshfs -o allow_other,defer_permissions netid@tigergpu.princeton.edu:/tigress// / ``` -Launch TensorBoard locally: +Launch TensorBoard locally (assuming that it is installed on your local computer): ``` -python -m tensorflow.tensorboard --logdir /mnt//Graph +python -m tensorboard.main --logdir /Graph ``` +A URL should be emitted to the console output. Navigate to this link in your browser. If the TensorBoard interface does not open, try directing your browser to `localhost:6006`. + You should see something like: ![tensorboard example](https://github.com/PPPLDeepLearning/plasma-python/blob/master/docs/tb.png) @@ -237,7 +243,7 @@ python performance_analysis.py ``` this uses the resulting file produced as a result of training the neural network as an input, and produces several `.png` files with plots as an output. -In addition, you can check the scalar variable summaries for training loss, validation loss and validation ROC logged at `/tigress/netid/csv_logs` (each run will produce a new log file with a timestamp in name). +In addition, you can check the scalar variable summaries for training loss, validation loss and validation ROC logged at `/tigress//csv_logs` (each run will produce a new log file with a timestamp in name). A sample code to analyze can be found in `examples/notebooks`. For instance: @@ -266,5 +272,4 @@ show(p, notebook_handle=True) ### Learning curve summaries per mini-batch -To extract per mini-batch summaries, use the output produced by FRNN logged to the standard out (in case of the batch jobs, it will all be contained in the Slurm output file). Refer to the following notebook to perform the analysis of learning curve on a mini-batch level: -https://github.com/PPPLDeepLearning/plasma-python/blob/master/examples/notebooks/FRNN_scaling.ipynb +To extract per mini-batch summaries, use the output produced by FRNN logged to the standard out (in case of the batch jobs, it will all be contained in the Slurm output file). Refer to the following notebook to perform the analysis of learning curve on a mini-batch level: [FRNN_scaling.ipynb](https://github.com/PPPLDeepLearning/plasma-python/blob/master/examples/notebooks/FRNN_scaling.ipynb) From 8756dda3bf3506284d0fb915b263ed0167d46a40 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 16 Oct 2019 12:14:27 -0500 Subject: [PATCH 124/272] Update PrincetonUTutorial.md --- docs/PrincetonUTutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/PrincetonUTutorial.md b/docs/PrincetonUTutorial.md index 71402a17..0e9f35d5 100644 --- a/docs/PrincetonUTutorial.md +++ b/docs/PrincetonUTutorial.md @@ -252,7 +252,7 @@ import pandas as pd import numpy as np from bokeh.plotting import figure, show, output_file, save -data = pd.read_csv("/mnt//csv_logs/.csv") +data = pd.read_csv("/csv_logs/.csv") from bokeh.io import output_notebook output_notebook() From 5dd58c81837d46bacf65bc76d60193eec47253f6 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 16 Oct 2019 14:27:43 -0400 Subject: [PATCH 125/272] Update module versions, etc. in slurm.cmd Make the script consistent with https://github.com/PPPLDeepLearning/plasma-python/blob/master/docs/PrincetonUTutorial.md --- examples/slurm.cmd | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/examples/slurm.cmd b/examples/slurm.cmd index 1e22ad99..3dcae884 100644 --- a/examples/slurm.cmd +++ b/examples/slurm.cmd @@ -1,25 +1,29 @@ #!/bin/bash #SBATCH -t 01:00:00 -#SBATCH -N 3 +#SBATCH -N 4 #SBATCH --ntasks-per-node=4 #SBATCH --ntasks-per-socket=2 #SBATCH --gres=gpu:4 #SBATCH -c 4 #SBATCH --mem-per-cpu=0 -module load anaconda/4.4.0 -source activate PPPL -module load cudatoolkit/8.0 -module load cudnn/cuda-8.0/6.0 -module load openmpi/cuda-8.0/intel-17.0/2.1.0/64 -module load intel/17.0/64/17.0.4.196 +# Example Slurm configuration for TigerGPU nodes (4 nodes, 16 GPUs total) +# Each node = 2.4 GHz Xeon Broadwell E5-2680 v4 + 4x 1328 MHz P100 GPU -#remove checkpoints for a benchmark run -rm /scratch/gpfs/$USER/model_checkpoints/* -rm /scratch/gpfs/$USER/results/* -rm /scratch/gpfs/$USER/csv_logs/* -rm /scratch/gpfs/$USER/Graph/* -rm /scratch/gpfs/$USER/normalization/* +module load anaconda3 +conda activate my_env +module load cudatoolkit +module load cudnn +module load openmpi/cuda-8.0/intel-17.0/3.0.0/64 +module load intel/19.0/64/19.0.3.199 +module load hdf5/intel-17.0/intel-mpi/1.10.0 -export OMPI_MCA_btl="tcp,self,sm" +# remove checkpoints for a benchmark run +rm /tigress/$USER/model_checkpoints/* +rm /tigress/$USER/results/* +rm /tigress/$USER/csv_logs/* +rm /tigress/$USER/Graph/* +rm /tigress/$USER/normalization/* + +export OMPI_MCA_btl="tcp,self,vader" srun python mpi_learn.py From 2c4143ab6902c381bc7d87cb74fd2e3d6e70bde3 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 16 Oct 2019 15:01:49 -0400 Subject: [PATCH 126/272] Only call batch_iterator_func.__exit__() if defined in conf.yaml Another bug introduced in #39 merging "jdev" branch. Originally, MPIModel class member "batch_iterator_func" was unconditionally set to ProcessGenerator(self.batch_iterator()). Now, may only be set to self.batch_iterator() if conf['training']['use_process_generator'] == False --- examples/slurm.cmd | 2 +- plasma/models/mpi_runner.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/slurm.cmd b/examples/slurm.cmd index 3dcae884..65272017 100644 --- a/examples/slurm.cmd +++ b/examples/slurm.cmd @@ -11,7 +11,7 @@ # Each node = 2.4 GHz Xeon Broadwell E5-2680 v4 + 4x 1328 MHz P100 GPU module load anaconda3 -conda activate my_env +conda activate frnn-tf13.1 module load cudatoolkit module load cudnn module load openmpi/cuda-8.0/intel-17.0/3.0.0/64 diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index e3807709..e1d55ba6 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -217,7 +217,10 @@ def set_batch_iterator_func(self): self.batch_iterator_func = self.batch_iterator() def close(self): - self.batch_iterator_func.__exit__() + if (self.conf is not None + and 'use_process_generator' in conf['training'] + and conf['training']['use_process_generator']): + self.batch_iterator_func.__exit__() def set_lr(self, lr): self.lr = lr @@ -918,6 +921,7 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, def get_stop_training(callbacks): + # TODO(KGF): this funciton is unused for cb in callbacks.callbacks: if isinstance(cb, cbks.EarlyStopping): print("Checking for early stopping") From e0e347485d71fca8adf9416126241f8ebf7435be Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 17 Oct 2019 11:00:34 -0400 Subject: [PATCH 127/272] Revert conda env name change in parent commit [skip ci] --- examples/slurm.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/slurm.cmd b/examples/slurm.cmd index 65272017..3dcae884 100644 --- a/examples/slurm.cmd +++ b/examples/slurm.cmd @@ -11,7 +11,7 @@ # Each node = 2.4 GHz Xeon Broadwell E5-2680 v4 + 4x 1328 MHz P100 GPU module load anaconda3 -conda activate frnn-tf13.1 +conda activate my_env module load cudatoolkit module load cudnn module load openmpi/cuda-8.0/intel-17.0/3.0.0/64 From 1ed2ca0300a4a079775a0f6a78d08ed894cb2c89 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 17 Oct 2019 11:22:02 -0500 Subject: [PATCH 128/272] Add more details about SSHFS mounting --- docs/PrincetonUTutorial.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/PrincetonUTutorial.md b/docs/PrincetonUTutorial.md index 0e9f35d5..c1b74c15 100644 --- a/docs/PrincetonUTutorial.md +++ b/docs/PrincetonUTutorial.md @@ -213,16 +213,17 @@ A regular FRNN run will produce several outputs and callbacks. Currently supports graph visualization, histograms of weights, activations and biases, and scalar variable summaries of losses and accuracies. -The summaries are written in real time to `/tigress//Graph`. For macOS, you can set up the `sshfs` mount of the `/tigress` filesystem and view those summaries in your browser. +The summaries are written in real time to `/tigress//Graph`. For macOS, you can set up the `sshfs` mount of the [`/tigress`](https://researchcomputing.princeton.edu/storage/tigress) filesystem and view those summaries in your browser. To install SSHFS on a macOS system, you could follow the instructions here: https://github.com/osxfuse/osxfuse/wiki/SSHFS Or use [Homebrew](https://brew.sh/), `brew cask install osxfuse; brew install sshfs`. Note, to install and/or use `osxfuse` you may need to enable its kernel extension in: System Preferences → Security & Privacy → General -then do something like: +After installation, execute: ``` sshfs -o allow_other,defer_permissions netid@tigergpu.princeton.edu:/tigress// / ``` +The local destination folder may be an existing (possibly nonempty) folder. If it does not exist, SSHFS will create the folder. You can confirm that the operation succeeded via the `mount` command, which prints the list of currently mounted filesystems if no arguments are given. Launch TensorBoard locally (assuming that it is installed on your local computer): ``` @@ -234,6 +235,19 @@ You should see something like: ![tensorboard example](https://github.com/PPPLDeepLearning/plasma-python/blob/master/docs/tb.png) +When you are finished with analyzing the summaries in TensorBoard, you may wish to unmount the remote filesystem: +``` +umount +``` +The local destination folder will remain present, but it will no longer contain the remote files. It will be returned to its previous state, either empty or containing the original local files. Note, the `umount` command is appropriate for macOS systems; some Linux systems instead offer the `fusermount` command. + +These commands may be useful when the SSH connection is lost and an existing mount point cannot be re-mounted, e.g. errors such as: +``` +mount_osxfuse: mount point is itself on a OSXFUSE volume +``` + +More aggressive options such as `umount -f ` and alternative approaches may be necessary; see [discussion here](https://github.com/osxfuse/osxfuse/issues/45#issuecomment-21943107). + #### Learning curves and ROC per epoch Besides TensorBoard summaries you can produce the ROC curves for validation and test data as well as visualizations of shots: From 575b8350694b977780a0159e194bead3c003efca Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 17 Oct 2019 15:25:45 -0400 Subject: [PATCH 129/272] Need to use allow_pickle=True when loading data in performance.py --- plasma/utils/performance.py | 52 ++++++++++++------------------------- 1 file changed, 16 insertions(+), 36 deletions(-) diff --git a/plasma/utils/performance.py b/plasma/utils/performance.py index 5699bc62..b4edf9cd 100644 --- a/plasma/utils/performance.py +++ b/plasma/utils/performance.py @@ -359,10 +359,9 @@ def load_ith_file(self): results_files = os.listdir(self.results_dir) print(results_files) dat = np.load(self.results_dir + results_files[self.i], - allow_pickle=False) + allow_pickle=True) print("Loading results file {}".format( self.results_dir + results_files[self.i])) - if self.verbose: print('configuration: {} '.format(dat['conf'])) @@ -399,7 +398,7 @@ def assert_same_lists(self, shot_list, truth_arr, disr_arr): print(shot_list.shots[i].number) print((s.shape, truth_arr[i].shape, disr_arr[i])) assert(truth_arr[i].shape[0] == s.shape[0]-30) - print("Same Shape!") + print("Same shape!") def print_conf(self): pprint(self.saved_conf) @@ -505,26 +504,23 @@ def compute_tradeoffs_and_print(self, mode): P_thresh_opt = P_thresh_range[idx] self.summarize_shot_prediction_stats_by_mode( P_thresh_opt, mode, verbose=True) - print( - '============= AT P_THRESH = {} ============='.format( - P_thresh_opt)) + print('============= AT P_THRESH = {} ============='.format( + P_thresh_opt)) else: print('No such P_thresh found') print('') # last index where for missed_thresh in missed_threshs: - print( - '============= MISSED RATE < {} ============='.format( - missed_thresh)) + print('============= MISSED RATE < {} ============='.format( + missed_thresh)) if(any(missed_range < missed_thresh)): idx = np.where(missed_range <= missed_thresh)[0][-1] P_thresh_opt = P_thresh_range[idx] self.summarize_shot_prediction_stats_by_mode( P_thresh_opt, mode, verbose=True) - print( - '============= AT P_THRESH = {} ============='.format( - P_thresh_opt)) + print('============= AT P_THRESH = {} ============='.format( + P_thresh_opt)) else: print('No such P_thresh found') print('') @@ -554,7 +550,6 @@ def compute_tradeoffs_and_print_from_training(self): # first index where... for fp_thresh in fp_threshs: - print('============= TRAINING FP RATE < {} ============='.format( fp_thresh)) print('============= TEST PERFORMANCE: =============') @@ -660,7 +655,6 @@ def plot_individual_shot(self, P_thresh_opt, shot_num, normalize=True, def get_prediction_type_for_individual_shot(self, P_thresh, shot, mode='test'): p, t, is_disr = self.get_pred_truth_disr_by_shot(shot) - TP, FP, FN, TN, early, late = self.get_shot_prediction_stats( P_thresh, p, t, is_disr) prediction_type = self.get_prediction_type(TP, FP, FN, TN, early, late) @@ -687,7 +681,6 @@ def example_plots(self, P_thresh_opt, mode='test', types_to_plot=['FP'], p = pred[i] is_disr = is_disruptive[i] shot = shot_list.shots[i] - TP, FP, FN, TN, early, late = self.get_shot_prediction_stats( P_thresh_opt, p, t, is_disr) prediction_type = self.get_prediction_type( @@ -802,20 +795,12 @@ def plot_shot(self, shot, save_fig=True, normalize=True, truth=None, # ax.axvline(len(truth)-T_max_warn,color='r')#,label='max # warning time') # ,label='min warning time') - ax.axvline( - len(truth) - - self.T_min_warn, - color='r', - linewidth=0.5) + ax.axvline(len(truth) - self.T_min_warn, color='r', + linewidth=0.5) ax.set_xlabel('T [ms]', size=fontsize) # ax.axvline(2400) - ax.legend( - loc=( - 0.5, - 0.7), - fontsize=fontsize-5, - labelspacing=0.1, - frameon=False) + ax.legend(loc=(0.5, 0.7), fontsize=fontsize-5, + labelspacing=0.1, frameon=False) plt.setp(ax.get_yticklabels(), fontsize=fontsize) plt.setp(ax.get_xticklabels(), fontsize=fontsize) # plt.xlim(0,200) @@ -890,15 +875,10 @@ def plot_shot_old(self, shot, save_fig=True, normalize=True, truth=None, # ax.set_ylim([1e-5,1.1e0]) ax.set_ylim([-2, 2]) if len(truth)-self.T_max_warn >= 0: - ax.axvline( - len(truth)-self.T_max_warn, - color='r', - label='min warning time') - ax.axvline( - len(truth) - - self.T_min_warn, - color='r', - label='max warning time') + ax.axvline(len(truth)-self.T_max_warn, color='r', + label='min warning time') + ax.axvline(len(truth) - self.T_min_warn, color='r', + label='max warning time') ax.set_xlabel('T [ms]') # ax.legend(loc = 'lower left',fontsize=10) plt.setp(ax.get_yticklabels(), fontsize=7) From 80f3a72d53e44ee5109363c5121def7d55cd4bd3 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 22 Oct 2019 11:59:43 -0500 Subject: [PATCH 130/272] Update PrincetonUTutorial.md --- docs/PrincetonUTutorial.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/PrincetonUTutorial.md b/docs/PrincetonUTutorial.md index c1b74c15..e65f516a 100644 --- a/docs/PrincetonUTutorial.md +++ b/docs/PrincetonUTutorial.md @@ -56,6 +56,7 @@ you should see something like this: $ which mpicc /usr/local/openmpi/cuda-8.0/3.0.0/intel170/x86_64/bin/mpicc ``` +Especially note the presence of the CUDA directory in this path. This indicates that the loaded OpenMPI library is [CUDA-aware](https://www.open-mpi.org/faq/?category=runcuda). If you `conda activate` the Anaconda environment **after** loading the OpenMPI library, your application would be built with the MPI library from Anaconda, which has worse performance on this cluster and could lead to errors. See [On Computing Well: Installing and Running ‘mpi4py’ on the Cluster](https://oncomputingwell.princeton.edu/2018/11/installing-and-running-mpi4py-on-the-cluster/) for a related discussion. @@ -142,12 +143,25 @@ The workflow is to request an interactive session: ```bash salloc -N [X] --ntasks-per-node=4 --ntasks-per-socket=2 --gres=gpu:4 -c 4 --mem-per-cpu=0 -t 0-6:00 ``` + +[//]: # (Note, the modules might not/are not inherited from the shell that spawns the interactive Slurm session. Need to reload anaconda module, activate environment, and reload other compiler/library modules) + +Re-load the above modules and reactivate your conda environment. Confirm that the correct CUDA-aware OpenMPI library is in your interactive Slurm sessions's shell search path: +```bash +$ which mpirun +/usr/local/openmpi/cuda-8.0/3.0.0/intel170/x86_64/bin/mpirun +``` Then, launch the application from the command line: ```bash mpirun -N 4 python mpi_learn.py ``` -where `-N` is a synonym for `-npernode` in OpenMPI. Do **not** use `srun` to launch the job inside an interactive session. +where `-N` is a synonym for `-npernode` in OpenMPI. Do **not** use `srun` to launch the job inside an interactive session. If +you an encounter an error such as "unrecognized argument N", it is likely that your modules are incorrect and point to an Intel MPI distribution instead of CUDA-aware OpenMPI. Intel MPI is based on MPICH, which does not offer the `-npernode` option. You can confirm this by checking: +```bash +$ which mpirun +/opt/intel/compilers_and_libraries_2019.3.199/linux/mpi/intel64/bin/mpirun +``` [//]: # (This option appears to be redundant given the salloc options; "mpirun python mpi_learn.py" appears to work just the same.) @@ -155,7 +169,7 @@ where `-N` is a synonym for `-npernode` in OpenMPI. Do **not** use `srun` to lau [//]: # (Consistent with https://www.open-mpi.org/faq/?category=slurm ?) -[//]: # (certain output seems to be repeated by ntasks-per-node, e.g. echoing the conf.yaml. Expected?) +[//]: # (certain output seems to be repeated by ntasks-per-node, e.g. echoing the conf.yaml. Expected? Or, replace the print calls with print_unique) ### Understanding the data From c09d7015a71ff745eeeac8da95ab89a42d1f5a01 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 22 Oct 2019 18:29:42 -0400 Subject: [PATCH 131/272] Add outputs of performance_analysis.py to gitignore --- .gitignore | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0125b4ab..3b24d3a7 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,11 @@ # Generated by test plot_*.html +# Outputs from analysis scripts +*.png +out.txt +*.npz + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -97,6 +102,7 @@ ENV/ .ropeproject # Job scheduler output +################ # Slurm *.out @@ -107,4 +113,7 @@ ENV/ # PBS # *.o* -# *.e* \ No newline at end of file +# *.e* + +# Etc +*.local \ No newline at end of file From 67f2cdbe632113e96bdbe2cc8c6930a11ee45825 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 22 Oct 2019 18:30:53 -0400 Subject: [PATCH 132/272] Pass allow_pickle=True to final np.savez() call in MPI script (even though option is not documented for this function, and deprecation should not cause an error until v1.18.0, which has not been released) Prevents the following error with numpy v1.16.2, tf v1.3.0: /home/kfelker/.conda/envs/frnn-tf1.3.0/lib/python3.6/site-packages/numpy/lib/npyio.py:750: FutureWarning: Object arrays will not be saved by default in the future because `allow_pickle` will default to False. You should add `allow_pickle=True` explicitly to eliminate this warning. pickle_kwargs=pickle_kwargs) Traceback (most recent call last): File "./mpi_learn.py", line 151, in shot_list_test=shot_list_test, conf=conf) File "/home/kfelker/.conda/envs/frnn-tf1.3.0/lib/python3.6/site-packages/numpy/lib/npyio.py", line 639, in savez _savez(file, args, kwds, False, allow_pickle=_allow_pickle) File "/home/kfelker/.conda/envs/frnn-tf1.3.0/lib/python3.6/site-packages/numpy/lib/npyio.py", line 750, in _savez pickle_kwargs=pickle_kwargs) File "/home/kfelker/.conda/envs/frnn-tf1.3.0/lib/python3.6/site-packages/numpy/lib/format.py", line 637, in write_array raise ValueError("Object arrays cannot be saved when " ValueError: Object arrays cannot be saved when allow_pickle=False --- examples/mpi_learn.py | 5 ++++- plasma/models/mpi_runner.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/mpi_learn.py b/examples/mpi_learn.py index 79a6d783..97b2c691 100644 --- a/examples/mpi_learn.py +++ b/examples/mpi_learn.py @@ -148,7 +148,10 @@ y_prime_test=y_prime_test, disruptive=disruptive, disruptive_train=disruptive_train, disruptive_test=disruptive_test, shot_list_train=shot_list_train, - shot_list_test=shot_list_test, conf=conf) + shot_list_test=shot_list_test, conf=conf, + # TODO(KGF): changing allow_pickle behavior not documented for + # https://docs.scipy.org/doc/numpy/reference/generated/numpy.savez.html + allow_pickle=True) sys.stdout.flush() if task_index == 0: diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index e1d55ba6..29eb1573 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -217,6 +217,8 @@ def set_batch_iterator_func(self): self.batch_iterator_func = self.batch_iterator() def close(self): + # TODO(KGF): extend __exit__() fn capability when this member + # = self.batch_iterator() (i.e. is not a ProcessGenerator()) if (self.conf is not None and 'use_process_generator' in conf['training'] and conf['training']['use_process_generator']): From b4b147c195120f638c492d3fbffe2f5b7d98f5cb Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 22 Oct 2019 18:43:11 -0400 Subject: [PATCH 133/272] Clean up default conf.yaml --- examples/conf.yaml | 103 +++++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 50 deletions(-) diff --git a/examples/conf.yaml b/examples/conf.yaml index 2b977191..e125a391 100644 --- a/examples/conf.yaml +++ b/examples/conf.yaml @@ -1,56 +1,57 @@ -#conf.py will parse the yaml and extract parameters based on what is specified +# conf.py will parse the yaml and extract parameters based on what is specified -#will do stuff in fs_path / [username] / signal_data | shot_lists | processed shots, etc. +# will do stuff in fs_path / [username] / signal_data | shot_lists | processed shots, etc. fs_path: '/tigress' -target: 'hinge' #'maxhinge' #'maxhinge' #'binary' #'hinge' +target: 'hinge' # 'maxhinge' # 'maxhinge' # 'binary' # 'hinge' num_gpus: 4 paths: - signal_prepath: '/signal_data/' #/signal_data/jet/ + signal_prepath: '/signal_data/' # /signal_data/jet/ shot_list_dir: '/shot_lists/' tensorboard_save_path: '/Graph/' - data: d3d_data_0D #'d3d_to_jet_data' #'d3d_to_jet_data' # 'jet_to_d3d_data' #jet_data - specific_signals: [] #['q95','li','ip','betan','energy','lm','pradcore','pradedge','pradtot','pin','torquein','tmamp1','tmamp2','tmfreq1','tmfreq2','pechin','energydt','ipdirect','etemp_profile','edens_profile'] #if left empty will use all valid signals defined on a machine. Only use if need a custom set + data: d3d_data_0D # 'd3d_to_jet_data' # 'd3d_to_jet_data' # 'jet_to_d3d_data' # jet_data + # if specific_signals: [] left empty, it will use all valid signals defined on a machine. Only use if need a custom set + specific_signals: [] # ['q95','li','ip','betan','energy','lm','pradcore','pradedge','pradtot','pin','torquein','tmamp1','tmamp2','tmfreq1','tmfreq2','pechin','energydt','ipdirect','etemp_profile','edens_profile'] executable: "mpi_learn.py" shallow_executable: "learn.py" data: - bleed_in: 0 #how many shots from the test sit to use in training? - bleed_in_repeat_fac: 1 #how many times to repeat shots in training and validation? + bleed_in: 0 # how many shots from the test sit to use in training? + bleed_in_repeat_fac: 1 # how many times to repeat shots in training and validation? bleed_in_remove_from_test: True bleed_in_equalize_sets: False - signal_to_augment: None #'plasma current' #or None + # TODO(KGF): make next parameter use 'none' instead of None + signal_to_augment: None # 'plasma current' # or None augmentation_mode: 'none' augment_during_training: False cut_shot_ends: True T_min_warn: 30 recompute: False recompute_normalization: False - #specifies which of the signals in the signals_dirs order contains the plasma current info + # specifies which of the signals in the signals_dirs order contains the plasma current info current_index: 0 plotting: False - #train/validate split - #how many shots to use - use_shots: 200000 #1000 #200000 - positive_example_penalty: 1.0 #by what factor to upweight positive examples? - #normalization timescale + # how many shots to use + use_shots: 200000 # 1000 # 200000 + positive_example_penalty: 1.0 # by what factor to upweight positive examples? + # normalization timescale dt: 0.001 - #maximum TTD considered + # maximum TTD considered T_max: 1000.0 - #The shortest works best so far: less overfitting. log TTd prediction also works well. 0.5 better than 0.2 - T_warning: 1.024 #1.024 #1.024 #0.512 #0.25 #1.0 #1.0 #warning time in seconds + # The shortest works best so far: less overfitting. log TTd prediction also works well. 0.5 better than 0.2 + T_warning: 1.024 # 1.024 # 1.024 # 0.512 # 0.25 # 1.0 # 1.0 # warning time in seconds current_thresh: 750000 current_end_thresh: 10000 - #the characteristic decay length of the decaying moving average window + # the characteristic decay length of the decaying moving average window window_decay: 2 - #the width of the actual window + # the width of the actual window window_size: 10 - #TODO optimize + # TODO(KGF): optimize the normalizer parameters normalizer: 'var' norm_stat_range: 100.0 equalize_classes: False - # shallow_sample_prob: 0.01 #the fraction of samples with which to train the shallow model + # shallow_sample_prob: 0.01 # the fraction of samples with which to train the shallow model floatx: 'float32' model: @@ -59,31 +60,31 @@ model: torch: False shallow: False shallow_model: - num_samples: 1000000 #1000000 #the number of samples to use for training - type: "xgboost" #"xgboost" #"xgboost" #"random_forest" "xgboost" - n_estimators: 100 #for random forest - max_depth: 3 #for random forest and xgboost (def = 3) - C: 1.0 #for svm - kernel: "rbf" #rbf, sigmoid, linear, poly, for svm - learning_rate: 0.1 #xgboost - scale_pos_weight: 10.0 #xgboost - final_hidden_layer_size: 10 #final layers has this many neurons, every layer before twice as many + num_samples: 1000000 # 1000000 # the number of samples to use for training + type: "xgboost" # "xgboost" #"random_forest" + n_estimators: 100 # for random forest + max_depth: 3 # for random forest and xgboost (def = 3) + C: 1.0 # for svm + kernel: "rbf" # rbf, sigmoid, linear, poly, for svm + learning_rate: 0.1 # used in xgboost + scale_pos_weight: 10.0 # used in xgboost + final_hidden_layer_size: 10 # final layers has this many neurons, every layer before twice as many num_hidden_layers: 3 learning_rate_mlp: 0.0001 mlp_regularization: 0.0001 - skip_train: False #should a finished model be loaded if available - #length of LSTM memory + skip_train: False # should a finished model be loaded if available + # length of LSTM memory pred_length: 200 pred_batch_size: 128 - #TODO optimize + # TODO(KGF): optimize length of LSTM memory length: 128 skip: 1 - #hidden layer size - #TODO optimize + # hidden layer size + # TODO(KGF): optimize size of RNN layers rnn_size: 200 - #size 100 slight overfitting, size 20 no overfitting. 200 is not better than 100. Prediction much better with size 100, size 20 cannot capture the data. + # size 100 slight overfitting, size 20 no overfitting. 200 is not better than 100. Prediction much better with size 100, size 20 cannot capture the data. rnn_type: 'LSTM' - #TODO optimize + # TODO(KGF): optimize number of RNN layers rnn_layers: 2 num_conv_filters: 128 size_conv_filters: 3 @@ -91,39 +92,41 @@ model: pool_size: 2 dense_size: 128 extra_dense_input: False - #have not found a difference yet + # have not found a difference yet optimizer: 'adam' clipnorm: 10.0 regularization: 0.001 dense_regularization: 0.001 - #1e-4 is too high, 5e-7 is too low. 5e-5 seems best at 256 batch size, full dataset and ~10 epochs, and lr decay of 0.90. 1e-4 also works well if we decay a lot (i.e ~0.7 or more) - lr: 0.00002 #0.00001 #0.0005 #for adam plots 0.0000001 #0.00005 #0.00005 #0.00005 - lr_decay: 0.97 #0.98 #0.9 + # lr=1e-4 is too high, 5e-7 is too low. 5e-5 seems best at 256 batch size, full dataset + # and ~10 epochs, and lr decay of 0.90 + # lr=1e-4 also works well if we decay a lot (i.e ~0.7 or more) + lr: 0.00002 # 0.00001 # 0.0005 # for adam plots 0.0000001 # 0.00005 # 0.00005 # 0.00005 + lr_decay: 0.97 # 0.98 # 0.9 stateful: True return_sequences: True dropout_prob: 0.1 - #only relevant if we want to do mpi training. The number of steps with a single replica + # only relevant if we want to do mpi training. The number of steps with a single replica warmup_steps: 0 - ignore_timesteps: 100 #how many initial timesteps to ignore during evaluation (to let the internal state settle) + ignore_timesteps: 100 # how many initial timesteps to ignore during evaluation (to let the internal state settle) backend: 'tensorflow' training: as_array_of_shots: True shuffle_training: True train_frac: 0.75 validation_frac: 0.33 - batch_size: 128 #256 - #THIS WAS THE CULPRIT FOR NO TRAINING! Lower than 1000 performs very poorly + batch_size: 128 # 256 + # THE MAX_PATCH_LENGTH WAS THE CULPRIT FOR NO TRAINING! Lower than 1000 performs very poorly max_patch_length: 100000 - #How many shots are we loading at once? + # How many shots are we loading at once? num_shots_at_once: 200 - num_epochs: 1000 + num_epochs: 1000 # large number = maximum number of epochs. Early stopping will occur if loss does not decrease use_mock_data: False data_parallel: False hyperparam_tuning: False batch_generator_warmup_steps: 0 use_process_generator: False - num_batches_minimum: 20 #minimum number of batches per epoch - ranking_difficulty_fac: 1.0 #how much to upweight incorrectly classified shots during training + num_batches_minimum: 20 # minimum number of batches per epoch + ranking_difficulty_fac: 1.0 # how much to upweight incorrectly classified shots during training callbacks: list: ['earlystop'] metrics: ['val_loss','val_roc','train_loss'] From 1bcf9622e3ead9c3ea814995d51b6da2ff398737 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 23 Oct 2019 12:03:41 -0400 Subject: [PATCH 134/272] Revert change from 67f2cdbe63 API change only occurred for Intel NumPy distribution --- examples/mpi_learn.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/mpi_learn.py b/examples/mpi_learn.py index 97b2c691..8c4e7828 100644 --- a/examples/mpi_learn.py +++ b/examples/mpi_learn.py @@ -148,10 +148,12 @@ y_prime_test=y_prime_test, disruptive=disruptive, disruptive_train=disruptive_train, disruptive_test=disruptive_test, shot_list_train=shot_list_train, - shot_list_test=shot_list_test, conf=conf, - # TODO(KGF): changing allow_pickle behavior not documented for - # https://docs.scipy.org/doc/numpy/reference/generated/numpy.savez.html - allow_pickle=True) + shot_list_test=shot_list_test, conf=conf) + + # TODO(KGF): Intel NumPy fork + # https://conda.anaconda.org/intel/linux-64/numpy-1.16.2-py36h7b7c402_0.tar.bz2 + # applies cve_2019_6446_fix.patch, which unlike main NumPy, adds + # requirement for "allow_pickle=True" to savez() calls sys.stdout.flush() if task_index == 0: From 79738ae2f524fe463ba64b5434c8bc2838f652ff Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 23 Oct 2019 12:19:50 -0400 Subject: [PATCH 135/272] Add flake8 to Conda requirements file (eventually will be environments.yaml) --- requirements-travis.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-travis.txt b/requirements-travis.txt index 5da3c8a5..77056dd9 100644 --- a/requirements-travis.txt +++ b/requirements-travis.txt @@ -1,5 +1,6 @@ scipy pandas +flake8 h5py pyparsing pyyaml From 43110e7be42f32ca2bc6f35a3ac7ab090c6f97e5 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 23 Oct 2019 12:44:40 -0400 Subject: [PATCH 136/272] Loosen version requirements for matplotlib, keras --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 3f6b58ce..da722f07 100644 --- a/setup.py +++ b/setup.py @@ -26,9 +26,9 @@ # license = "Apache Software License v2", test_suite="tests", install_requires=[ - 'keras>2.0.8', + 'keras>=2.0.5', 'pathos', - 'matplotlib==2.0.2', + 'matplotlib>=2.0.2', 'hyperopt', 'mpi4py', 'xgboost', From d0139802b22f9a0ac7e722ac435275fe7ecc65a9 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 24 Oct 2019 14:53:51 -0500 Subject: [PATCH 137/272] Reorganize sections --- docs/PrincetonUTutorial.md | 151 +++++++++++++++++++------------------ 1 file changed, 77 insertions(+), 74 deletions(-) diff --git a/docs/PrincetonUTutorial.md b/docs/PrincetonUTutorial.md index e65f516a..6493b7a1 100644 --- a/docs/PrincetonUTutorial.md +++ b/docs/PrincetonUTutorial.md @@ -1,6 +1,7 @@ -## Tutorials -*Last updated 2019-10-16.* +# TigerGPU Tutorial +*Last updated 2019-10-24.* +## Building the package ### Login to TigerGPU First, login to TigerGPU cluster headnode via ssh: @@ -9,7 +10,7 @@ ssh -XC @tigergpu.princeton.edu ``` Note, `-XC` is optional; it is only necessary if you are planning on performing remote visualization, e.g. the output `.png` files from the below [section](#Learning-curves-and-ROC-per-epoch). Trusted X11 forwarding can be used with `-Y` instead of `-X` and may prevent timeouts, but it disables X11 SECURITY extension controls. Compression `-C` reduces the bandwidth usage and may be useful on slow connections. -### Sample usage on TigerGPU +### Sample installation on TigerGPU Next, check out the source code from github: ``` @@ -48,7 +49,7 @@ python setup.py install Where `my_env` should contain the Python packages as per `requirements-travis.txt` file. -#### Common issue +### Common build issue: cluster's MPI library and `mpi4py` Common issue is Intel compiler mismatch in the `PATH` and what you use in the module. With the modules loaded as above, you should see something like this: @@ -60,7 +61,9 @@ Especially note the presence of the CUDA directory in this path. This indicates If you `conda activate` the Anaconda environment **after** loading the OpenMPI library, your application would be built with the MPI library from Anaconda, which has worse performance on this cluster and could lead to errors. See [On Computing Well: Installing and Running ‘mpi4py’ on the Cluster](https://oncomputingwell.princeton.edu/2018/11/installing-and-running-mpi4py-on-the-cluster/) for a related discussion. -#### Location of the data on Tigress + +## Understanding and preparing the input data +### Location of the data on Tigress The JET and D3D datasets contain multi-modal time series of sensory measurements leading up to deleterious events called plasma disruptions. The datasets are located in the `/tigress/FRNN` project directory of the [GPFS](https://www.ibm.com/support/knowledgecenter/en/SSPT3X_3.0.0/com.ibm.swg.im.infosphere.biginsights.product.doc/doc/bi_gpfs_overview.html) filesystem on Princeton University clusters. @@ -71,7 +74,29 @@ ln -s /tigress/FRNN/shot_lists shot_lists ln -s /tigress/FRNN/signal_data signal_data ``` -#### Preprocessing +### Configuring the dataset +All the configuration parameters are summarised in `examples/conf.yaml`. In this section, we highlight the important ones used to control the input data. + +Currently, FRNN is capable of working with JET and D3D data as well as thecross-machine regime. The switch is done in the configuration file: + +```yaml +paths: + ... + data: 'jet_data_0D' +``` +use `d3d_data` for D3D signals, use `jet_to_d3d_data` ir `d3d_to_jet_data` for cross-machine regime. + +By default, FRNN will select, preprocess, and normalize all valid signals available in the above dataset. To chose only specific signals use: +```yaml +paths: + ... + specific_signals: [q95,ip] +``` +if left empty `[]` will use all valid signals defined on a machine. Only set this variable if you need a custom set of signals. + +Other parameters configured in the `conf.yaml` include batch size, learning rate, neural network topology and special conditions foir hyperparameter sweeps. + +### Preprocessing the input data ```bash cd examples/ @@ -79,20 +104,45 @@ python guarantee_preprocessed.py ``` This will preprocess the data and save rescaled copies of the signals in `/tigress//processed_shots`, `/tigress//processed_shotlists` and `/tigress//normalization` -You would only have to run preprocessing once for each dataset. The dataset is specified in the config file `examples/conf.yaml`: +Preprocessing must be performed only once per each dataset. For example, consider the following dataset specified in the config file `examples/conf.yaml`: ```yaml paths: data: jet_data_0D ``` Preprocessing this dataset takes about 20 minutes to preprocess in parallel and can normally be done on the cluster headnode. -#### Training and inference +### Current signals and notations + +Signal name | Description +--- | --- +q95 | q95 safety factor +ip | plasma current +li | internal inductance +lm | Locked mode amplitude +dens | Plasma density +energy | stored energy +pin | Input Power (beam for d3d) +pradtot | Radiated Power +pradcore | Radiated Power Core +pradedge | Radiated Power Edge +pechin | ECH input power, not always on +pechin | ECH input power, not always on +betan | Normalized Beta +energydt | stored energy time derivative +torquein | Input Beam Torque +tmamp1 | Tearing Mode amplitude (rotating 2/1) +tmamp2 | Tearing Mode amplitude (rotating 3/2) +tmfreq1 | Tearing Mode frequency (rotating 2/1) +tmfreq2 | Tearing Mode frequency (rotating 3/2) +ipdirect | plasma current direction + +## Training and inference -Use Slurm scheduler to perform batch or interactive analysis on TigerGPU cluster. +Use the Slurm job scheduler to perform batch or interactive analysis on TigerGPU cluster. -##### Batch analysis +### Batch job -For batch analysis, make sure to allocate 1 MPI process per GPU. Save the following to `slurm.cmd` file (or make changes to the existing `examples/slurm.cmd`): +For non-interactive batch analysis, make sure to allocate exactly 1 MPI process per GPU. Save the following to `slurm.cmd` file (or make changes to the existing `examples/slurm.cmd`): ```bash #!/bin/bash @@ -135,7 +185,7 @@ Optionally, add an email notification option in the Slurm configuration about th #SBATCH --mail-type=ALL ``` -##### Interactive analysis +### Interactive job Interactive option is preferred for **debugging** or running in the **notebook**, for all other case batch is preferred. The workflow is to request an interactive session: @@ -165,65 +215,13 @@ $ which mpirun [//]: # (This option appears to be redundant given the salloc options; "mpirun python mpi_learn.py" appears to work just the same.) -[//]: # (HOWEVER, "srun python mpi_learn.py", "srun --ntasks-per-node python mpi_learn.py", etc. NEVER works--- it just hangs without any output. Why?) - -[//]: # (Consistent with https://www.open-mpi.org/faq/?category=slurm ?) - -[//]: # (certain output seems to be repeated by ntasks-per-node, e.g. echoing the conf.yaml. Expected? Or, replace the print calls with print_unique) +[//]: # (HOWEVER, "srun python mpi_learn.py", "srun --ntasks-per-node python mpi_learn.py", etc. NEVER works--- it just hangs without any output. Why? ANSWER: salloc starts a session on the node using srun under the covers, which may consume a GPU in the allocation. Next srun call will hang due to a lack of required resources. Wrapper fix to this may have been extended from mpirun to srun on 2019-10-22) - -### Understanding the data - -All the configuration parameters are summarised in `examples/conf.yaml`. Highlighting the important ones to control the data. -Currently, FRNN is capable of working with JET and D3D data as well as cross-machine regime. The switch is done in the configuration file: - -```yaml -paths: - ... - data: 'jet_data_0D' -``` -use `d3d_data` for D3D signals, use `jet_to_d3d_data` ir `d3d_to_jet_data` for cross-machine regime. - -By default, FRNN will select, preprocess and normalize all valid signals available. To chose only specific signals use: -```yaml -paths: - ... - specific_signals: [q95,ip] -``` -if left empty `[]` will use all valid signals defined on a machine. Only use if need a custom set. - -Other parameters configured in the conf.yaml include batch size, learning rate, neural network topology and special conditions foir hyperparameter sweeps. - -### Current signals and notations - -Signal name | Description ---- | --- -q95 | q95 safety factor -ip | plasma current -li | internal inductance -lm | Locked mode amplitude -dens | Plasma density -energy | stored energy -pin | Input Power (beam for d3d) -pradtot | Radiated Power -pradcore | Radiated Power Core -pradedge | Radiated Power Edge -pechin | ECH input power, not always on -pechin | ECH input power, not always on -betan | Normalized Beta -energydt | stored energy time derivative -torquein | Input Beam Torque -tmamp1 | Tearing Mode amplitude (rotating 2/1) -tmamp2 | Tearing Mode amplitude (rotating 3/2) -tmfreq1 | Tearing Mode frequency (rotating 2/1) -tmfreq2 | Tearing Mode frequency (rotating 3/2) -ipdirect | plasma current direction - -### Visualizing learning +## Visualizing learning A regular FRNN run will produce several outputs and callbacks. -#### TensorBoard visualization +### TensorBoard visualization Currently supports graph visualization, histograms of weights, activations and biases, and scalar variable summaries of losses and accuracies. @@ -262,19 +260,23 @@ mount_osxfuse: mount point is itself on More aggressive options such as `umount -f ` and alternative approaches may be necessary; see [discussion here](https://github.com/osxfuse/osxfuse/issues/45#issuecomment-21943107). -#### Learning curves and ROC per epoch +## Custom visualization +Besides TensorBoard summaries, you can visualize the accuracy of the trained FRNN model using the custom Python scripts and notebooks included in the repository. + +### Learning curves, example shots, and ROC per epoch -Besides TensorBoard summaries you can produce the ROC curves for validation and test data as well as visualizations of shots: +You can produce the ROC curves for validation and test data as well as visualizations of shots by using: ``` cd examples/ python performance_analysis.py ``` -this uses the resulting file produced as a result of training the neural network as an input, and produces several `.png` files with plots as an output. +The `performance_analysis.py` script uses the file produced as a result of training the neural network as an input, and produces several `.png` files with plots as an output. -In addition, you can check the scalar variable summaries for training loss, validation loss and validation ROC logged at `/tigress//csv_logs` (each run will produce a new log file with a timestamp in name). +[//]: # (Add details about sig_161308test.npz, disruptive_alarms_test.npz, 4x metric* png, accum_disruptions.png, test_roc.npz) -A sample code to analyze can be found in `examples/notebooks`. For instance: +In addition, you can check the scalar variable summaries for training loss, validation loss, and validation ROC logged at `/tigress//csv_logs` (each run will produce a new log file with a timestamp in name). +Sample notebooks for analyzing the files in this directory can be found in `examples/notebooks/`. For instance, the [LearningCurves.ipynb](https://github.com/PPPLDeepLearning/plasma-python/blob/master/examples/notebooks/LearningCurves.ipynb) notebook contains a variation on the following code snippet: ```python import pandas as pd import numpy as np @@ -297,7 +299,8 @@ p.line(data['epoch'].values, data['train_loss'].values, legend="Test description p.legend.location = "top_right" show(p, notebook_handle=True) ``` +The resulting plot should match the `train_loss` plot in the Scalars tab of the TensorBoard summary. -### Learning curve summaries per mini-batch +#### Learning curve summaries per mini-batch -To extract per mini-batch summaries, use the output produced by FRNN logged to the standard out (in case of the batch jobs, it will all be contained in the Slurm output file). Refer to the following notebook to perform the analysis of learning curve on a mini-batch level: [FRNN_scaling.ipynb](https://github.com/PPPLDeepLearning/plasma-python/blob/master/examples/notebooks/FRNN_scaling.ipynb) +To extract per mini-batch summaries, we require a finer granularity of checkpoint data than what it is logged to the per-epoch lines of `csv_logs/` files. We must directly use the output produced by FRNN logged to the standard output stream. In the case of the non-interactive Slurm batch jobs, it will all be contained in the Slurm output file, e.g. `slurm-3842170.out`. Refer to the following notebook to perform the analysis of learning curve on a mini-batch level: [FRNN_scaling.ipynb](https://github.com/PPPLDeepLearning/plasma-python/blob/master/examples/notebooks/FRNN_scaling.ipynb) From f25721736a54d4e707e849debf379b9ada124be6 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Mon, 28 Oct 2019 13:50:00 -0500 Subject: [PATCH 138/272] Try fixing general_object_hash() By swapping order of frozenset(sorted( to sorted(frozenset(, which is then converted to a tuple and hashed with myhash(). The previous setup returned different unique_id from the same conf.yaml input between different invocations of the Python kernel. Root cause was an arbitrary reordering of the items in all nested dictionaries: E.g. the conf['training'] nested dictionary was hashed twice as: < (('data_parallel', 289442542916728710689234258135348595619), ('num_shots_at_once', 89470398321342590012591641233937892196), ('shuffle_training', 199116753875903447238788905392232456494), ('max_patch_length', 28194631872103329448916222404393283486), ('hyperparam_tuning', 289442542916728710689234258135348595619), ('num_batches_minimum', 241470210063517338676282364296715124371), ('ranking_difficulty_fac', 212610832367597099128411062743096585095), ('batch_generator_warmup_steps', 220016661218577614175283909151483631217), ('train_frac', 74563758320577896151616107095405697202), ('validation_frac', 313419366237728314877788597545102977126), ('as_array_of_shots', 199116753875903447238788905392232456494), ('use_mock_data', 289442542916728710689234258135348595619), ('batch_size', 205743917378578649043592219116698884292), ('num_epochs', 201299305426511499865320550659865534950), ('use_process_generator', 289442542916728710689234258135348595619)) < final_hash = 238379894380234523292314865703766581029 --- > (('max_patch_length', 28194631872103329448916222404393283486), ('batch_generator_warmup_steps', 220016661218577614175283909151483631217), ('ranking_difficulty_fac', 212610832367597099128411062743096585095), ('use_mock_data', 289442542916728710689234258135348595619), ('num_batches_minimum', 241470210063517338676282364296715124371), ('validation_frac', 313419366237728314877788597545102977126), ('hyperparam_tuning', 289442542916728710689234258135348595619), ('shuffle_training', 199116753875903447238788905392232456494), ('as_array_of_shots', 199116753875903447238788905392232456494), ('num_shots_at_once', 89470398321342590012591641233937892196), ('use_process_generator', 289442542916728710689234258135348595619), ('batch_size', 205743917378578649043592219116698884292), ('data_parallel', 289442542916728710689234258135348595619), ('train_frac', 74563758320577896151616107095405697202), ('num_epochs', 201299305426511499865320550659865534950)) > final_hash = 326207239196943809073363948252009087299 --- plasma/utils/downloading.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/plasma/utils/downloading.py b/plasma/utils/downloading.py index 27c9fb93..b81457a4 100644 --- a/plasma/utils/downloading.py +++ b/plasma/utils/downloading.py @@ -37,7 +37,7 @@ def general_object_hash(o): Makes a hash from a dictionary, list, tuple or set to any level, that contains only other hashable types (including any lists, tuples, sets, and dictionaries). Relies on dill for serialization -""" + """ if isinstance(o, (set, tuple, list)): return tuple([general_object_hash(e) for e in o]) @@ -46,19 +46,17 @@ def general_object_hash(o): return myhash(o) new_o = deepcopy(o) + # recursively call this function when given a dictionary for k, v in new_o.items(): + # replace the dict entry value with its hash new_o[k] = general_object_hash(v) - return myhash(tuple(frozenset(sorted(new_o.items())))) + return myhash(tuple(sorted(frozenset(new_o.items())))) def myhash(x): return int(hashlib.md5((dill.dumps(x).decode('unicode_escape')).encode( 'utf-8')).hexdigest(), 16) - # return int(hashlib.md5((dill.dumps(x))).hexdigest(),16) - # return - # int(hashlib.md5((dill.dumps(x))))#.decode('unicode_escape')).encode( - # 'utf-8')).hexdigest(),16) def get_missing_value_array(): From 0acdaf95415c28dead7032f54031d0c26f7e6364 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Mon, 28 Oct 2019 14:40:29 -0500 Subject: [PATCH 139/272] Fix formatting of some signal preprocessing print statements --- data/signals.py | 2 +- plasma/conf_parser.py | 2 +- plasma/models/builder.py | 7 ++---- plasma/utils/downloading.py | 43 +++++++++++++------------------------ 4 files changed, 19 insertions(+), 35 deletions(-) diff --git a/data/signals.py b/data/signals.py index a4f9f4c3..34a53db2 100644 --- a/data/signals.py +++ b/data/signals.py @@ -361,7 +361,7 @@ def fetch_nstx_data(signal_path, shot_num, c): all_signals_restricted = all_signals -print('all signals (determines which signals are downloaded & preprocessed):') +print('All signals (determines which signals are downloaded & preprocessed):') print(all_signals.values()) fully_defined_signals = { diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index 057c1391..c41431b9 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -368,7 +368,7 @@ def parameters(input_file): params['paths']['all_signals'] = sort_by_channels( list(params['paths']['all_signals_dict'].values())) - print("Selected signals (determines which signals are used for ", + print("Selected signals (determines which signals are used for", "training):\n{}".format(params['paths']['use_signals'])) params['paths']['shot_files_all'] = ( diff --git a/plasma/models/builder.py b/plasma/models/builder.py index d7b2c0a8..9e643d37 100644 --- a/plasma/models/builder.py +++ b/plasma/models/builder.py @@ -37,15 +37,12 @@ def __init__(self, conf): self.conf = conf def get_unique_id(self): - # num_epochs = self.conf['training']['num_epochs'] this_conf = deepcopy(self.conf) - # don't make hash dependent on number of epochs or T_min_warn - # (they can both be modified) + # ignore hash depednecy on number of epochs or T_min_warn (they are + # both modifiable). Map local copy of all confs to the same values this_conf['training']['num_epochs'] = 0 this_conf['data']['T_min_warn'] = 30 unique_id = general_object_hash(this_conf) - # unique_id = int(hashlib.md5((dill.dumps(this_conf).decode( - # 'unicode_escape')).encode('utf-8')).hexdigest(), 16) return unique_id def get_0D_1D_indices(self): diff --git a/plasma/utils/downloading.py b/plasma/utils/downloading.py index b81457a4..f8db18da 100644 --- a/plasma/utils/downloading.py +++ b/plasma/utils/downloading.py @@ -12,7 +12,7 @@ # import gadata # from plasma.primitives.shots import ShotList -''' +'''MDSplus references: http://www.mdsplus.org/index.php?title=Documentation:Tutorial:RemoteAccess&open=76203664636339686324830207&page=Documentation%2FThe+MDSplus+tutorial%2FRemote+data+access+in+MDSplus http://piscope.psfc.mit.edu/index.php/MDSplus_%26_python#Simple_example_of_reading_MDSplus_data http://www.mdsplus.org/documentation/beginners/expressions.shtml @@ -112,8 +112,8 @@ def save_shot(shot_num_queue, c, signals, save_prepath, machine, sentinel=-1): shot_complete = True for signal in signals: signal_path = signal.get_path(machine) - save_path_full = signal.get_file_path( - save_prepath, machine, shot_num) + save_path_full = signal.get_file_path(save_prepath, machine, + shot_num) success = False mapping = None if os.path.isfile(save_path_full): @@ -121,10 +121,9 @@ def save_shot(shot_num_queue, c, signals, save_prepath, machine, sentinel=-1): print('-', end='') success = True else: - print( - 'Signal {}, shot {} '.format(signal_path, shot_num), - 'was downloaded incorrectly (empty file). ', - 'Redownloading.') + print('Signal {}, shot {} '.format(signal_path, shot_num), + 'was downloaded incorrectly (empty file). ', + 'Redownloading.') if not success: try: try: @@ -159,9 +158,8 @@ def save_shot(shot_num_queue, c, signals, save_prepath, machine, sentinel=-1): fmt='%.5e') print('.', end='') except BaseException: - print( - 'Could not save shot {}, signal {}'.format( - shot_num, signal_path)) + print('Could not save shot {}, signal {}'.format( + shot_num, signal_path)) print('Warning: Incomplete!!!') raise sys.stdout.flush() @@ -195,12 +193,8 @@ def download_shot_numbers(shot_numbers, save_prepath, machine, signals): for i in range(num_cores): queue.put(sentinel) connections = [Connection(machine.server) for _ in range(num_cores)] - processes = [ - mp.Process( - target=fn, - args=( - queue, - connections[i])) for i in range(num_cores)] + processes = [mp.Process(target=fn, args=(queue, connections[i])) + for i in range(num_cores)] print('running in parallel on {} processes'.format(num_cores)) @@ -210,13 +204,9 @@ def download_shot_numbers(shot_numbers, save_prepath, machine, signals): p.join() -def download_all_shot_numbers( - prepath, - save_path, - shot_list_files, - signals_full): +def download_all_shot_numbers(prepath, save_path, shot_list_files, + signals_full): max_len = 30000 - machine = shot_list_files.machine signals = [] for sig in signals_full: @@ -226,18 +216,15 @@ def download_all_shot_numbers( sig, machine)) else: signals.append(sig) - save_prepath = prepath+save_path + '/' + save_prepath = prepath + save_path + '/' shot_numbers, _ = shot_list_files.get_shot_numbers_and_disruption_times() # can only use queue of max size 30000 shot_numbers_chunks = [shot_numbers[i:i+max_len] for i in np.xrange(0, len(shot_numbers), max_len)] start_time = time.time() for shot_numbers_chunk in shot_numbers_chunks: - download_shot_numbers( - shot_numbers_chunk, - save_prepath, - machine, - signals) + download_shot_numbers(shot_numbers_chunk, save_prepath, machine, + signals) print('Finished downloading {} shots in {} seconds'.format( len(shot_numbers), time.time()-start_time)) From 2d4dad842a4cb36488cad9bad0630752e032961d Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 29 Oct 2019 11:49:14 -0500 Subject: [PATCH 140/272] Move all hashing into a single, new file --- plasma/conf_parser.py | 13 +---- plasma/models/builder.py | 3 +- plasma/primitives/data.py | 21 +++----- plasma/utils/downloading.py | 30 ----------- plasma/utils/hashing.py | 99 +++++++++++++++++++++++++++++++++++++ 5 files changed, 111 insertions(+), 55 deletions(-) create mode 100644 plasma/utils/hashing.py diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index c41431b9..13b9932f 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -1,13 +1,12 @@ from plasma.primitives.shots import ShotListFiles import data.signals as sig +from plasma.utils.hashing import myhash_signals # from data.signals import ( # all_signals, fully_defined_signals_1D, # jet, d3d) # nstx import getpass import yaml -import hashlib - def parameters(input_file): """Parse yaml file of configuration parameters.""" @@ -28,7 +27,7 @@ def parameters(input_file): params['paths']['shot_list_dir'] = ( base_path + params['paths']['shot_list_dir']) params['paths']['output_path'] = output_path - h = get_unique_signal_hash(sig.all_signals.values()) + h = myhash_signals(sig.all_signals.values()) params['paths']['global_normalizer_path'] = ( output_path + '/normalization/normalization_signal_group_{}.npz'.format(h)) @@ -384,14 +383,6 @@ def parameters(input_file): return params -def get_unique_signal_hash(signals): - return int(hashlib.md5(''.join(tuple(map(lambda x: "{}".format( - x.__hash__()), sorted(signals)))).encode('utf-8')).hexdigest(), 16) - # return int(hashlib.md5(''.join( - # tuple(map(lambda x: x.description, sorted(signals)))).encode( - # 'utf-8')).hexdigest(), 16) - - def sort_by_channels(list_of_signals): # make sure 1D signals come last! This is necessary for model builder. return sorted(list_of_signals, key=lambda x: x.num_channels) diff --git a/plasma/models/builder.py b/plasma/models/builder.py index 9e643d37..535a0f00 100644 --- a/plasma/models/builder.py +++ b/plasma/models/builder.py @@ -21,7 +21,8 @@ import sys import numpy as np from copy import deepcopy -from plasma.utils.downloading import makedirs_process_safe, general_object_hash +from plasma.utils.downloading import makedirs_process_safe +from plasma.utils.hashing import general_object_hash class LossHistory(Callback): diff --git a/plasma/primitives/data.py b/plasma/primitives/data.py index 4710ff77..63d30abc 100644 --- a/plasma/primitives/data.py +++ b/plasma/primitives/data.py @@ -8,6 +8,7 @@ from plasma.utils.processing import get_individual_shot_file from plasma.utils.downloading import get_missing_value_array +from plasma.utils.hashing import myhash # class SignalCollection: # """GA Data Obj""" @@ -65,10 +66,8 @@ def is_saved(self, prepath, shot): def load_data_from_txt_safe(self, prepath, shot, dtype='float32'): file_path = self.get_file_path(prepath, shot.machine, shot.number) if not self.is_saved(prepath, shot): - print( - 'Signal {}, shot {} was never downloaded'.format( - self.description, - shot.number)) + print('Signal {}, shot {} was never downloaded'.format( + self.description, shot.number)) return None, False if os.path.getsize(file_path) == 0: @@ -79,15 +78,13 @@ def load_data_from_txt_safe(self, prepath, shot, dtype='float32'): try: data = np.loadtxt(file_path, dtype=dtype) if np.all(data == get_missing_value_array()): - print( - 'Signal {}, shot {} contains no data'.format( - self.description, shot.number)) + print('Signal {}, shot {} contains no data'.format( + self.description, shot.number)) return None, False except Exception as e: print(e) - print( - 'Couldnt load signal {} shot {}. Removing.'.format( - file_path, shot.number)) + print('Couldnt load signal {} shot {}. Removing.'.format( + file_path, shot.number)) os.remove(file_path) return None, False @@ -210,9 +207,7 @@ def __lt__(self, other): other.description_plus_paths()) def __hash__(self): - import hashlib - return int(hashlib.md5( - self.description_plus_paths().encode('utf-8')).hexdigest(), 16) + return myhash(self.description_plus_paths()) def __str__(self): return self.description diff --git a/plasma/utils/downloading.py b/plasma/utils/downloading.py index f8db18da..ae940c66 100644 --- a/plasma/utils/downloading.py +++ b/plasma/utils/downloading.py @@ -6,9 +6,6 @@ import sys import time import numpy as np -import dill -import hashlib -from copy import deepcopy # import gadata # from plasma.primitives.shots import ShotList @@ -32,33 +29,6 @@ pass -def general_object_hash(o): - """ - Makes a hash from a dictionary, list, tuple or set to any level, that - contains only other hashable types (including any lists, tuples, sets, and - dictionaries). Relies on dill for serialization - """ - - if isinstance(o, (set, tuple, list)): - return tuple([general_object_hash(e) for e in o]) - - elif not isinstance(o, dict): - return myhash(o) - - new_o = deepcopy(o) - # recursively call this function when given a dictionary - for k, v in new_o.items(): - # replace the dict entry value with its hash - new_o[k] = general_object_hash(v) - - return myhash(tuple(sorted(frozenset(new_o.items())))) - - -def myhash(x): - return int(hashlib.md5((dill.dumps(x).decode('unicode_escape')).encode( - 'utf-8')).hexdigest(), 16) - - def get_missing_value_array(): return np.array([-1.0]) diff --git a/plasma/utils/hashing.py b/plasma/utils/hashing.py new file mode 100644 index 00000000..85e7d91e --- /dev/null +++ b/plasma/utils/hashing.py @@ -0,0 +1,99 @@ +from __future__ import print_function +import dill +import hashlib +import copy + + +def general_object_hash(o): + """ + Serialize and hash a dictionary, list, tuple, or set (nested to any level), + containing only other hashable types (including any lists, tuples, sets, & + dictionaries). Relies on myhash(), which relies on dill for serialization + + Reference: https://stackoverflow.com/questions/5884066/hashing-a-dictionary + + Requirements: + - Supports nested dictionaries (possibly with str keys only?) + - Well-defined for a given object beyond its lifetime + - Stable for multiple invocations of the Python kernel and across different + machines + """ + + # TODO(KGF): currently, likely no unordered set/frozenset dict vals in conf + # dictionary passed to this fn. There is no sorting in this 1st branch, so + # resulting tuple of hashes would have arbitrary ordering... + if isinstance(o, (set, frozenset, tuple, list)): + # for (possibly mutable) ordered sequence types (list, tuple) and + # unordered set types (set, frozenset): independently hash each element + # in the container, and convert to immutable tuple + return tuple([general_object_hash(e) for e in o]) + + elif not isinstance(o, dict): + # sequence and set types handled above, mapping types (dict) handled below. + # Other objs can be directly serialized and hashed, including: + # - Text sequence (str) + # - Binary sequence (bytes) + # - Numeric types + return myhash_obj(o) + + new_o = copy.deepcopy(o) + # recursively call this function when given a dictionary + for k, v in new_o.items(): + # in the deep copy, replace the value of dict entry with its hash (int) + new_o[k] = general_object_hash(v) + + # TODO(KGF): consider alternative suggested in above post. Instead of + # sorted + frozenset + dill.dumps(), json.dumps() can do 1st + 3rd steps + # Keys must all be strings in this method. Add explicit check of this cond. + + # import json + # json.dumps(new_o, sort_keys=True, ensure_ascii=True) # or False? + + # With all values of the (possibly nested) dict obj replaced by integer + # hashes or tuples of hashes, sort a list of unique (key, hash) items, and + # convert to immutable tuple obj before passing to serialize+hash method + return myhash_obj(tuple(sorted(frozenset(new_o.items())))) + # TODO(KGF): sorted() is stable, so it won't swap relative order of + # elements that compare equal. However, if the initial aribtrary ordering + # changes between kernel invocations, then two "equal" items will remain + # reordered after sorted(), thus changing the final hash. Probably fine if + # the lowest level of hashed object in nested struct avoids hash collisions + + +def myhash_obj(x): + ''' + Serialize a generic Python object using dill, decode the bytes obj, + then pass the Unicode string to the particular hash function. + ''' + return myhash(dill.dumps(x).decode('unicode_escape')) + + +def myhash_signals(signals): + ''' + Given a List of Signal class instances, sort by their str representations + (descriptions), concatenate their hexadecimal hashes (converted to + base-10), and hash the resulting str + ''' + return myhash(''.join(tuple(map(lambda x: "{}".format(x.__hash__()), + sorted(signals))))) + + +def myhash(x): + ''' + Hash a str with MD5 and return as a decimal (integer) + + hashlib is used instead of Python built-in method hash() + in order to guarantee a stable/well-defined hash algorithm + + Since Python 3.3, PYTHONHASHSEED=random is the default setting, which + randomizes the seed used for salting the hash() function to prevent DoS + attacks that exploit hash collisions. + + See http://ocert.org/advisories/ocert-2011-003.html. + + Python 3.4 further improves the default hash algorithm (PEP 456). + ''' + # re-encode the string into bytes object with UTF-8, create hash class obj + # using MD5 algorithm, then return hex digits as str type, which finally + # int(..., 16) ---> convert hexadecimal hash to base-10 integer + return int(hashlib.md5(x.encode('utf-8')).hexdigest(), 16) From b38eb1107ea017b0146a5d82fa702df7a259a1b2 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 29 Oct 2019 11:54:00 -0500 Subject: [PATCH 141/272] Shorten line --- plasma/utils/hashing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plasma/utils/hashing.py b/plasma/utils/hashing.py index 85e7d91e..b5c5a1d4 100644 --- a/plasma/utils/hashing.py +++ b/plasma/utils/hashing.py @@ -29,7 +29,7 @@ def general_object_hash(o): return tuple([general_object_hash(e) for e in o]) elif not isinstance(o, dict): - # sequence and set types handled above, mapping types (dict) handled below. + # sequence and set types handled above, mapping types (dict) below. # Other objs can be directly serialized and hashed, including: # - Text sequence (str) # - Binary sequence (bytes) From 8137d79212515d812f95823ee27e958cb8c5a2cf Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 29 Oct 2019 16:35:35 -0400 Subject: [PATCH 142/272] Try ignoring NumPy>=1.17.0 incompatibility with TensorFlow<2.0.0 As a test, only suppressing single import of Keras (and hence TF backend) that occurs when importing conf directly. Reference: https://github.com/tensorflow/tensorflow/issues/30427 --- plasma/models/builder.py | 5 ++--- plasma/models/mpi_runner.py | 18 +++++++++--------- plasma/models/targets.py | 10 ++++++++-- plasma/utils/state_reset.py | 4 +--- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/plasma/models/builder.py b/plasma/models/builder.py index 535a0f00..5e2c9eb0 100644 --- a/plasma/models/builder.py +++ b/plasma/models/builder.py @@ -72,9 +72,8 @@ def get_0D_1D_indices(self): num_0D += 1 is_1D_region = False curr_idx += num_channels - return np.array(indices_0d).astype( - np.int32), np.array(indices_1d).astype( - np.int32), num_0D, num_1D + return (np.array(indices_0d).astype(np.int32), + np.array(indices_1d).astype(np.int32), num_0D, num_1D) def build_model(self, predict, custom_batch_size=None): conf = self.conf diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 29eb1573..0be18ebf 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -5,12 +5,10 @@ from plasma.utils.evaluation import get_loss_from_list from plasma.models import builder from plasma.models.loader import ProcessGenerator -from plasma.utils.state_reset import reset_states # , get_states +from plasma.utils.state_reset import reset_states from plasma.conf import conf from pprint import pprint from mpi4py import MPI -# import mpi4py -# import getpass ''' ######################################################### This file trains a deep learning model to predict @@ -56,7 +54,7 @@ if NUM_GPUS > 1: os.environ['CUDA_VISIBLE_DEVICES'] = '{}'.format(MY_GPU) # ,mode=NanGuardMode' - os.environ['KERAS_BACKEND'] = 'tensorflow' + os.environ['KERAS_BACKEND'] = 'tensorflow' # default setting import tensorflow as tf from keras.backend.tensorflow_backend import set_session gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.95, @@ -71,7 +69,7 @@ 'device=gpu{},floatX=float32,base_compiledir={}'.format( MY_GPU, base_compile_dir)) # ,mode=NanGuardMode' # import theano -# import keras + # import keras for i in range(num_workers): comm.Barrier() if i == task_index: @@ -862,13 +860,15 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, if ('monitor_test' in conf['callbacks'].keys() and conf['callbacks']['monitor_test']): times = conf['callbacks']['monitor_times'] - roc_areas, losses = mpi_make_predictions_and_evaluate_multiple_times(conf, shot_list_validate, loader, times) # noqa - for roc, t in zip(roc_areas, times): + areas, _ = mpi_make_predictions_and_evaluate_multiple_times( + conf, shot_list_validate, loader, times) + for roc, t in zip(areas, times): print_unique('epoch {}, val_roc_{} = {}'.format( int(round(e)), t, roc)) if shot_list_test is not None: - roc_areas, losses = mpi_make_predictions_and_evaluate_multiple_times(conf, shot_list_test, loader, times) # noqa - for roc, t in zip(roc_areas, times): + areas, _ = mpi_make_predictions_and_evaluate_multiple_times( + conf, shot_list_test, loader, times) + for roc, t in zip(areas, times): print_unique('epoch {}, test_roc_{} = {}'.format( int(round(e)), t, roc)) diff --git a/plasma/models/targets.py b/plasma/models/targets.py index 558d644f..1960ea23 100644 --- a/plasma/models/targets.py +++ b/plasma/models/targets.py @@ -1,12 +1,18 @@ import numpy as np import abc -from keras.losses import hinge # squared_hinge, mean_absolute_percentage_error from plasma.utils.evaluation import ( mse_np, binary_crossentropy_np, hinge_np, # mae_np, squared_hinge_np, ) -import keras.backend as K + +import warnings +# TODO(KGF): temporarily suppress numpy>=1.17.0 warning with TF<2.0.0 +# ~6x tensorflow/python/framework/dtypes.py:529: FutureWarning ... +warnings.filterwarnings('ignore', category=FutureWarning) +import keras.backend as K # noqa +from keras.losses import hinge # noqa +warnings.resetwarnings() # Requirement: larger value must mean disruption more likely. diff --git a/plasma/utils/state_reset.py b/plasma/utils/state_reset.py index 172023bf..2d4591a3 100644 --- a/plasma/utils/state_reset.py +++ b/plasma/utils/state_reset.py @@ -1,5 +1,4 @@ from __future__ import print_function -import keras.backend as K def get_states(model): @@ -8,10 +7,9 @@ def get_states(model): if hasattr(layer, "states"): layer_states = [] for state in layer.states: - # print(K.get_value(state)[0][0:3]) + import keras.backend as K layer_states.append(K.get_value(state)) all_states.append(layer_states) - # print(all_states) return all_states From a171f7bc963df2c31f422c39214552a8fdc9b7c4 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 30 Oct 2019 16:49:06 -0400 Subject: [PATCH 143/272] Do not call warnings.resetwarnings(); use context manager Narrowly-tailor regex for excluding particular FutureWarning message encountered in the TensorFlow module import Need default filtered warnings and additional filters added by other modules. --- plasma/models/targets.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/plasma/models/targets.py b/plasma/models/targets.py index 1960ea23..384bb35e 100644 --- a/plasma/models/targets.py +++ b/plasma/models/targets.py @@ -5,14 +5,16 @@ mse_np, binary_crossentropy_np, hinge_np, # mae_np, squared_hinge_np, ) - import warnings # TODO(KGF): temporarily suppress numpy>=1.17.0 warning with TF<2.0.0 # ~6x tensorflow/python/framework/dtypes.py:529: FutureWarning ... -warnings.filterwarnings('ignore', category=FutureWarning) -import keras.backend as K # noqa -from keras.losses import hinge # noqa -warnings.resetwarnings() +with warnings.catch_warnings(record=True) as w: + warnings.filterwarnings('ignore', + category=FutureWarning, + message=r"passing \(type, 1\) or '1type' as a synonym of type is deprecated", # noqa + module="tensorflow") + import keras.backend as K # noqa + from keras.losses import hinge # noqa # Requirement: larger value must mean disruption more likely. From ac79e252bcb9be6407617ca85731d9f4636df717 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Mon, 4 Nov 2019 15:54:20 -0600 Subject: [PATCH 144/272] Remove debugging print() calls and commented out __init__.py content --- plasma/__init__.py | 15 --------------- plasma/conf_parser.py | 4 ++++ plasma/models/mpi_runner.py | 13 ++++++++----- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/plasma/__init__.py b/plasma/__init__.py index 0e21193f..e69de29b 100644 --- a/plasma/__init__.py +++ b/plasma/__init__.py @@ -1,15 +0,0 @@ -# from plasma.conf import * -# from plasma.jet_signals import * - -# from plasma.model.builder import * -# from plasma.model.runner import * -# from plasma.model.targets import * - -# from plasma.preprocessor.load import * -# from plasma.preprocessor.normalize import * -# from plasma.preprocessor.preprocess import * - -# from plasma.primitives.shots import * - -# from plasma.utils.preprocessing import * -# from plasma.utils.performance_analysis_utils import * diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index 13b9932f..06436c72 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -10,6 +10,10 @@ def parameters(input_file): """Parse yaml file of configuration parameters.""" + # TODO(KGF): the following line imports TensorFlow as a Keras backend + # by default (absent env variable KERAS_BACKEND and/or config file + # $HOME/.keras/keras.json) "from plasma.conf import conf" + # via "import keras.backend as K" in targets.py from plasma.models.targets import ( HingeTarget, MaxHingeTarget, BinaryTarget, TTDTarget, TTDInvTarget, TTDLinearTarget diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 0be18ebf..df527f00 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -519,7 +519,7 @@ def train_epoch(self): self.comm.Barrier() sys.stdout.flush() print_unique('Compilation finished in {:.2f}s'.format( - time.time()-t0_comp)) + time.time() - t0_comp)) t_start = time.time() sys.stdout.flush() @@ -627,7 +627,8 @@ def add_params(params1, params2): def get_shot_list_path(conf): - # KGF: not compatible with flexible conf.py hierarchy + # TODO(KGF): incompatible with flexible conf.py hierarchy; see setting of + # 'normalizer_path', 'global_normalizer_path' return conf['paths']['base_path'] + '/normalization/shot_lists.npz' @@ -761,6 +762,9 @@ def mpi_make_predictions_and_evaluate_multiple_times(conf, shot_list, loader, def mpi_train(conf, shot_list_train, shot_list_validate, loader, callbacks_list=None, shot_list_test=None): loader.set_inference_mode(False) + + # TODO(KGF): this is not defined in conf.yaml, but added to processed dict + # for the first time here: conf['num_workers'] = comm.Get_size() specific_builder = builder.ModelBuilder(conf) @@ -827,7 +831,7 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, cmp_fn = min while e < num_epochs-1: - print_unique("begin epoch {} 0".format(e)) + print_unique("begin epoch {}".format(e)) if task_index == 0: callbacks.on_epoch_begin(int(round(e))) mpi_model.set_lr(lr*lr_decay**e) @@ -908,9 +912,8 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, tensorboard.on_epoch_end(val_generator, val_steps, int(round(e)), epoch_logs) - print_unique("end epoch {} 0".format(e)) stop_training = comm.bcast(stop_training, root=0) - print_unique("end epoch {} 1".format(e)) + print_unique("end epoch {}".format(e)) if stop_training: print("Stopping training due to early stopping") break From b1ff7e6106b0ac4a22541d32c80c77a572c4877c Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 5 Nov 2019 14:17:36 -0500 Subject: [PATCH 145/272] Move TF warning suppression to top-level __init__.py --- plasma/__init__.py | 7 +++++++ plasma/models/targets.py | 10 ---------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/plasma/__init__.py b/plasma/__init__.py index e69de29b..963dded2 100644 --- a/plasma/__init__.py +++ b/plasma/__init__.py @@ -0,0 +1,7 @@ +import warnings +# TODO(KGF): temporarily suppress numpy>=1.17.0 warning with TF<2.0.0 +# ~6x tensorflow/python/framework/dtypes.py:529: FutureWarning ... +warnings.filterwarnings('ignore', + category=FutureWarning, + message=r"passing \(type, 1\) or '1type' as a synonym of type is deprecated", # noqa + module="tensorflow") diff --git a/plasma/models/targets.py b/plasma/models/targets.py index 384bb35e..a478125a 100644 --- a/plasma/models/targets.py +++ b/plasma/models/targets.py @@ -5,16 +5,6 @@ mse_np, binary_crossentropy_np, hinge_np, # mae_np, squared_hinge_np, ) -import warnings -# TODO(KGF): temporarily suppress numpy>=1.17.0 warning with TF<2.0.0 -# ~6x tensorflow/python/framework/dtypes.py:529: FutureWarning ... -with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('ignore', - category=FutureWarning, - message=r"passing \(type, 1\) or '1type' as a synonym of type is deprecated", # noqa - module="tensorflow") - import keras.backend as K # noqa - from keras.losses import hinge # noqa # Requirement: larger value must mean disruption more likely. From 87a275afab59794b3d8a4c9feb56536f9884c5d0 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 5 Nov 2019 13:18:12 -0600 Subject: [PATCH 146/272] Use more print_unique() in mpi_runner.py --- examples/conf.yaml | 4 ++-- plasma/models/mpi_runner.py | 34 +++++++++++++++++++--------------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/examples/conf.yaml b/examples/conf.yaml index e125a391..dfda1145 100644 --- a/examples/conf.yaml +++ b/examples/conf.yaml @@ -4,7 +4,7 @@ fs_path: '/tigress' target: 'hinge' # 'maxhinge' # 'maxhinge' # 'binary' # 'hinge' -num_gpus: 4 +num_gpus: 4 # per node paths: signal_prepath: '/signal_data/' # /signal_data/jet/ @@ -105,7 +105,7 @@ model: stateful: True return_sequences: True dropout_prob: 0.1 - # only relevant if we want to do mpi training. The number of steps with a single replica + # only relevant if we want to do MPI training. The number of steps with a single replica warmup_steps: 0 ignore_timesteps: 100 # how many initial timesteps to ignore during evaluation (to let the internal state settle) backend: 'tensorflow' diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index df527f00..c9593468 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -7,7 +7,6 @@ from plasma.models.loader import ProcessGenerator from plasma.utils.state_reset import reset_states from plasma.conf import conf -from pprint import pprint from mpi4py import MPI ''' ######################################################### @@ -25,6 +24,7 @@ ######################################################### ''' + import os import sys import time @@ -37,6 +37,13 @@ import socket sys.setrecursionlimit(10000) + +def pprint_unique(obj): + from pprint import pprint + if task_index == 0: + pprint(obj) + + # import keras sequentially because it otherwise reads from ~/.keras/keras.json # with too many threads: # from mpi_launch_tensorflow import get_mpi_task_index @@ -80,8 +87,7 @@ import keras.callbacks as cbks -if task_index == 0: - pprint(conf) +pprint_unique(conf) class MPIOptimizer(object): @@ -492,7 +498,7 @@ def train_epoch(self): (batch_xs, batch_ys, batches_to_reset, num_so_far_curr, num_total, is_warmup_period) = next(batch_iterator_func) except StopIteration: - print("Resetting batch iterator.") + print_unique("Resetting batch iterator.") self.num_so_far_accum = self.num_so_far_indiv self.set_batch_iterator_func() batch_iterator_func = self.batch_iterator_func @@ -535,13 +541,12 @@ def train_epoch(self): t2 = time.time() write_str_0 = self.calculate_speed(t0, t1, t2, num_replicas) curr_loss = self.mpi_average_scalars(1.0*loss, num_replicas) - # if self.task_index == 0: - # print(self.model.get_weights()[0][0][:4]) + # print_unique(self.model.get_weights()[0][0][:4]) loss_averager.add_val(curr_loss) ave_loss = loss_averager.get_val() - eta = self.estimate_remaining_time( - t0 - t_start, - self.num_so_far - self.epoch*num_total, num_total) + eta = self.estimate_remaining_time(t0 - t_start, + self.num_so_far - self.epoch*num_total, + num_total) write_str = ( '\r[{}] step: {} [ETA: {:.2f}s] [{:.2f}/{}], '.format( self.task_index, step, eta, @@ -793,13 +798,12 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, print("Optimizer not implemented yet") exit(1) - print('{} epochs left to go'.format(num_epochs - 1 - e)) + print_unique('{} epochs left to go'.format(num_epochs - 1 - e)) - batch_generator = partial( - loader.training_batch_generator_partial_reset, - shot_list=shot_list_train) + batch_generator = partial(loader.training_batch_generator_partial_reset, + shot_list=shot_list_train) - print("warmup {}".format(warmup_steps)) + print_unique("warmup steps = {}".format(warmup_steps)) mpi_model = MPIModel(train_model, optimizer, comm, batch_generator, batch_size, lr=lr, warmup_steps=warmup_steps, num_batches_minimum=num_batches_minimum, conf=conf) @@ -915,7 +919,7 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, stop_training = comm.bcast(stop_training, root=0) print_unique("end epoch {}".format(e)) if stop_training: - print("Stopping training due to early stopping") + print_unique("Stopping training due to early stopping") break if task_index == 0: From b90d579932a6e5fcf97350af1b74bafb096a9ef5 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 5 Nov 2019 13:44:46 -0600 Subject: [PATCH 147/272] Replace accidentally-removed imports from targets.py --- plasma/models/targets.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plasma/models/targets.py b/plasma/models/targets.py index a478125a..7f80187d 100644 --- a/plasma/models/targets.py +++ b/plasma/models/targets.py @@ -5,7 +5,8 @@ mse_np, binary_crossentropy_np, hinge_np, # mae_np, squared_hinge_np, ) - +import keras.backend as K +from keras.losses import hinge # Requirement: larger value must mean disruption more likely. From 9962727cad897b70c78e6c11c6ace6f214be7f98 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 5 Nov 2019 14:19:26 -0600 Subject: [PATCH 148/272] Attempt dedpulication of code in mpi_learn.py from mpi_runner.py Also, call mpi_runner.print_unique() fn instead of print() in mpi_learn.py --- examples/mpi_learn.py | 58 ++++++++++++++++++++----------------- plasma/models/mpi_runner.py | 28 +++++++++--------- 2 files changed, 45 insertions(+), 41 deletions(-) diff --git a/examples/mpi_learn.py b/examples/mpi_learn.py index 8c4e7828..753edca1 100644 --- a/examples/mpi_learn.py +++ b/examples/mpi_learn.py @@ -1,11 +1,12 @@ from plasma.models.mpi_runner import ( - mpi_train, mpi_make_predictions_and_evaluate + mpi_train, mpi_make_predictions_and_evaluate, + comm, task_index, print_unique ) -from mpi4py import MPI +# from mpi4py import MPI from plasma.preprocessor.preprocess import guarantee_preprocessed from plasma.models.loader import Loader from plasma.conf import conf -from pprint import pprint +# from pprint import pprint ''' ######################################################### This file trains a deep learning model to predict @@ -37,9 +38,8 @@ if conf['model']['shallow']: - print( - "Shallow learning using MPI is not supported yet. ", - "Set conf['model']['shallow'] to False.") + print("Shallow learning using MPI is not supported yet. ", + "Set conf['model']['shallow'] to False.") exit(1) if conf['data']['normalizer'] == 'minmax': from plasma.preprocessor.normalize import MinMaxNormalizer as Normalizer @@ -57,23 +57,29 @@ print('unkown normalizer. exiting') exit(1) -comm = MPI.COMM_WORLD -task_index = comm.Get_rank() -num_workers = comm.Get_size() -NUM_GPUS = conf['num_gpus'] -MY_GPU = task_index % NUM_GPUS +# TODO(KGF): this part of the code is duplicated in mpi_runner.py +# comm = MPI.COMM_WORLD +# task_index = comm.Get_rank() +# num_workers = comm.Get_size() +# NUM_GPUS = conf['num_gpus'] +# MY_GPU = task_index % NUM_GPUS +# backend = conf['model']['backend'] + +# if task_index == 0: +# pprint(conf) + +# TODO(KGF): confirm that this second PRNG seed setting is not needed +# (before normalization; done again before MPI training) +# np.random.seed(task_index) +# random.seed(task_index) -np.random.seed(task_index) -random.seed(task_index) -if task_index == 0: - pprint(conf) only_predict = len(sys.argv) > 1 custom_path = None if only_predict: custom_path = sys.argv[1] - print("predicting using path {}".format(custom_path)) + print_unique("predicting using path {}".format(custom_path)) ##################################################### @@ -89,11 +95,11 @@ shot_list_test) = guarantee_preprocessed(conf) -print("normalization", end='') +print_unique("normalization", end='') normalizer = Normalizer(conf) normalizer.train() loader = Loader(conf, normalizer) -print("...done") +print_unique("...done") # ensure training has a separate random seed for every worker np.random.seed(task_index) @@ -104,7 +110,7 @@ # load last model for testing loader.set_inference_mode(True) -print('saving results') +print_unique('saving results') y_prime = [] y_gold = [] disruptive = [] @@ -117,12 +123,11 @@ loss_test) = mpi_make_predictions_and_evaluate(conf, shot_list_test, loader, custom_path) -if task_index == 0: - print('=========Summary========') - print('Train Loss: {:.3e}'.format(loss_train)) - print('Train ROC: {:.4f}'.format(roc_train)) - print('Test Loss: {:.3e}'.format(loss_test)) - print('Test ROC: {:.4f}'.format(roc_test)) +print_unique('=========Summary========') +print_unique('Train Loss: {:.3e}'.format(loss_train)) +print_unique('Train ROC: {:.4f}'.format(roc_train)) +print_unique('Test Loss: {:.3e}'.format(loss_test)) +print_unique('Test ROC: {:.4f}'.format(roc_test)) if task_index == 0: @@ -156,5 +161,4 @@ # requirement for "allow_pickle=True" to savez() calls sys.stdout.flush() -if task_index == 0: - print('finished.') +print_unique('finished.') diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index c9593468..1a794f7f 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -38,12 +38,6 @@ sys.setrecursionlimit(10000) -def pprint_unique(obj): - from pprint import pprint - if task_index == 0: - pprint(obj) - - # import keras sequentially because it otherwise reads from ~/.keras/keras.json # with too many threads: # from mpi_launch_tensorflow import get_mpi_task_index @@ -51,12 +45,18 @@ def pprint_unique(obj): task_index = comm.Get_rank() num_workers = comm.Get_size() - NUM_GPUS = conf['num_gpus'] MY_GPU = task_index % NUM_GPUS - backend = conf['model']['backend'] + +def pprint_unique(obj): + from pprint import pprint + if task_index == 0: + pprint(obj) + + +# initialization code for mpi_runner.py module: if backend == 'tf' or backend == 'tensorflow': if NUM_GPUS > 1: os.environ['CUDA_VISIBLE_DEVICES'] = '{}'.format(MY_GPU) @@ -544,13 +544,13 @@ def train_epoch(self): # print_unique(self.model.get_weights()[0][0][:4]) loss_averager.add_val(curr_loss) ave_loss = loss_averager.get_val() - eta = self.estimate_remaining_time(t0 - t_start, - self.num_so_far - self.epoch*num_total, - num_total) + eta = self.estimate_remaining_time( + t0 - t_start, self.num_so_far - self.epoch*num_total, + num_total) write_str = ( '\r[{}] step: {} [ETA: {:.2f}s] [{:.2f}/{}], '.format( - self.task_index, step, eta, - 1.0*self.num_so_far, num_total) + self.task_index, step, eta, 1.0*self.num_so_far, + num_total) + 'loss: {:.5f} [{:.5f}] | '.format(ave_loss, curr_loss) + 'walltime: {:.4f} | '.format( time.time() - self.start_time)) @@ -834,7 +834,7 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, best_so_far = np.inf cmp_fn = min - while e < num_epochs-1: + while e < (num_epochs - 1): print_unique("begin epoch {}".format(e)) if task_index == 0: callbacks.on_epoch_begin(int(round(e))) From 6583004621181f0be0b48e59d9dc7248549cb3a1 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 5 Nov 2019 14:22:26 -0600 Subject: [PATCH 149/272] Create new class method for printing final summary of loaded/saved shots --- plasma/preprocessor/normalize.py | 36 ++++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/plasma/preprocessor/normalize.py b/plasma/preprocessor/normalize.py index 97d819e8..a19ccad1 100644 --- a/plasma/preprocessor/normalize.py +++ b/plasma/preprocessor/normalize.py @@ -73,6 +73,10 @@ def save_stats(self): def load_stats(self): pass + @abc.abstractmethod + def print_summary(self, action='loaded'): + pass + def set_inference_mode(self, val): self.inference_mode = val @@ -256,8 +260,9 @@ def incorporate_stats(self, stats): self.stds[machine] = np.concatenate( (self.stds[machine], stds), axis=0) self.num_processed[machine] = self.num_processed[machine] + 1 - self.num_disruptive[machine] = self.num_disruptive[machine] + \ - (1 if stats.is_disruptive else 0) + self.num_disruptive[machine] = ( + self.num_disruptive[machine] + + (1 if stats.is_disruptive else 0)) def apply(self, shot): apply_positivity(shot) @@ -286,17 +291,10 @@ def save_stats(self): # num_processed = dat['num_processed'] # num_disruptive = dat['num_disruptive'] self.ensure_save_directory() - np.savez( - self.path, - means=self.means, - stds=self.stds, - num_processed=self.num_processed, - num_disruptive=self.num_disruptive, - machines=self.machines) - print( - 'saved normalization data from {} shots ( {} disruptive )'.format( - self.num_processed, - self.num_disruptive)) + np.savez(self.path, means=self.means, stds=self.stds, + num_processed=self.num_processed, + num_disruptive=self.num_disruptive, machines=self.machines) + self.print_summary(action='saved') def load_stats(self): assert self.previously_saved_stats()[0], "stats not saved before" @@ -308,9 +306,11 @@ def load_stats(self): self.machines = dat['machines'][()] for machine in self.means: print('Machine {}:'.format(machine)) - print('loaded normalization data from ', - '{} shots ( {} disruptive )'.format(self.num_processed, - self.num_disruptive)) + self.print_summary() + + def print_summary(self, action='loaded'): + print('{} normalization data from {} shots ( {} disruptive )'.format( + action, self.num_processed, self.num_disruptive)) class VarNormalizer(MeanVarNormalizer): @@ -416,8 +416,8 @@ def incorporate_stats(self, stats): self.maximums[m] = (self.num_processed[m]*self.maximums + maximums)/(self.num_processed[m] + 1.0) self.num_processed[m] = self.num_processed[m] + 1 - self.num_disruptive[m] = self.num_disruptive[m] + \ - (1 if stats.is_disruptive else 0) + self.num_disruptive[m] = (self.num_disruptive[m] + + (1 if stats.is_disruptive else 0)) def apply(self, shot): apply_positivity(shot) From 7ced7587bf6803abd310a562a580ab1e9855f8f5 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 5 Nov 2019 15:13:04 -0600 Subject: [PATCH 150/272] Use two different fns for controlling MPI printint to stdout --- examples/mpi_learn.py | 1 - plasma/models/mpi_runner.py | 57 ++++++++++++++++++++++++++----------- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/examples/mpi_learn.py b/examples/mpi_learn.py index 753edca1..7f590979 100644 --- a/examples/mpi_learn.py +++ b/examples/mpi_learn.py @@ -129,7 +129,6 @@ print_unique('Test Loss: {:.3e}'.format(loss_test)) print_unique('Test ROC: {:.4f}'.format(roc_test)) - if task_index == 0: disruptive_train = np.array(disruptive_train) disruptive_test = np.array(disruptive_test) diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 1a794f7f..b6038c76 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -554,16 +554,16 @@ def train_epoch(self): + 'loss: {:.5f} [{:.5f}] | '.format(ave_loss, curr_loss) + 'walltime: {:.4f} | '.format( time.time() - self.start_time)) - print_unique(write_str + write_str_0) + write_unique(write_str + write_str_0) step += 1 else: - print_unique('\r[{}] warmup phase, num so far: {}'.format( + write_unique('\r[{}] warmup phase, num so far: {}'.format( self.task_index, self.num_so_far)) effective_epochs = 1.0*self.num_so_far/num_total epoch_previous = self.epoch self.epoch = effective_epochs - print_unique('\nEpoch {:.2f} finished ({:.2f} epochs passed)'.format( + write_unique('\nEpoch {:.2f} finished ({:.2f} epochs passed)'.format( 1.0 * self.epoch, self.epoch - epoch_previous) + ' in {:.2f} seconds.\n'.format(t2 - t_start)) return (step, ave_loss, curr_loss, self.num_so_far, effective_epochs) @@ -576,7 +576,7 @@ def estimate_remaining_time(self, time_so_far, work_so_far, work_total): def get_effective_lr(self, num_replicas): effective_lr = self.lr * num_replicas if effective_lr > self.max_lr: - print_unique('Warning: effective learning rate set to {}, '.format( + write_unique('Warning: effective learning rate set to {}, '.format( effective_lr) + 'larger than maximum {}. Clipping.'.format( self.max_lr)) effective_lr = self.max_lr @@ -604,18 +604,41 @@ def calculate_speed(self, t0, t_after_deltas, t_after_update, num_replicas, effective_batch_size, self.batch_size, num_replicas, self.get_effective_lr(num_replicas), self.lr, num_replicas) if verbose: - print_unique(print_str) + write_unique(print_str) return print_str -def print_unique(print_str): +def print_unique(print_output, end='\n', flush=False): + """ + Only master MPI rank 0 calls print(). + + Trivial wrapper function to print() + """ + if task_index == 0: + print(print_output, end=end, flush=flush) + + +def write_unique(write_str): + """ + Only master MPI rank 0 writes to and flushes stdout. + + A specialized case of print_unique(). Unlike print(), sys.stdout.write(): + - Must pass a string; will not cast argument + - end='\n' kwarg of print() is not available + (often the argument here is prepended with \r=carriage return in order to + simulate a terminal output that overwrites itself) + """ + # TODO(KGF): \r carriage returns appear as ^M in Unix-encoded .out files + # from non-interactive Slurm batch jobs. Convert these to true Unix + # line feeds / newlines (^J, \n) when we can detect such a stdout if task_index == 0: - sys.stdout.write(print_str) + sys.stdout.write(write_str) sys.stdout.flush() -def print_all(print_str): - sys.stdout.write('[{}] '.format(task_index) + print_str) +def write_all(write_str): + '''All MPI ranks write to stdout, appending [rank]''' + sys.stdout.write('[{}] '.format(task_index) + write_str) sys.stdout.flush() @@ -798,12 +821,12 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, print("Optimizer not implemented yet") exit(1) - print_unique('{} epochs left to go'.format(num_epochs - 1 - e)) + write_unique('{} epochs left to go'.format(num_epochs - 1 - e)) batch_generator = partial(loader.training_batch_generator_partial_reset, shot_list=shot_list_train) - print_unique("warmup steps = {}".format(warmup_steps)) + write_unique("warmup steps = {}".format(warmup_steps)) mpi_model = MPIModel(train_model, optimizer, comm, batch_generator, batch_size, lr=lr, warmup_steps=warmup_steps, num_batches_minimum=num_batches_minimum, conf=conf) @@ -835,11 +858,11 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, cmp_fn = min while e < (num_epochs - 1): - print_unique("begin epoch {}".format(e)) + write_unique("begin epoch {}".format(e)) if task_index == 0: callbacks.on_epoch_begin(int(round(e))) mpi_model.set_lr(lr*lr_decay**e) - print_unique('\nEpoch {}/{}'.format(e, num_epochs)) + write_unique('\nEpoch {}/{}'.format(e, num_epochs)) (step, ave_loss, curr_loss, num_so_far, effective_epochs) = mpi_model.train_epoch() @@ -871,13 +894,13 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, areas, _ = mpi_make_predictions_and_evaluate_multiple_times( conf, shot_list_validate, loader, times) for roc, t in zip(areas, times): - print_unique('epoch {}, val_roc_{} = {}'.format( + write_unique('epoch {}, val_roc_{} = {}'.format( int(round(e)), t, roc)) if shot_list_test is not None: areas, _ = mpi_make_predictions_and_evaluate_multiple_times( conf, shot_list_test, loader, times) for roc, t in zip(areas, times): - print_unique('epoch {}, test_roc_{} = {}'.format( + write_unique('epoch {}, test_roc_{} = {}'.format( int(round(e)), t, roc)) epoch_logs['val_roc'] = roc_area @@ -917,9 +940,9 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, int(round(e)), epoch_logs) stop_training = comm.bcast(stop_training, root=0) - print_unique("end epoch {}".format(e)) + write_unique("end epoch {}".format(e)) if stop_training: - print_unique("Stopping training due to early stopping") + write_unique("Stopping training due to early stopping") break if task_index == 0: From 052b345c084642e52f7b2f570c387e9c8212bf36 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 5 Nov 2019 15:21:38 -0600 Subject: [PATCH 151/272] Re-add newline to end of "normalization" stdout May have made sense to suppress it and later add "...done" to indicate progress, but only if nothing is written to stdout in the meantime. --- examples/mpi_learn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/mpi_learn.py b/examples/mpi_learn.py index 7f590979..d6e45e40 100644 --- a/examples/mpi_learn.py +++ b/examples/mpi_learn.py @@ -95,7 +95,7 @@ shot_list_test) = guarantee_preprocessed(conf) -print_unique("normalization", end='') +print_unique("begin normalization...") # , end='') normalizer = Normalizer(conf) normalizer.train() loader = Loader(conf, normalizer) From 55639ce76331095c81c37bf6b5a3dd9431d5b9ee Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 5 Nov 2019 15:30:39 -0600 Subject: [PATCH 152/272] Prepare to call print_unique() or analog in normalize.py --- examples/mpi_learn.py | 18 +---------------- plasma/preprocessor/normalize.py | 34 ++++++++++---------------------- 2 files changed, 11 insertions(+), 41 deletions(-) diff --git a/examples/mpi_learn.py b/examples/mpi_learn.py index d6e45e40..de819b69 100644 --- a/examples/mpi_learn.py +++ b/examples/mpi_learn.py @@ -2,11 +2,9 @@ mpi_train, mpi_make_predictions_and_evaluate, comm, task_index, print_unique ) -# from mpi4py import MPI from plasma.preprocessor.preprocess import guarantee_preprocessed from plasma.models.loader import Loader from plasma.conf import conf -# from pprint import pprint ''' ######################################################### This file trains a deep learning model to predict @@ -57,24 +55,11 @@ print('unkown normalizer. exiting') exit(1) -# TODO(KGF): this part of the code is duplicated in mpi_runner.py -# comm = MPI.COMM_WORLD -# task_index = comm.Get_rank() -# num_workers = comm.Get_size() - -# NUM_GPUS = conf['num_gpus'] -# MY_GPU = task_index % NUM_GPUS -# backend = conf['model']['backend'] - -# if task_index == 0: -# pprint(conf) - # TODO(KGF): confirm that this second PRNG seed setting is not needed # (before normalization; done again before MPI training) # np.random.seed(task_index) # random.seed(task_index) - only_predict = len(sys.argv) > 1 custom_path = None if only_predict: @@ -94,8 +79,7 @@ (shot_list_train, shot_list_validate, shot_list_test) = guarantee_preprocessed(conf) - -print_unique("begin normalization...") # , end='') +print_unique("begin normalization...") normalizer = Normalizer(conf) normalizer.train() loader = Loader(conf, normalizer) diff --git a/plasma/preprocessor/normalize.py b/plasma/preprocessor/normalize.py index a19ccad1..1008bde7 100644 --- a/plasma/preprocessor/normalize.py +++ b/plasma/preprocessor/normalize.py @@ -73,9 +73,9 @@ def save_stats(self): def load_stats(self): pass - @abc.abstractmethod def print_summary(self, action='loaded'): - pass + print('{} normalization data from {} shots ( {} disruptive )'.format( + action, self.num_processed, self.num_disruptive)) def set_inference_mode(self, val): self.inference_mode = val @@ -132,8 +132,7 @@ def train_on_files(self, shot_files, use_shots, all_machines): start_time = time.time() for (i, stats) in enumerate(pool.imap_unordered( - self.train_on_single_shot, - shot_list_picked)): + self.train_on_single_shot, shot_list_picked)): # for (i,stats) in # enumerate(map(self.train_on_single_shot,shot_list_picked)): if stats.machine in machines_to_compute: @@ -308,10 +307,6 @@ def load_stats(self): print('Machine {}:'.format(machine)) self.print_summary() - def print_summary(self, action='loaded'): - print('{} normalization data from {} shots ( {} disruptive )'.format( - action, self.num_processed, self.num_disruptive)) - class VarNormalizer(MeanVarNormalizer): def apply(self, shot): @@ -341,7 +336,6 @@ def __str__(self): class AveragingVarNormalizer(VarNormalizer): - def apply(self, shot): apply_positivity(shot) super(AveragingVarNormalizer, self).apply(shot) @@ -444,17 +438,10 @@ def save_stats(self): # num_processed = dat['num_processed'] # num_disruptive = dat['num_disruptive'] self.ensure_save_directory() - np.savez( - self.path, - minimums=self.minimums, - maximums=self.maximums, - num_processed=self.num_processed, - num_disruptive=self.num_disruptive, - machines=self.machines) - print( - 'saved normalization data from {} shots ( {} disruptive )'.format( - self.num_processed, - self.num_disruptive)) + np.savez(self.path, minimums=self.minimums, maximums=self.maximums, + num_processed=self.num_processed, + num_disruptive=self.num_disruptive, machines=self.machines) + self.print_summary(action='saved') def load_stats(self): assert(self.previously_saved_stats()[0]) @@ -464,10 +451,9 @@ def load_stats(self): self.num_processed = dat['num_processed'][()] self.num_disruptive = dat['num_disruptive'][()] self.machines = dat['machines'][()] - print( - 'loaded normalization data from {} shots ( {} disruptive )'.format( - self.num_processed, - self.num_disruptive)) + for machine in self.means: + print('Machine {}:'.format(machine)) + self.print_summary() def get_individual_shot_file(prepath, shot_num, ext='.txt'): From 39709fde190c349892259777f73049aa19bb3ca8 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 5 Nov 2019 16:09:10 -0600 Subject: [PATCH 153/272] Extract all MPI variables to separate, global/shared module Uncomment setting of NumPy random number seed at top of mpi_learn.py --- examples/mpi_learn.py | 51 ++++++---- plasma/global_vars.py | 63 ++++++++++++ plasma/models/mpi_runner.py | 186 ++++++++++++++---------------------- 3 files changed, 166 insertions(+), 134 deletions(-) create mode 100644 plasma/global_vars.py diff --git a/examples/mpi_learn.py b/examples/mpi_learn.py index de819b69..935be33e 100644 --- a/examples/mpi_learn.py +++ b/examples/mpi_learn.py @@ -1,6 +1,6 @@ +import plasma.global_vars as g from plasma.models.mpi_runner import ( - mpi_train, mpi_make_predictions_and_evaluate, - comm, task_index, print_unique + mpi_train, mpi_make_predictions_and_evaluate ) from plasma.preprocessor.preprocess import guarantee_preprocessed from plasma.models.loader import Loader @@ -55,46 +55,55 @@ print('unkown normalizer. exiting') exit(1) -# TODO(KGF): confirm that this second PRNG seed setting is not needed -# (before normalization; done again before MPI training) -# np.random.seed(task_index) -# random.seed(task_index) +# set PRNG seed, unique for each worker, based on MPI task index for +# reproducible shuffling in guranteed_preprocessed() and training steps +np.random.seed(g.task_index) +random.seed(g.task_index) only_predict = len(sys.argv) > 1 custom_path = None if only_predict: custom_path = sys.argv[1] - print_unique("predicting using path {}".format(custom_path)) + g.print_unique("predicting using path {}".format(custom_path)) ##################################################### # NORMALIZATION # ##################################################### # make sure preprocessing has been run, and is saved as a file -if task_index == 0: +if g.task_index == 0: # TODO(KGF): check tuple unpack (shot_list_train, shot_list_validate, shot_list_test) = guarantee_preprocessed(conf) -comm.Barrier() +g.comm.Barrier() (shot_list_train, shot_list_validate, shot_list_test) = guarantee_preprocessed(conf) -print_unique("begin normalization...") +g.print_unique("begin normalization...") normalizer = Normalizer(conf) normalizer.train() loader = Loader(conf, normalizer) -print_unique("...done") +g.print_unique("...done") + +##################################################### +# TRAINING # +##################################################### # ensure training has a separate random seed for every worker -np.random.seed(task_index) -random.seed(task_index) +# TODO(KGF): can probably delete the next two lines. Check. +np.random.seed(g.task_index) +random.seed(g.task_index) if not only_predict: mpi_train(conf, shot_list_train, shot_list_validate, loader, shot_list_test=shot_list_test) +##################################################### +# TESTING # +##################################################### + # load last model for testing loader.set_inference_mode(True) -print_unique('saving results') +g.print_unique('saving results') y_prime = [] y_gold = [] disruptive = [] @@ -107,13 +116,13 @@ loss_test) = mpi_make_predictions_and_evaluate(conf, shot_list_test, loader, custom_path) -print_unique('=========Summary========') -print_unique('Train Loss: {:.3e}'.format(loss_train)) -print_unique('Train ROC: {:.4f}'.format(roc_train)) -print_unique('Test Loss: {:.3e}'.format(loss_test)) -print_unique('Test ROC: {:.4f}'.format(roc_test)) +g.print_unique('=========Summary========') +g.print_unique('Train Loss: {:.3e}'.format(loss_train)) +g.print_unique('Train ROC: {:.4f}'.format(roc_train)) +g.print_unique('Test Loss: {:.3e}'.format(loss_test)) +g.print_unique('Test ROC: {:.4f}'.format(roc_test)) -if task_index == 0: +if g.task_index == 0: disruptive_train = np.array(disruptive_train) disruptive_test = np.array(disruptive_test) @@ -144,4 +153,4 @@ # requirement for "allow_pickle=True" to savez() calls sys.stdout.flush() -print_unique('finished.') +g.print_unique('finished.') diff --git a/plasma/global_vars.py b/plasma/global_vars.py new file mode 100644 index 00000000..a97c8a83 --- /dev/null +++ b/plasma/global_vars.py @@ -0,0 +1,63 @@ +import sys + +# global variable defaults for non-MPI runs +comm = None +task_index = 0 +num_workers = 1 +NUM_GPUS = 0 +MY_GPU = 0 +backend = '' + + +def init_MPI(conf): + from mpi4py import MPI + global comm, task_index, num_workers + global NUM_GPUS, MY_GPU, backend + + comm = MPI.COMM_WORLD + task_index = comm.Get_rank() + num_workers = comm.Get_size() + + NUM_GPUS = conf['num_gpus'] + MY_GPU = task_index % NUM_GPUS + backend = conf['model']['backend'] + + +def pprint_unique(obj): + from pprint import pprint + if task_index == 0: + pprint(obj) + + +def print_unique(print_output, end='\n', flush=False): + """ + Only master MPI rank 0 calls print(). + + Trivial wrapper function to print() + """ + if task_index == 0: + print(print_output, end=end, flush=flush) + + +def write_unique(write_str): + """ + Only master MPI rank 0 writes to and flushes stdout. + + A specialized case of print_unique(). Unlike print(), sys.stdout.write(): + - Must pass a string; will not cast argument + - end='\n' kwarg of print() is not available + (often the argument here is prepended with \r=carriage return in order to + simulate a terminal output that overwrites itself) + """ + # TODO(KGF): \r carriage returns appear as ^M in Unix-encoded .out files + # from non-interactive Slurm batch jobs. Convert these to true Unix + # line feeds / newlines (^J, \n) when we can detect such a stdout + if task_index == 0: + sys.stdout.write(write_str) + sys.stdout.flush() + + +def write_all(write_str): + '''All MPI ranks write to stdout, appending [rank]''' + sys.stdout.write('[{}] '.format(task_index) + write_str) + sys.stdout.flush() diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index b6038c76..898e4de1 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -1,4 +1,5 @@ from __future__ import print_function +import plasma.global_vars as g from plasma.primitives.ops import mpi_sum_f16 from plasma.utils.performance import PerformanceAnalyzer from plasma.utils.processing import concatenate_sublists @@ -41,25 +42,16 @@ # import keras sequentially because it otherwise reads from ~/.keras/keras.json # with too many threads: # from mpi_launch_tensorflow import get_mpi_task_index -comm = MPI.COMM_WORLD -task_index = comm.Get_rank() -num_workers = comm.Get_size() - -NUM_GPUS = conf['num_gpus'] -MY_GPU = task_index % NUM_GPUS -backend = conf['model']['backend'] - - -def pprint_unique(obj): - from pprint import pprint - if task_index == 0: - pprint(obj) +# set global variables for entire module regarding MPI environment +# TODO(KGF): consider moving this fn/init call to mpi_learn.py and/or +# setting "mpi_initialized" global bool flag, since that is client-facing +g.init_MPI() # initialization code for mpi_runner.py module: -if backend == 'tf' or backend == 'tensorflow': - if NUM_GPUS > 1: - os.environ['CUDA_VISIBLE_DEVICES'] = '{}'.format(MY_GPU) +if g.backend == 'tf' or g.backend == 'tensorflow': + if g.NUM_GPUS > 1: + os.environ['CUDA_VISIBLE_DEVICES'] = '{}'.format(g.MY_GPU) # ,mode=NanGuardMode' os.environ['KERAS_BACKEND'] = 'tensorflow' # default setting import tensorflow as tf @@ -71,23 +63,23 @@ def pprint_unique(obj): else: os.environ['KERAS_BACKEND'] = 'theano' base_compile_dir = '{}/tmp/{}-{}'.format( - conf['paths']['output_path'], socket.gethostname(), task_index) + conf['paths']['output_path'], socket.gethostname(), g.task_index) os.environ['THEANO_FLAGS'] = ( 'device=gpu{},floatX=float32,base_compiledir={}'.format( - MY_GPU, base_compile_dir)) # ,mode=NanGuardMode' + g.MY_GPU, base_compile_dir)) # ,mode=NanGuardMode' # import theano # import keras -for i in range(num_workers): - comm.Barrier() - if i == task_index: - print('[{}] importing Keras'.format(task_index)) +for i in range(g.num_workers): + g.comm.Barrier() + if i == g.task_index: + print('[{}] importing Keras'.format(g.task_index)) from keras import backend as K # from keras.optimizers import * from keras.utils.generic_utils import Progbar import keras.callbacks as cbks -pprint_unique(conf) +g.pprint_unique(conf) class MPIOptimizer(object): @@ -108,8 +100,8 @@ def __init__(self, lr): def get_deltas(self, raw_deltas): deltas = [] - for g in raw_deltas: - deltas.append(self.lr*g) + for grad in raw_deltas: + deltas.append(self.lr*grad) self.iterations += 1 return deltas @@ -126,9 +118,9 @@ def get_deltas(self, raw_deltas): if self.iterations == 0: self.velocity_list = [np.zeros_like(g) for g in raw_deltas] - for (i, g) in enumerate(raw_deltas): + for (i, grad) in enumerate(raw_deltas): self.velocity_list[i] = ( - self.momentum * self.velocity_list[i] + self.lr * g) + self.momentum * self.velocity_list[i] + self.lr * grad) deltas.append(self.velocity_list[i]) self.iterations += 1 @@ -146,15 +138,15 @@ def __init__(self, lr): def get_deltas(self, raw_deltas): if self.iterations == 0: - self.m_list = [np.zeros_like(g) for g in raw_deltas] - self.v_list = [np.zeros_like(g) for g in raw_deltas] + self.m_list = [np.zeros_like(grad) for grad in raw_deltas] + self.v_list = [np.zeros_like(grad) for grad in raw_deltas] t = self.iterations + 1 lr_t = self.lr * np.sqrt(1-self.beta_2**t)/(1-self.beta_1**t) deltas = [] - for (i, g) in enumerate(raw_deltas): - m_t = (self.beta_1 * self.m_list[i]) + (1 - self.beta_1) * g - v_t = (self.beta_2 * self.v_list[i]) + (1 - self.beta_2) * (g**2) + for (i, grad) in enumerate(raw_deltas): + m_t = (self.beta_1 * self.m_list[i]) + (1-self.beta_1) * grad + v_t = (self.beta_2 * self.v_list[i]) + (1-self.beta_2) * (grad**2) delta_t = lr_t * m_t / (np.sqrt(v_t) + self.eps) deltas.append(delta_t) self.m_list[i] = m_t @@ -182,8 +174,8 @@ class MPIModel(): def __init__(self, model, optimizer, comm, batch_iterator, batch_size, num_replicas=None, warmup_steps=1000, lr=0.01, num_batches_minimum=100, conf=None): - random.seed(task_index) - np.random.seed(task_index) + random.seed(g.task_index) + np.random.seed(g.task_index) self.conf = conf self.start_time = time.time() self.epoch = 0 @@ -194,12 +186,13 @@ def __init__(self, model, optimizer, comm, batch_iterator, batch_size, self.optimizer = optimizer self.max_lr = 0.1 self.DUMMY_LR = 0.001 - self.comm = comm self.batch_size = batch_size self.batch_iterator = batch_iterator self.set_batch_iterator_func() self.warmup_steps = warmup_steps self.num_batches_minimum = num_batches_minimum + # TODO(KGF): duplicate/may be in conflict with global_vars.py + self.comm = comm self.num_workers = comm.Get_size() self.task_index = comm.Get_rank() self.history = cbks.History() @@ -265,11 +258,11 @@ def compile(self, optimizer, clipnorm, loss='mse'): self.ensure_equal_weights() def ensure_equal_weights(self): - if task_index == 0: + if g.task_index == 0: new_weights = self.model.get_weights() else: new_weights = None - nw = comm.bcast(new_weights, root=0) + nw = g.comm.bcast(new_weights, root=0) self.model.set_weights(nw) def train_on_batch_and_get_deltas(self, X_batch, Y_batch, verbose=False): @@ -498,7 +491,7 @@ def train_epoch(self): (batch_xs, batch_ys, batches_to_reset, num_so_far_curr, num_total, is_warmup_period) = next(batch_iterator_func) except StopIteration: - print_unique("Resetting batch iterator.") + g.print_unique("Resetting batch iterator.") self.num_so_far_accum = self.num_so_far_indiv self.set_batch_iterator_func() batch_iterator_func = self.batch_iterator_func @@ -524,7 +517,7 @@ def train_epoch(self): batch_xs, batch_ys, verbose) self.comm.Barrier() sys.stdout.flush() - print_unique('Compilation finished in {:.2f}s'.format( + g.print_unique('Compilation finished in {:.2f}s'.format( time.time() - t0_comp)) t_start = time.time() sys.stdout.flush() @@ -541,7 +534,7 @@ def train_epoch(self): t2 = time.time() write_str_0 = self.calculate_speed(t0, t1, t2, num_replicas) curr_loss = self.mpi_average_scalars(1.0*loss, num_replicas) - # print_unique(self.model.get_weights()[0][0][:4]) + # g.print_unique(self.model.get_weights()[0][0][:4]) loss_averager.add_val(curr_loss) ave_loss = loss_averager.get_val() eta = self.estimate_remaining_time( @@ -554,16 +547,16 @@ def train_epoch(self): + 'loss: {:.5f} [{:.5f}] | '.format(ave_loss, curr_loss) + 'walltime: {:.4f} | '.format( time.time() - self.start_time)) - write_unique(write_str + write_str_0) + g.write_unique(write_str + write_str_0) step += 1 else: - write_unique('\r[{}] warmup phase, num so far: {}'.format( + g.write_unique('\r[{}] warmup phase, num so far: {}'.format( self.task_index, self.num_so_far)) effective_epochs = 1.0*self.num_so_far/num_total epoch_previous = self.epoch self.epoch = effective_epochs - write_unique('\nEpoch {:.2f} finished ({:.2f} epochs passed)'.format( + g.write_unique('\nEpoch {:.2f} finished ({:.2f} epochs passed)'.format( 1.0 * self.epoch, self.epoch - epoch_previous) + ' in {:.2f} seconds.\n'.format(t2 - t_start)) return (step, ave_loss, curr_loss, self.num_so_far, effective_epochs) @@ -576,9 +569,10 @@ def estimate_remaining_time(self, time_so_far, work_so_far, work_total): def get_effective_lr(self, num_replicas): effective_lr = self.lr * num_replicas if effective_lr > self.max_lr: - write_unique('Warning: effective learning rate set to {}, '.format( - effective_lr) + 'larger than maximum {}. Clipping.'.format( - self.max_lr)) + g.write_unique( + 'Warning: effective learning rate set to {}, '.format( + effective_lr) + + 'larger than maximum {}. Clipping.'.format(self.max_lr)) effective_lr = self.max_lr return effective_lr @@ -604,44 +598,10 @@ def calculate_speed(self, t0, t_after_deltas, t_after_update, num_replicas, effective_batch_size, self.batch_size, num_replicas, self.get_effective_lr(num_replicas), self.lr, num_replicas) if verbose: - write_unique(print_str) + g.write_unique(print_str) return print_str -def print_unique(print_output, end='\n', flush=False): - """ - Only master MPI rank 0 calls print(). - - Trivial wrapper function to print() - """ - if task_index == 0: - print(print_output, end=end, flush=flush) - - -def write_unique(write_str): - """ - Only master MPI rank 0 writes to and flushes stdout. - - A specialized case of print_unique(). Unlike print(), sys.stdout.write(): - - Must pass a string; will not cast argument - - end='\n' kwarg of print() is not available - (often the argument here is prepended with \r=carriage return in order to - simulate a terminal output that overwrites itself) - """ - # TODO(KGF): \r carriage returns appear as ^M in Unix-encoded .out files - # from non-interactive Slurm batch jobs. Convert these to true Unix - # line feeds / newlines (^J, \n) when we can detect such a stdout - if task_index == 0: - sys.stdout.write(write_str) - sys.stdout.flush() - - -def write_all(write_str): - '''All MPI ranks write to stdout, appending [rank]''' - sys.stdout.write('[{}] '.format(task_index) + write_str) - sys.stdout.flush() - - def multiply_params(params, eps): return [el*eps for el in params] @@ -680,7 +640,7 @@ def load_shotlists(conf): def mpi_make_predictions(conf, shot_list, loader, custom_path=None): loader.set_inference_mode(True) - np.random.seed(task_index) + np.random.seed(g.task_index) shot_list.sort() # make sure all replicas have the same list specific_builder = builder.ModelBuilder(conf) @@ -693,26 +653,26 @@ def mpi_make_predictions(conf, shot_list, loader, custom_path=None): # broadcast model weights then set it explicitely: fix for Py3.6 if sys.version_info[0] > 2: - if task_index == 0: + if g.task_index == 0: new_weights = model.get_weights() else: new_weights = None - nw = comm.bcast(new_weights, root=0) + nw = g.comm.bcast(new_weights, root=0) model.set_weights(nw) model.reset_states() - if task_index == 0: + if g.task_index == 0: pbar = Progbar(len(shot_list)) shot_sublists = shot_list.sublists(conf['model']['pred_batch_size'], do_shuffle=False, equal_size=True) y_prime_global = [] y_gold_global = [] disruptive_global = [] - if task_index != 0: + if g.task_index != 0: loader.verbose = False for (i, shot_sublist) in enumerate(shot_sublists): - if i % num_workers == task_index: + if i % g.num_workers == g.task_index: X, y, shot_lengths, disr = loader.load_as_X_y_pred(shot_sublist) # load data and fit on data @@ -730,18 +690,19 @@ def mpi_make_predictions(conf, shot_list, loader, custom_path=None): disruptive += disr # print_all('\nFinished with i = {}'.format(i)) - if i % num_workers == num_workers - 1 or i == len(shot_sublists) - 1: - comm.Barrier() - y_prime_global += concatenate_sublists(comm.allgather(y_prime)) - y_gold_global += concatenate_sublists(comm.allgather(y_gold)) + if (i % g.num_workers == g.num_workers - 1 + or i == len(shot_sublists) - 1): + g.comm.Barrier() + y_prime_global += concatenate_sublists(g.comm.allgather(y_prime)) + y_gold_global += concatenate_sublists(g.comm.allgather(y_gold)) disruptive_global += concatenate_sublists( - comm.allgather(disruptive)) - comm.Barrier() + g.comm.allgather(disruptive)) + g.comm.Barrier() y_prime = [] y_gold = [] disruptive = [] - if task_index == 0: + if g.task_index == 0: pbar.add(1.0*len(shot_sublist)) y_prime_global = y_prime_global[:len(shot_list)] @@ -793,7 +754,7 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, # TODO(KGF): this is not defined in conf.yaml, but added to processed dict # for the first time here: - conf['num_workers'] = comm.Get_size() + conf['num_workers'] = g.comm.Get_size() specific_builder = builder.ModelBuilder(conf) train_model = specific_builder.build_model(False) @@ -821,19 +782,19 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, print("Optimizer not implemented yet") exit(1) - write_unique('{} epochs left to go'.format(num_epochs - 1 - e)) + g.write_unique('{} epochs left to go'.format(num_epochs - 1 - e)) batch_generator = partial(loader.training_batch_generator_partial_reset, shot_list=shot_list_train) - write_unique("warmup steps = {}".format(warmup_steps)) - mpi_model = MPIModel(train_model, optimizer, comm, batch_generator, + g.write_unique("warmup steps = {}".format(warmup_steps)) + mpi_model = MPIModel(train_model, optimizer, g.comm, batch_generator, batch_size, lr=lr, warmup_steps=warmup_steps, num_batches_minimum=num_batches_minimum, conf=conf) mpi_model.compile(conf['model']['optimizer'], clipnorm, conf['data']['target'].loss) tensorboard = None - if backend != "theano" and task_index == 0: + if g.backend != "theano" and g.task_index == 0: tensorboard_save_path = conf['paths']['tensorboard_save_path'] write_grads = conf['callbacks']['write_grads'] tensorboard = TensorBoard(log_dir=tensorboard_save_path, @@ -842,7 +803,7 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, tensorboard.set_model(mpi_model.model) mpi_model.model.summary() - if task_index == 0: + if g.task_index == 0: callbacks = mpi_model.build_callbacks(conf, callbacks_list) callbacks.set_model(mpi_model.model) callback_metrics = conf['callbacks']['metrics'] @@ -858,18 +819,18 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, cmp_fn = min while e < (num_epochs - 1): - write_unique("begin epoch {}".format(e)) - if task_index == 0: + g.write_unique("begin epoch {}".format(e)) + if g.task_index == 0: callbacks.on_epoch_begin(int(round(e))) mpi_model.set_lr(lr*lr_decay**e) - write_unique('\nEpoch {}/{}'.format(e, num_epochs)) + g.write_unique('\nEpoch {}/{}'.format(e, num_epochs)) (step, ave_loss, curr_loss, num_so_far, effective_epochs) = mpi_model.train_epoch() e = e_old + effective_epochs loader.verbose = False # True during the first iteration - if task_index == 0: + if g.task_index == 0: specific_builder.save_model_weights(train_model, int(round(e))) epoch_logs = {} @@ -894,13 +855,13 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, areas, _ = mpi_make_predictions_and_evaluate_multiple_times( conf, shot_list_validate, loader, times) for roc, t in zip(areas, times): - write_unique('epoch {}, val_roc_{} = {}'.format( + g.write_unique('epoch {}, val_roc_{} = {}'.format( int(round(e)), t, roc)) if shot_list_test is not None: areas, _ = mpi_make_predictions_and_evaluate_multiple_times( conf, shot_list_test, loader, times) for roc, t in zip(areas, times): - write_unique('epoch {}, test_roc_{} = {}'.format( + g.write_unique('epoch {}, test_roc_{} = {}'.format( int(round(e)), t, roc)) epoch_logs['val_roc'] = roc_area @@ -909,7 +870,7 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, best_so_far = cmp_fn(epoch_logs[conf['callbacks']['monitor']], best_so_far) stop_training = False - if task_index == 0: + if g.task_index == 0: print('=========Summary======== for epoch{}'.format(step)) print('Training Loss numpy: {:.3e}'.format(ave_loss)) print('Validation Loss: {:.3e}'.format(loss)) @@ -932,20 +893,19 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, train_model, int(round(e))) # tensorboard - if backend != 'theano': + if g.backend != 'theano': val_generator = partial(loader.training_batch_generator, shot_list=shot_list_validate)() val_steps = 1 tensorboard.on_epoch_end(val_generator, val_steps, int(round(e)), epoch_logs) - - stop_training = comm.bcast(stop_training, root=0) - write_unique("end epoch {}".format(e)) + stop_training = g.comm.bcast(stop_training, root=0) + g.write_unique("end epoch {}".format(e)) if stop_training: - write_unique("Stopping training due to early stopping") + g.write_unique("Stopping training due to early stopping") break - if task_index == 0: + if g.task_index == 0: callbacks.on_train_end() tensorboard.on_train_end() From 8a30ddd53bf8da6c77f6ad5104f4ea267850bcf5 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 5 Nov 2019 16:12:18 -0600 Subject: [PATCH 154/272] Forgot to pass conf to MPI init fn --- plasma/models/mpi_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 898e4de1..08b9c765 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -46,7 +46,7 @@ # set global variables for entire module regarding MPI environment # TODO(KGF): consider moving this fn/init call to mpi_learn.py and/or # setting "mpi_initialized" global bool flag, since that is client-facing -g.init_MPI() +g.init_MPI(conf) # initialization code for mpi_runner.py module: if g.backend == 'tf' or g.backend == 'tensorflow': From 168075047c59d896819112e38c1eea802cb6b863 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 5 Nov 2019 16:32:02 -0600 Subject: [PATCH 155/272] Suppress duplicated stdout output from guarantee_preprocessed() --- examples/mpi_learn.py | 3 ++- plasma/preprocessor/preprocess.py | 27 ++++++++++++++++----------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/examples/mpi_learn.py b/examples/mpi_learn.py index 935be33e..fc6ab95d 100644 --- a/examples/mpi_learn.py +++ b/examples/mpi_learn.py @@ -76,8 +76,9 @@ (shot_list_train, shot_list_validate, shot_list_test) = guarantee_preprocessed(conf) g.comm.Barrier() +# TODO(KGF): why call guarantee_preprocessed() a second time with all ranks? (shot_list_train, shot_list_validate, - shot_list_test) = guarantee_preprocessed(conf) + shot_list_test) = guarantee_preprocessed(conf, verbose=True) g.print_unique("begin normalization...") normalizer = Normalizer(conf) diff --git a/plasma/preprocessor/preprocess.py b/plasma/preprocessor/preprocess.py index 9861f4a4..e93e7309 100644 --- a/plasma/preprocessor/preprocess.py +++ b/plasma/preprocessor/preprocess.py @@ -9,6 +9,7 @@ ''' from __future__ import print_function +import plasma.global_vars as g from os import listdir # , remove import time import sys @@ -230,14 +231,16 @@ def apply_bleed_in(conf, shot_list_train, shot_list_validate, shot_list_test): return shot_list_train, shot_list_validate, shot_list_test -def guarantee_preprocessed(conf): +def guarantee_preprocessed(conf, verbose=False): pp = Preprocessor(conf) if pp.all_are_preprocessed(): - print("shots already processed.") + if verbose: + g.print_unique("shots already processed.") (shot_list_train, shot_list_validate, shot_list_test) = pp.load_shotlists() else: - print("preprocessing all shots", end='') + if verbose: + g.print_unique("preprocessing all shots...") # , end='') pp.clean_shot_lists() shot_list = pp.preprocess_all() shot_list.sort() @@ -245,18 +248,20 @@ def guarantee_preprocessed(conf): # num_shots = len(shot_list_train) + len(shot_list_test) validation_frac = conf['training']['validation_frac'] if validation_frac <= 0.05: - print('Setting validation to a minimum of 0.05') + if verbose: + g.print_unique('Setting validation to a minimum of 0.05') validation_frac = 0.05 shot_list_train, shot_list_validate = shot_list_train.split_direct( 1.0-validation_frac, do_shuffle=True) pp.save_shotlists(shot_list_train, shot_list_validate, shot_list_test) shot_list_train, shot_list_validate, shot_list_test = apply_bleed_in( conf, shot_list_train, shot_list_validate, shot_list_test) - print('validate: {} shots, {} disruptive'.format( - len(shot_list_validate), shot_list_validate.num_disruptive())) - print('training: {} shots, {} disruptive'.format( - len(shot_list_train), shot_list_train.num_disruptive())) - print('testing: {} shots, {} disruptive'.format( - len(shot_list_test), shot_list_test.num_disruptive())) - print("...done") + if verbose: + g.print_unique('validate: {} shots, {} disruptive'.format( + len(shot_list_validate), shot_list_validate.num_disruptive())) + g.print_unique('training: {} shots, {} disruptive'.format( + len(shot_list_train), shot_list_train.num_disruptive())) + g.print_unique('testing: {} shots, {} disruptive'.format( + len(shot_list_test), shot_list_test.num_disruptive())) + g.print_unique("...done") return shot_list_train, shot_list_validate, shot_list_test From df41970222185665bdec6aa7ae9fa56d91c5cbd7 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 5 Nov 2019 17:01:22 -0600 Subject: [PATCH 156/272] Suppress duplicated stdout output from normalize.py --- examples/mpi_learn.py | 14 ++++++++++++++ plasma/models/mpi_runner.py | 4 ++-- plasma/preprocessor/normalize.py | 13 +++++++------ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/examples/mpi_learn.py b/examples/mpi_learn.py index fc6ab95d..b938f23a 100644 --- a/examples/mpi_learn.py +++ b/examples/mpi_learn.py @@ -80,12 +80,26 @@ (shot_list_train, shot_list_validate, shot_list_test) = guarantee_preprocessed(conf, verbose=True) +# TODO(KGF): shouldn't normalize.train() be called like guaranteed_preprocessed +# above? I.e. if Normalizer.previously_saved_stats() does not load a computed +# normalizer for all machines ("loaded normalization data from {d3d: 3449, jet: 2918} # noqa +# shots ( {d3d: 810, jet: 74} disruptive )" ), then only the master MPI rank +# calls normalizer.train() ??? g.print_unique("begin normalization...") normalizer = Normalizer(conf) normalizer.train() loader = Loader(conf, normalizer) g.print_unique("...done") +# TODO(KGF): note, "python examples/guaranteed_preprocessed.py" does NOT train +# the normalizer. Try deleting the previously-computed file, e.g. +# normalization/normalization_signal_group_250640798211266795112500621861190558178.npz # noqa +# or set conf['data']['recompute_normalization'] = True to see example stdout + +# TODO(KGF): both preprocess.py and normalize.py are littered with print() +# calls that should probably be replaced with print_unique() when they are not +# purely loading previously-computed quantities from file + ##################################################### # TRAINING # ##################################################### diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 08b9c765..e22d36bd 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -782,12 +782,12 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, print("Optimizer not implemented yet") exit(1) - g.write_unique('{} epochs left to go'.format(num_epochs - 1 - e)) + g.print_unique('{} epochs left to go'.format(num_epochs - 1 - e)) batch_generator = partial(loader.training_batch_generator_partial_reset, shot_list=shot_list_train) - g.write_unique("warmup steps = {}".format(warmup_steps)) + g.print_unique("warmup steps = {}".format(warmup_steps)) mpi_model = MPIModel(train_model, optimizer, g.comm, batch_generator, batch_size, lr=lr, warmup_steps=warmup_steps, num_batches_minimum=num_batches_minimum, conf=conf) diff --git a/plasma/preprocessor/normalize.py b/plasma/preprocessor/normalize.py index 1008bde7..e57a79d7 100644 --- a/plasma/preprocessor/normalize.py +++ b/plasma/preprocessor/normalize.py @@ -9,6 +9,7 @@ ''' from __future__ import print_function +import plasma.global_vars as g import os import time import sys @@ -74,7 +75,7 @@ def load_stats(self): pass def print_summary(self, action='loaded'): - print('{} normalization data from {} shots ( {} disruptive )'.format( + g.print_unique('{} normalization data from {} shots ( {} disruptive )'.format( action, self.num_processed, self.num_disruptive)) def set_inference_mode(self, val): @@ -149,7 +150,7 @@ def train_on_files(self, shot_files, use_shots, all_machines): self.save_stats() else: self.load_stats() - print(self) + g.print_unique(self) def cut_end_of_shot(self, shot): cut_shot_ends = self.conf['data']['cut_shot_ends'] @@ -222,7 +223,7 @@ def __str__(self): for machine in self.means: means = np.median(self.means[machine], axis=0) stds = np.median(self.stds[machine], axis=0) - s += 'Machine: {}:\nMean Var Normalizer.\n'.format(machine) + s += 'Machine = {}:\nMean Var Normalizer.\n'.format(machine) s += 'means: {}\nstds: {}'.format(means, stds) return s @@ -304,8 +305,8 @@ def load_stats(self): self.num_disruptive = dat['num_disruptive'][()] self.machines = dat['machines'][()] for machine in self.means: - print('Machine {}:'.format(machine)) - self.print_summary() + g.print_unique('Machine = {}:'.format(machine)) + self.print_summary() class VarNormalizer(MeanVarNormalizer): @@ -452,7 +453,7 @@ def load_stats(self): self.num_disruptive = dat['num_disruptive'][()] self.machines = dat['machines'][()] for machine in self.means: - print('Machine {}:'.format(machine)) + g.print_unique('Machine {}:'.format(machine)) self.print_summary() From 409d5d5cc4a58fd5441e1887bb2da97ebb4e214a Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 5 Nov 2019 17:04:30 -0600 Subject: [PATCH 157/272] Shorten line --- plasma/preprocessor/normalize.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plasma/preprocessor/normalize.py b/plasma/preprocessor/normalize.py index e57a79d7..65067508 100644 --- a/plasma/preprocessor/normalize.py +++ b/plasma/preprocessor/normalize.py @@ -75,8 +75,9 @@ def load_stats(self): pass def print_summary(self, action='loaded'): - g.print_unique('{} normalization data from {} shots ( {} disruptive )'.format( - action, self.num_processed, self.num_disruptive)) + g.print_unique( + '{} normalization data from {} shots ( {} disruptive )'.format( + action, self.num_processed, self.num_disruptive)) def set_inference_mode(self, val): self.inference_mode = val From a571715e41ba0019da0219f182c722a7bfe8dd89 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 5 Nov 2019 17:43:34 -0600 Subject: [PATCH 158/272] Try ensuring that computing normalizer is only performed by master rank --- examples/mpi_learn.py | 29 ++++++++++------------------- plasma/models/mpi_runner.py | 3 ++- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/examples/mpi_learn.py b/examples/mpi_learn.py index b938f23a..5b96efbe 100644 --- a/examples/mpi_learn.py +++ b/examples/mpi_learn.py @@ -70,44 +70,35 @@ ##################################################### # NORMALIZATION # ##################################################### -# make sure preprocessing has been run, and is saved as a file +normalizer = Normalizer(conf) + if g.task_index == 0: - # TODO(KGF): check tuple unpack + # make sure preprocessing has been run, and results are saved to files + # if not, only master MPI rank spawns thread pool to perform preprocessing (shot_list_train, shot_list_validate, shot_list_test) = guarantee_preprocessed(conf) + # similarly, train normalizer (if necessary) w/ master MPI rank only + normalizer.train() g.comm.Barrier() -# TODO(KGF): why call guarantee_preprocessed() a second time with all ranks? +g.print_unique("begin preprocessor+normalization (all MPI ranks)...") +# second call has ALL MPI ranks load preprocessed shots from .npz files (shot_list_train, shot_list_validate, shot_list_test) = guarantee_preprocessed(conf, verbose=True) - -# TODO(KGF): shouldn't normalize.train() be called like guaranteed_preprocessed -# above? I.e. if Normalizer.previously_saved_stats() does not load a computed -# normalizer for all machines ("loaded normalization data from {d3d: 3449, jet: 2918} # noqa -# shots ( {d3d: 810, jet: 74} disruptive )" ), then only the master MPI rank -# calls normalizer.train() ??? -g.print_unique("begin normalization...") -normalizer = Normalizer(conf) +# second call to normalizer training normalizer.train() loader = Loader(conf, normalizer) g.print_unique("...done") -# TODO(KGF): note, "python examples/guaranteed_preprocessed.py" does NOT train -# the normalizer. Try deleting the previously-computed file, e.g. -# normalization/normalization_signal_group_250640798211266795112500621861190558178.npz # noqa -# or set conf['data']['recompute_normalization'] = True to see example stdout - # TODO(KGF): both preprocess.py and normalize.py are littered with print() # calls that should probably be replaced with print_unique() when they are not # purely loading previously-computed quantities from file +# (or we can continue to ensure that they are only ever executed by 1 rank) ##################################################### # TRAINING # ##################################################### # ensure training has a separate random seed for every worker -# TODO(KGF): can probably delete the next two lines. Check. -np.random.seed(g.task_index) -random.seed(g.task_index) if not only_predict: mpi_train(conf, shot_list_train, shot_list_validate, loader, shot_list_test=shot_list_test) diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index e22d36bd..4a9d8b65 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -517,7 +517,8 @@ def train_epoch(self): batch_xs, batch_ys, verbose) self.comm.Barrier() sys.stdout.flush() - g.print_unique('Compilation finished in {:.2f}s'.format( + # TODO(KGF): check line feed/carriage returns around this + g.print_unique('\nCompilation finished in {:.2f}s'.format( time.time() - t0_comp)) t_start = time.time() sys.stdout.flush() From 706f44d153e13de35dbaeb62ab0e57955d95374d Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 5 Nov 2019 18:01:19 -0600 Subject: [PATCH 159/272] Always set recompute_normalization to False inside Normalizer after compute --- examples/mpi_learn.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/mpi_learn.py b/examples/mpi_learn.py index 5b96efbe..3d0d4545 100644 --- a/examples/mpi_learn.py +++ b/examples/mpi_learn.py @@ -71,7 +71,6 @@ # NORMALIZATION # ##################################################### normalizer = Normalizer(conf) - if g.task_index == 0: # make sure preprocessing has been run, and results are saved to files # if not, only master MPI rank spawns thread pool to perform preprocessing @@ -85,7 +84,10 @@ (shot_list_train, shot_list_validate, shot_list_test) = guarantee_preprocessed(conf, verbose=True) # second call to normalizer training +normalizer.conf['data']['recompute_normalization'] = False normalizer.train() +# KGF: may want to set it back... +# normalizer.conf['data']['recompute_normalization'] = conf['data']['recompute_normalization'] # noqa loader = Loader(conf, normalizer) g.print_unique("...done") From 2fc80892143bc9dd8881530c84cd9882158c420c Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Tue, 5 Nov 2019 18:29:50 -0600 Subject: [PATCH 160/272] Suppress duplicate output from 2x calls to Normalizer.train() Using new verbose kwarg --- examples/mpi_learn.py | 4 +-- plasma/preprocessor/normalize.py | 53 ++++++++++++++++++-------------- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/examples/mpi_learn.py b/examples/mpi_learn.py index 3d0d4545..a23e251a 100644 --- a/examples/mpi_learn.py +++ b/examples/mpi_learn.py @@ -77,7 +77,7 @@ (shot_list_train, shot_list_validate, shot_list_test) = guarantee_preprocessed(conf) # similarly, train normalizer (if necessary) w/ master MPI rank only - normalizer.train() + normalizer.train() # verbose=False only suppresses if purely loading g.comm.Barrier() g.print_unique("begin preprocessor+normalization (all MPI ranks)...") # second call has ALL MPI ranks load preprocessed shots from .npz files @@ -85,7 +85,7 @@ shot_list_test) = guarantee_preprocessed(conf, verbose=True) # second call to normalizer training normalizer.conf['data']['recompute_normalization'] = False -normalizer.train() +normalizer.train(verbose=True) # KGF: may want to set it back... # normalizer.conf['data']['recompute_normalization'] = conf['data']['recompute_normalization'] # noqa loader = Loader(conf, normalizer) diff --git a/plasma/preprocessor/normalize.py b/plasma/preprocessor/normalize.py index 65067508..19606121 100644 --- a/plasma/preprocessor/normalize.py +++ b/plasma/preprocessor/normalize.py @@ -67,11 +67,11 @@ def apply(self, shot): pass @abc.abstractmethod - def save_stats(self): + def save_stats(self, verbose=False): pass @abc.abstractmethod - def load_stats(self): + def load_stats(self, verbose=False): pass def print_summary(self, action='loaded'): @@ -88,7 +88,7 @@ def ensure_machine(self, machine): self.num_disruptive[machine] = 0 # Modify the above to change the specifics of the normalization scheme - def train(self): + def train(self, verbose=False): conf = self.conf # only use training shots here!! "Don't touch testing shots" # + conf['paths']['shot_files_test'] @@ -106,9 +106,11 @@ def train(self): # shot_list_dir = conf['paths']['shot_list_dir'] use_shots = max(400, conf['data']['use_shots']) - return self.train_on_files(shot_files_use, use_shots, all_machines) + return self.train_on_files(shot_files_use, use_shots, all_machines, + verbose=verbose) - def train_on_files(self, shot_files, use_shots, all_machines): + def train_on_files(self, shot_files, use_shots, all_machines, + verbose=False): conf = self.conf all_signals = conf['paths']['all_signals'] shot_list = ShotList() @@ -121,10 +123,9 @@ def train_on_files(self, shot_files, use_shots, all_machines): if recompute: machines_to_compute = all_machines previously_saved = False - if not previously_saved or len(machines_to_compute) > 0: if previously_saved: - self.load_stats() + self.load_stats(verbose=True) print('computing normalization for machines {}'.format( machines_to_compute)) use_cores = max(1, mp.cpu_count()-2) @@ -142,16 +143,18 @@ def train_on_files(self, shot_files, use_shots, all_machines): self.machines.add(stats.machine) sys.stdout.write('\r' + '{}/{}'.format(i, len(shot_list_picked))) - pool.close() pool.join() - print('Finished Training Normalizer on ', + print('\nFinished Training Normalizer on ', '{} files in {} seconds'.format(len(shot_list_picked), time.time()-start_time)) - self.save_stats() + self.save_stats(verbose=True) else: - self.load_stats() - g.print_unique(self) + self.load_stats(verbose=verbose) + # print representation of trained Normalizer to stdout: + # Machine, NormalizerName, per-signal normalization stats/params + if verbose: + g.print_unique(self) def cut_end_of_shot(self, shot): cut_shot_ends = self.conf['data']['cut_shot_ends'] @@ -287,7 +290,7 @@ def apply(self, shot): # self.apply_positivity_mask(shot) # self.apply_mask(shot) - def save_stats(self): + def save_stats(self, verbose=False): # standard_deviations = dat['standard_deviations'] # num_processed = dat['num_processed'] # num_disruptive = dat['num_disruptive'] @@ -295,9 +298,10 @@ def save_stats(self): np.savez(self.path, means=self.means, stds=self.stds, num_processed=self.num_processed, num_disruptive=self.num_disruptive, machines=self.machines) - self.print_summary(action='saved') + if verbose: + self.print_summary(action='saved') - def load_stats(self): + def load_stats(self, verbose=False): assert self.previously_saved_stats()[0], "stats not saved before" dat = np.load(self.path, encoding="latin1", allow_pickle=True) self.means = dat['means'][()] @@ -305,9 +309,10 @@ def load_stats(self): self.num_processed = dat['num_processed'][()] self.num_disruptive = dat['num_disruptive'][()] self.machines = dat['machines'][()] - for machine in self.means: - g.print_unique('Machine = {}:'.format(machine)) - self.print_summary() + # for machine in self.means: + # g.print_unique('Machine = {}:'.format(machine)) + if verbose: + self.print_summary() class VarNormalizer(MeanVarNormalizer): @@ -435,7 +440,7 @@ def apply(self, shot): # self.apply_positivity_mask(shot) # self.apply_mask(shot) - def save_stats(self): + def save_stats(self, verbose=False): # standard_deviations = dat['standard_deviations'] # num_processed = dat['num_processed'] # num_disruptive = dat['num_disruptive'] @@ -443,9 +448,10 @@ def save_stats(self): np.savez(self.path, minimums=self.minimums, maximums=self.maximums, num_processed=self.num_processed, num_disruptive=self.num_disruptive, machines=self.machines) - self.print_summary(action='saved') + if verbose: + self.print_summary(action='saved') - def load_stats(self): + def load_stats(self, verbose=False): assert(self.previously_saved_stats()[0]) dat = np.load(self.path, encoding="latin1", allow_pickle=True) self.minimums = dat['minimums'][()] @@ -453,8 +459,9 @@ def load_stats(self): self.num_processed = dat['num_processed'][()] self.num_disruptive = dat['num_disruptive'][()] self.machines = dat['machines'][()] - for machine in self.means: - g.print_unique('Machine {}:'.format(machine)) + # for machine in self.means: + # g.print_unique('Machine {}:'.format(machine)) + if verbose: self.print_summary() From 64bfa8a4731a30b9c97adfd16df612f94ec45404 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 6 Nov 2019 11:56:06 -0500 Subject: [PATCH 161/272] Conditionally import deprecated TensorFlow V1 APIs Based on a rough sketch of which versions started deprecating them --- plasma/conf_parser.py | 1 - plasma/models/mpi_runner.py | 14 +++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index 06436c72..3e9a3cf3 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -20,7 +20,6 @@ def parameters(input_file): ) with open(input_file, 'r') as yaml_file: params = yaml.load(yaml_file, Loader=yaml.SafeLoader) - params['user_name'] = getpass.getuser() output_path = params['fs_path'] + "/" + params['user_name'] base_path = output_path diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 4a9d8b65..3910be74 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -9,6 +9,7 @@ from plasma.utils.state_reset import reset_states from plasma.conf import conf from mpi4py import MPI +from pkg_resources import parse_version, get_distribution ''' ######################################################### This file trains a deep learning model to predict @@ -54,7 +55,18 @@ os.environ['CUDA_VISIBLE_DEVICES'] = '{}'.format(g.MY_GPU) # ,mode=NanGuardMode' os.environ['KERAS_BACKEND'] = 'tensorflow' # default setting - import tensorflow as tf + tf_ver = parse_version(get_distribution('tensorflow').version) + # compat/compat.py first committed on 2018-06-29 for Py 2 vs 3 + # (around, but not present in, the release of v1.9.0) + # v2 compatiblity code added, then moved from compat.py in Nov and Dec 2018 + # compat.v1 first mentioned in RELEASE.md in v1.13.0. + # But many TF deprecation warnings in 1.14.0, e.g.: + # "The name tf.GPUOptions is deprecated. Please use tf.compat.v1.GPUOptions + # instead". See tf_export.py + if tf_ver > parse_version('1.13.0'): + import tensorflow.compat.v1 as tf + else: + import tensorflow as tf from keras.backend.tensorflow_backend import set_session gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.95, allow_growth=True) From 2c86e395e72f4d1838f83a524613977bea2d002d Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 6 Nov 2019 12:24:22 -0500 Subject: [PATCH 162/272] Call new MPI init fn in mpi_learn.py; call print_unique() about signals --- data/signals.py | 16 +++++++++------- examples/mpi_learn.py | 1 + plasma/conf_parser.py | 20 ++++++++++++-------- plasma/global_vars.py | 8 +++++--- plasma/models/mpi_runner.py | 10 +++++----- setup.cfg | 5 ++++- 6 files changed, 36 insertions(+), 24 deletions(-) diff --git a/data/signals.py b/data/signals.py index 34a53db2..4c1c9b8e 100644 --- a/data/signals.py +++ b/data/signals.py @@ -1,4 +1,5 @@ from __future__ import print_function +import plasma.global_vars as g import numpy as np import sys @@ -57,27 +58,27 @@ def get_units(str): found = True except Exception as e: - print(e) + g.print_unique(e) sys.stdout.flush() pass # Retrieve data from PTDATA if node not found if not found: - # print("not in full path {}".format(signal)) + # g.print_unique("not in full path {}".format(signal)) data = c.get('_s = ptdata2("'+signal+'",'+str(shot)+')').data() if len(data) != 1: rank = np.ndim(data) found = True # Retrieve data from Pseudo-pointname if not in ptdata if not found: - # print("not in PTDATA {}".format(signal)) + # g.print_unique("not in PTDATA {}".format(signal)) data = c.get('_s = pseudo("'+signal+'",'+str(shot)+')').data() if len(data) != 1: rank = np.ndim(data) found = True # this means the signal wasn't found if not found: - print("No such signal: {}".format(signal)) + g.print_unique("No such signal: {}".format(signal)) pass # get time base @@ -125,7 +126,7 @@ def fetch_jet_data(signal_path, shot_num, c): signal_path, shot_num)).data() found = True except Exception as e: - print(e) + g.print_unique(e) sys.stdout.flush() # pass return time, data, ydata, found @@ -361,8 +362,9 @@ def fetch_nstx_data(signal_path, shot_num, c): all_signals_restricted = all_signals -print('All signals (determines which signals are downloaded & preprocessed):') -print(all_signals.values()) +g.print_unique('All signals (determines which signals are downloaded' + ' & preprocessed):') +g.print_unique(all_signals.values()) fully_defined_signals = { sig_name: sig for (sig_name, sig) in all_signals_restricted.items() if ( diff --git a/examples/mpi_learn.py b/examples/mpi_learn.py index a23e251a..59737219 100644 --- a/examples/mpi_learn.py +++ b/examples/mpi_learn.py @@ -1,4 +1,5 @@ import plasma.global_vars as g +g.init_MPI() from plasma.models.mpi_runner import ( mpi_train, mpi_make_predictions_and_evaluate ) diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index 3e9a3cf3..4dd9bbec 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -1,3 +1,5 @@ +from __future__ import print_function +import plasma.global_vars as g from plasma.primitives.shots import ShotListFiles import data.signals as sig from plasma.utils.hashing import myhash_signals @@ -74,7 +76,7 @@ def parameters(input_file): elif params['target'] == 'ttdlinear': params['data']['target'] = TTDLinearTarget else: - print('Unkown type of target. Exiting') + g.print_unique('Unkown type of target. Exiting') exit(1) # params['model']['output_activation'] = @@ -344,15 +346,17 @@ def parameters(input_file): params['paths']['use_signals_dict'] = sig.fully_defined_signals_1D else: - print("Unkown data set {}".format(params['paths']['data'])) + g.print_unique("Unknown dataset {}".format( + params['paths']['data'])) exit(1) if len(params['paths']['specific_signals']): for s in params['paths']['specific_signals']: if s not in params['paths']['use_signals_dict'].keys(): - print("Signal {} is not fully defined for {} machine. ", - "Skipping...".format( - s, params['paths']['data'].split("_")[0])) + g.print_unique( + "Signal {} is not fully defined for {} machine. ", + "Skipping...".format( + s, params['paths']['data'].split("_")[0])) params['paths']['specific_signals'] = list( filter( lambda x: x in params['paths']['use_signals_dict'].keys(), @@ -370,9 +374,9 @@ def parameters(input_file): params['paths']['all_signals'] = sort_by_channels( list(params['paths']['all_signals_dict'].values())) - print("Selected signals (determines which signals are used for", - "training):\n{}".format(params['paths']['use_signals'])) - + g.print_unique("Selected signals (determines which signals are used", + " for training):\n{}".format( + params['paths']['use_signals'])) params['paths']['shot_files_all'] = ( params['paths']['shot_files'] + params['paths']['shot_files_test']) params['paths']['all_machines'] = list( diff --git a/plasma/global_vars.py b/plasma/global_vars.py index a97c8a83..1b569036 100644 --- a/plasma/global_vars.py +++ b/plasma/global_vars.py @@ -1,3 +1,4 @@ +from __future__ import print_function import sys # global variable defaults for non-MPI runs @@ -9,15 +10,16 @@ backend = '' -def init_MPI(conf): +def init_MPI(): from mpi4py import MPI global comm, task_index, num_workers - global NUM_GPUS, MY_GPU, backend - comm = MPI.COMM_WORLD task_index = comm.Get_rank() num_workers = comm.Get_size() + +def init_GPU_backend(conf): + global NUM_GPUS, MY_GPU, backend NUM_GPUS = conf['num_gpus'] MY_GPU = task_index % NUM_GPUS backend = conf['model']['backend'] diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 3910be74..1746f29b 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -39,15 +39,15 @@ import socket sys.setrecursionlimit(10000) - # import keras sequentially because it otherwise reads from ~/.keras/keras.json # with too many threads: # from mpi_launch_tensorflow import get_mpi_task_index -# set global variables for entire module regarding MPI environment -# TODO(KGF): consider moving this fn/init call to mpi_learn.py and/or -# setting "mpi_initialized" global bool flag, since that is client-facing -g.init_MPI(conf) +# set global variables for entire module regarding MPI & GPU environment +g.init_GPU_backend(conf) +# moved this fn/init call to client-facing mpi_learn.py +# g.init_MPI() +# TODO(KGF): set "mpi_initialized" global bool flag? # initialization code for mpi_runner.py module: if g.backend == 'tf' or g.backend == 'tensorflow': diff --git a/setup.cfg b/setup.cfg index 912ef080..5896f227 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,4 +26,7 @@ ignore = E731, # W5: Line break warning # W503: line break before binary operator (use mutually exclusive W504) - W503 \ No newline at end of file + W503 +# suppres linter warning about MPI init fn call before module-level imports +per-file-ignores = + examples/mpi_learn.py:E402 From 676621b91ef615633e5d35def45653efd0f648fe Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 6 Nov 2019 12:43:35 -0500 Subject: [PATCH 163/272] Only specify filename, not relative path, when using per-file-ignores Allows flake8 to be run within examples/ directory Note, Emacs flycheck appears to not pickup this (or --exclude) flake8 option/config --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 5896f227..b106c53d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,4 +29,4 @@ ignore = W503 # suppres linter warning about MPI init fn call before module-level imports per-file-ignores = - examples/mpi_learn.py:E402 + mpi_learn.py:E402 From fec3e5ae379a8c57c8923e9dc67659aafe80a241 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 6 Nov 2019 12:55:28 -0500 Subject: [PATCH 164/272] Cleanup setup.cfg comments --- setup.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index b106c53d..db3e245d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,7 +3,6 @@ description-file = README.md [flake8] max-line-length = 79 -# exclude = cpplint.py ignore = # Subset of DEFAULT_IGNORE error codes from pycodestyle # (not universally accepted / not enforced by PEP 8 document) @@ -27,6 +26,6 @@ ignore = # W5: Line break warning # W503: line break before binary operator (use mutually exclusive W504) W503 -# suppres linter warning about MPI init fn call before module-level imports +# suppress linter warning about MPI init fn call before module-level imports per-file-ignores = mpi_learn.py:E402 From b4bc1d139e5b9feb88a34f51801cfc7371c46895 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 6 Nov 2019 15:37:53 -0500 Subject: [PATCH 165/272] Fix bug in output--- was passing part of printed obj as "end" --- plasma/conf_parser.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plasma/conf_parser.py b/plasma/conf_parser.py index 4dd9bbec..9c338fb7 100644 --- a/plasma/conf_parser.py +++ b/plasma/conf_parser.py @@ -365,7 +365,6 @@ def parameters(input_file): for k in params['paths']['specific_signals']} params['paths']['use_signals'] = sort_by_channels( list(selected_signals.values())) - else: # default case params['paths']['use_signals'] = sort_by_channels( @@ -374,8 +373,8 @@ def parameters(input_file): params['paths']['all_signals'] = sort_by_channels( list(params['paths']['all_signals_dict'].values())) - g.print_unique("Selected signals (determines which signals are used", - " for training):\n{}".format( + g.print_unique("Selected signals (determines which signals are used" + + " for training):\n{}".format( params['paths']['use_signals'])) params['paths']['shot_files_all'] = ( params['paths']['shot_files'] + params['paths']['shot_files_test']) From 5f0cb09150bf4321ada4db0145384f81c3c6b82f Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 6 Nov 2019 15:39:18 -0500 Subject: [PATCH 166/272] Add fn for flushing BOTH stderr and stdout, in MPI rank order --- plasma/global_vars.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/plasma/global_vars.py b/plasma/global_vars.py index 1b569036..6aaa6f3f 100644 --- a/plasma/global_vars.py +++ b/plasma/global_vars.py @@ -37,6 +37,7 @@ def print_unique(print_output, end='\n', flush=False): Trivial wrapper function to print() """ + # TODO(KGF): maybe only allow end='','\r','\n' to prevent bugs? if task_index == 0: print(print_output, end=end, flush=flush) @@ -63,3 +64,17 @@ def write_all(write_str): '''All MPI ranks write to stdout, appending [rank]''' sys.stdout.write('[{}] '.format(task_index) + write_str) sys.stdout.flush() + + +def flush_all_inorder(stdout=True, stderr=True): + """Force each MPI rank to flush its buffered writes to one or both of + the standard streams, in order of rank. + """ + for i in range(num_workers): + comm.Barrier() + if i == task_index: + if stdout: + sys.stdout.flush() + if stderr: + sys.stderr.flush() + comm.Barrier() From c5dfa9d8d5a1fa0af2a6fe998705d95ecf428cea Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 6 Nov 2019 15:40:58 -0500 Subject: [PATCH 167/272] Completely sort out order of initial Keras and/or TensorFlow init msgs Dont forget about stderr! --- plasma/models/builder.py | 27 ++++++++++++++++++++++++--- plasma/models/mpi_runner.py | 18 +++++++++++++++++- plasma/models/targets.py | 9 ++++++++- 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/plasma/models/builder.py b/plasma/models/builder.py index 5e2c9eb0..cdb5faa8 100644 --- a/plasma/models/builder.py +++ b/plasma/models/builder.py @@ -1,4 +1,8 @@ -from __future__ import division +from __future__ import division, print_function +import plasma.global_vars as g +# KGF: the first time Keras is ever imported via mpi_learn.py -> mpi_runner.py +import keras.backend as K +# KGF: see below synchronization--- output is launched here from keras.models import Sequential, Model from keras.layers import Input from keras.layers.core import ( @@ -14,8 +18,6 @@ from keras.callbacks import Callback from keras.regularizers import l2 # l1, l1_l2 -import keras.backend as K - import re import os import sys @@ -24,6 +26,23 @@ from plasma.utils.downloading import makedirs_process_safe from plasma.utils.hashing import general_object_hash +# Synchronize 2x stderr msg from TensorFlow initialization via Keras backend +# "Succesfully opened dynamic library... libcudart" "Using TensorFlow backend." +if g.comm is not None: + g.flush_all_inorder() +# if g.comm is not None: +# g.comm.Barrier() +# if g.task_index == 0: +# sys.stdout.flush() +# sys.stderr.flush() +# if g.comm is not None: +# g.comm.Barrier() +# TODO(KGF): need to create wrapper .py file (or place in some __init__.py) +# that detects, for an arbitrary import, if tensorflow has been initialized +# either directly from "import tensorflow ..." and/or via backend of +# "from keras.layers ..." +# OR if this is the first time. See below "first_time" variable. + class LossHistory(Callback): def on_train_begin(self, logs=None): @@ -262,6 +281,8 @@ def slicer_output_shape(input_shape, indices): x_out = Dense(1, activation=output_activation)(x_in) model = Model(inputs=x_input, outputs=x_out) # bug with tensorflow/Keras + # TODO(KGF): what is this bug? this is the only direct "tensorflow" + # import outside of mpi_runner.py and runner.py if (conf['model']['backend'] == 'tf' or conf['model']['backend'] == 'tensorflow'): first_time = "tensorflow" not in sys.modules diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 1746f29b..f29d4697 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -4,9 +4,12 @@ from plasma.utils.performance import PerformanceAnalyzer from plasma.utils.processing import concatenate_sublists from plasma.utils.evaluation import get_loss_from_list +# KGF: this is the first module that imports Keras: from plasma.models import builder from plasma.models.loader import ProcessGenerator from plasma.utils.state_reset import reset_states +# KGF: plasma.conf calls print_unique() for "Selected signals". Ensure that +# Keras "Using TensorFlow backend" stderr messages do not interfere in stdout from plasma.conf import conf from mpi4py import MPI from pkg_resources import parse_version, get_distribution @@ -39,6 +42,7 @@ import socket sys.setrecursionlimit(10000) +# TODO(KGF): remove the next 3 lines? # import keras sequentially because it otherwise reads from ~/.keras/keras.json # with too many threads: # from mpi_launch_tensorflow import get_mpi_task_index @@ -49,6 +53,8 @@ # g.init_MPI() # TODO(KGF): set "mpi_initialized" global bool flag? +g.flush_all_inorder() # see above about conf_parser.py stdout writes + # initialization code for mpi_runner.py module: if g.backend == 'tf' or g.backend == 'tensorflow': if g.NUM_GPUS > 1: @@ -67,11 +73,20 @@ import tensorflow.compat.v1 as tf else: import tensorflow as tf + # TODO(KGF): above, builder.py (bug workaround), mpi_launch_tensorflow.py, + # and runner.py are the only files that import tensorflow directly + from keras.backend.tensorflow_backend import set_session + # KGF: next 3 lines dump many TensorFlow diagnostics to stderr. + # All MPI ranks first "Successfully opened dynamic library libcuda" + # then, one by one: ID GPU, libcudart, libcublas, libcufft, ... + # Finally, "Device interconnect StreamExecutor with strength 1 edge matrix" gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.95, allow_growth=True) config = tf.ConfigProto(gpu_options=gpu_options) set_session(tf.Session(config=config)) + g.flush_all_inorder() + g.comm.Barrier() else: os.environ['KERAS_BACKEND'] = 'theano' base_compile_dir = '{}/tmp/{}-{}'.format( @@ -90,8 +105,9 @@ from keras.utils.generic_utils import Progbar import keras.callbacks as cbks - g.pprint_unique(conf) +g.flush_all_inorder() +g.comm.Barrier() class MPIOptimizer(object): diff --git a/plasma/models/targets.py b/plasma/models/targets.py index 7f80187d..567fb43b 100644 --- a/plasma/models/targets.py +++ b/plasma/models/targets.py @@ -1,3 +1,5 @@ +from __future__ import print_function +import plasma.global_vars as g import numpy as np import abc @@ -7,9 +9,14 @@ ) import keras.backend as K from keras.losses import hinge -# Requirement: larger value must mean disruption more likely. + +# synchronize output from TensorFlow initialization via Keras backend +if g.comm is not None: + g.flush_all_inorder() + g.comm.Barrier() +# Requirement: larger value must mean disruption more likely. class Target(object): activation = 'linear' loss = 'mse' From 66e1c5cd1bf64ea3efeee9d2226e12d1d50b892f Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 6 Nov 2019 16:12:18 -0500 Subject: [PATCH 168/272] Workaround for "WARNING: Logging before flag parsing goes to stderr." Encountered with TensorFlow v1.14.0 on Traverse https://github.com/tensorflow/tensorflow/issues/26691 --- plasma/__init__.py | 10 ++++++++++ plasma/models/mpi_runner.py | 4 +--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/plasma/__init__.py b/plasma/__init__.py index 963dded2..55cff6db 100644 --- a/plasma/__init__.py +++ b/plasma/__init__.py @@ -1,3 +1,13 @@ +import logging +try: + import absl.logging + # https://github.com/abseil/abseil-py/issues/99 + logging.root.removeHandler(absl.logging._absl_handler) + # https://github.com/abseil/abseil-py/issues/102 + absl.logging._warn_preinit_stderr = False +except Exception: + pass + import warnings # TODO(KGF): temporarily suppress numpy>=1.17.0 warning with TF<2.0.0 # ~6x tensorflow/python/framework/dtypes.py:529: FutureWarning ... diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index f29d4697..2c15b405 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -105,6 +105,7 @@ from keras.utils.generic_utils import Progbar import keras.callbacks as cbks +g.flush_all_inorder() g.pprint_unique(conf) g.flush_all_inorder() g.comm.Barrier() @@ -164,11 +165,9 @@ def __init__(self, lr): self.eps = 1e-8 def get_deltas(self, raw_deltas): - if self.iterations == 0: self.m_list = [np.zeros_like(grad) for grad in raw_deltas] self.v_list = [np.zeros_like(grad) for grad in raw_deltas] - t = self.iterations + 1 lr_t = self.lr * np.sqrt(1-self.beta_2**t)/(1-self.beta_1**t) deltas = [] @@ -179,7 +178,6 @@ def get_deltas(self, raw_deltas): deltas.append(delta_t) self.m_list[i] = m_t self.v_list[i] = v_t - self.iterations += 1 return deltas From aa2fd04563daeff11a2edfec3c2ba1053c3523fc Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 7 Nov 2019 10:57:09 -0500 Subject: [PATCH 169/272] Suppress most TensorFlow deprecation warnings with external Keras --- plasma/__init__.py | 10 ++++++++++ plasma/models/mpi_runner.py | 31 ++++++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/plasma/__init__.py b/plasma/__init__.py index 55cff6db..ac888c0e 100644 --- a/plasma/__init__.py +++ b/plasma/__init__.py @@ -1,3 +1,4 @@ +import os import logging try: import absl.logging @@ -15,3 +16,12 @@ category=FutureWarning, message=r"passing \(type, 1\) or '1type' as a synonym of type is deprecated", # noqa module="tensorflow") + +# Optional: disable the C-based library diagnostic info and warning messages: +# 2019-11-06 18:27:31.698908: I ... dynamic library libcublas.so.10 +# (independent from tf.logging.set_verbosity() diagnostic control) +# Must be set before first import of tensorflow v0.12+ +# (either directly or via Keras backend) +# os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' +# Ref: https://github.com/tensorflow/tensorflow/issues/1258 +# https://stackoverflow.com/questions/35911252/disable-tensorflow-debugging-information/38645250#38645250 # noqa diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 2c15b405..f0e6de92 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -69,7 +69,7 @@ # But many TF deprecation warnings in 1.14.0, e.g.: # "The name tf.GPUOptions is deprecated. Please use tf.compat.v1.GPUOptions # instead". See tf_export.py - if tf_ver > parse_version('1.13.0'): + if tf_ver >= parse_version('1.13.0'): import tensorflow.compat.v1 as tf else: import tensorflow as tf @@ -86,7 +86,6 @@ config = tf.ConfigProto(gpu_options=gpu_options) set_session(tf.Session(config=config)) g.flush_all_inorder() - g.comm.Barrier() else: os.environ['KERAS_BACKEND'] = 'theano' base_compile_dir = '{}/tmp/{}-{}'.format( @@ -143,7 +142,6 @@ def __init__(self, lr): def get_deltas(self, raw_deltas): deltas = [] - if self.iterations == 0: self.velocity_list = [np.zeros_like(g) for g in raw_deltas] @@ -784,7 +782,34 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, conf['num_workers'] = g.comm.Get_size() specific_builder = builder.ModelBuilder(conf) + + # TODO(KGF): next line suppresses ALL info and warning messages, not just + # deprecation warnings... + # tf.logging.set_verbosity(tf.logging.ERROR) + + # Internal TensorFlow flags, subject to change (v1.14.0+ only?) + try: + from tensorflow.python.util import module_wrapper as deprecation + except ImportError: + from tensorflow.python.util import deprecation_wrapper as deprecation + # deprecation._PRINT_DEPRECATION_WARNINGS = False # does nothing + deprecation._PER_MODULE_WARNING_LIMIT = 0 + # Suppresses warnings from "keras/backend/tensorflow_backend.py", except: + # "Rate should be set to `rate = 1 - keep_prob`" + # Also suppresses warnings from "keras/optimizers.py + # does NOT suppresses warn from "/tensorflow/python/ops/math_grad.py" + + # TODO(KGF): for TF>v1.13.0 (esp v1.14.0), this next line prompts a ton of + # deprecation warnings with externally-packaged Keras, e.g.: + # WARNING:tensorflow:From .../keras/backend/tensorflow_backend.py:174: + # The name tf.get_default_session is deprecated. + # Please use tf.compat.v1.get_default_session instead. train_model = specific_builder.build_model(False) + # Cannot fix these Keras internals via "import tensorflow.compat.v1 as tf" + # + # TODO(KGF): note, these are different than C-based info diagnostics e.g.: + # 2019-11-06 18:27:31.698908: I ... dynamic library libcublas.so.10 + # which are NOT suppressed by set_verbosity. See top level __init__.py # load the latest epoch we did. Returns -1 if none exist yet e = specific_builder.load_model_weights(train_model) From af09d4fac73487c84bbc8a4899fc1ed7fbee5559 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 7 Nov 2019 11:10:12 -0500 Subject: [PATCH 170/272] Share TF version as a global variable Previous deprecation warning suppression approach only works for v1.14.0+ --- plasma/__init__.py | 2 +- plasma/global_vars.py | 1 + plasma/models/mpi_runner.py | 37 ++++++++++++++++++------------------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/plasma/__init__.py b/plasma/__init__.py index ac888c0e..3be7f58e 100644 --- a/plasma/__init__.py +++ b/plasma/__init__.py @@ -1,4 +1,4 @@ -import os +# import os import logging try: import absl.logging diff --git a/plasma/global_vars.py b/plasma/global_vars.py index 6aaa6f3f..1c667f9a 100644 --- a/plasma/global_vars.py +++ b/plasma/global_vars.py @@ -8,6 +8,7 @@ NUM_GPUS = 0 MY_GPU = 0 backend = '' +tf_ver = None def init_MPI(): diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index f0e6de92..aa0763c0 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -61,7 +61,7 @@ os.environ['CUDA_VISIBLE_DEVICES'] = '{}'.format(g.MY_GPU) # ,mode=NanGuardMode' os.environ['KERAS_BACKEND'] = 'tensorflow' # default setting - tf_ver = parse_version(get_distribution('tensorflow').version) + g.tf_ver = parse_version(get_distribution('tensorflow').version) # compat/compat.py first committed on 2018-06-29 for Py 2 vs 3 # (around, but not present in, the release of v1.9.0) # v2 compatiblity code added, then moved from compat.py in Nov and Dec 2018 @@ -69,7 +69,7 @@ # But many TF deprecation warnings in 1.14.0, e.g.: # "The name tf.GPUOptions is deprecated. Please use tf.compat.v1.GPUOptions # instead". See tf_export.py - if tf_ver >= parse_version('1.13.0'): + if g.tf_ver >= parse_version('1.14.0'): import tensorflow.compat.v1 as tf else: import tensorflow as tf @@ -782,23 +782,22 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, conf['num_workers'] = g.comm.Get_size() specific_builder = builder.ModelBuilder(conf) - - # TODO(KGF): next line suppresses ALL info and warning messages, not just - # deprecation warnings... - # tf.logging.set_verbosity(tf.logging.ERROR) - - # Internal TensorFlow flags, subject to change (v1.14.0+ only?) - try: - from tensorflow.python.util import module_wrapper as deprecation - except ImportError: - from tensorflow.python.util import deprecation_wrapper as deprecation - # deprecation._PRINT_DEPRECATION_WARNINGS = False # does nothing - deprecation._PER_MODULE_WARNING_LIMIT = 0 - # Suppresses warnings from "keras/backend/tensorflow_backend.py", except: - # "Rate should be set to `rate = 1 - keep_prob`" - # Also suppresses warnings from "keras/optimizers.py - # does NOT suppresses warn from "/tensorflow/python/ops/math_grad.py" - + if g.tf_ver >= parse_version('1.14.0'): + # Internal TensorFlow flags, subject to change (v1.14.0+ only?) + try: + from tensorflow.python.util import module_wrapper as depr + except ImportError: + from tensorflow.python.util import deprecation_wrapper as depr + # depr._PRINT_DEPRECATION_WARNINGS = False # does nothing + depr._PER_MODULE_WARNING_LIMIT = 0 + # Suppresses warnings from "keras/backend/tensorflow_backend.py" + # except: "Rate should be set to `rate = 1 - keep_prob`" + # Also suppresses warnings from "keras/optimizers.py + # does NOT suppresses warn from "/tensorflow/python/ops/math_grad.py" + else: + # TODO(KGF): next line suppresses ALL info and warning messages, + # not just deprecation warnings... + tf.logging.set_verbosity(tf.logging.ERROR) # TODO(KGF): for TF>v1.13.0 (esp v1.14.0), this next line prompts a ton of # deprecation warnings with externally-packaged Keras, e.g.: # WARNING:tensorflow:From .../keras/backend/tensorflow_backend.py:174: From 425ff0bedfdfb747830fa15d9a2d3cadce5da38c Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 7 Nov 2019 14:14:27 -0500 Subject: [PATCH 171/272] Make write_all() safe for use in non-MPI contexts --- plasma/global_vars.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/plasma/global_vars.py b/plasma/global_vars.py index 1c667f9a..a07b0d7e 100644 --- a/plasma/global_vars.py +++ b/plasma/global_vars.py @@ -62,8 +62,14 @@ def write_unique(write_str): def write_all(write_str): - '''All MPI ranks write to stdout, appending [rank]''' - sys.stdout.write('[{}] '.format(task_index) + write_str) + '''All MPI ranks write to stdout, appending [rank]. + + No MPI barriers, no guaranteed ordering of output. + ''' + if comm is not None: + sys.stdout.write('[{}] '.format(task_index) + write_str) + else: + sys.stdout.write(write_str) sys.stdout.flush() From 4b549efc99fd7304ec90fafa5d92bd9a1b03d074 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 7 Nov 2019 14:15:21 -0500 Subject: [PATCH 172/272] Use write_all() for ModelBuilder.load_model_weights() diagnostics instead of print() --- plasma/models/builder.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/plasma/models/builder.py b/plasma/models/builder.py index cdb5faa8..b2e29ed7 100644 --- a/plasma/models/builder.py +++ b/plasma/models/builder.py @@ -30,13 +30,6 @@ # "Succesfully opened dynamic library... libcudart" "Using TensorFlow backend." if g.comm is not None: g.flush_all_inorder() -# if g.comm is not None: -# g.comm.Barrier() -# if g.task_index == 0: -# sys.stdout.flush() -# sys.stderr.flush() -# if g.comm is not None: -# g.comm.Barrier() # TODO(KGF): need to create wrapper .py file (or place in some __init__.py) # that detects, for an arbitrary import, if tensorflow has been initialized # either directly from "import tensorflow ..." and/or via backend of @@ -318,20 +311,21 @@ def load_model_weights(self, model, custom_path=None): if custom_path is None: epochs = self.get_all_saved_files() if len(epochs) == 0: - print('no previous checkpoint found') + g.write_all('no previous checkpoint found\n') return -1 else: max_epoch = max(epochs) - print('loading from epoch {}'.format(max_epoch)) + g.write_all('loading from epoch {}\n'.format(max_epoch)) model.load_weights(self.get_save_path(max_epoch)) return max_epoch else: epoch = self.extract_id_and_epoch_from_filename( os.path.basename(custom_path))[1] model.load_weights(custom_path) - print("Loading from custom epoch {}".format(epoch)) + g.write_all("Loading from custom epoch {}\n".format(epoch)) return epoch + # TODO(KGF): method only called in non-MPI runner.py. Deduplicate? def get_latest_save_path(self): epochs = self.get_all_saved_files() if len(epochs) == 0: From 90007275d7f406742de2ac0cd9a3413c920e8e5d Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 7 Nov 2019 14:16:17 -0500 Subject: [PATCH 173/272] Fix and improve diagnostics within main mpi_train() loop over epochs Preparing to re-index epochs with 1-based indexing --- plasma/models/mpi_runner.py | 49 +++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index aa0763c0..88d9a2f2 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -450,7 +450,6 @@ def build_callbacks(self, conf, callbacks_list): os.makedirs(csvlog_save_path) callbacks_list = conf['callbacks']['list'] - callbacks = [cbks.BaseLogger()] callbacks += [self.history] callbacks += [cbks.CSVLogger("{}callbacks-{}.log".format( @@ -581,9 +580,10 @@ def train_epoch(self): effective_epochs = 1.0*self.num_so_far/num_total epoch_previous = self.epoch self.epoch = effective_epochs - g.write_unique('\nEpoch {:.2f} finished ({:.2f} epochs passed)'.format( - 1.0 * self.epoch, self.epoch - epoch_previous) - + ' in {:.2f} seconds.\n'.format(t2 - t_start)) + g.write_unique( + '\nEpoch {:.2f} finished training ({:.2f} epochs passed)'.format( + 1.0 * self.epoch, self.epoch - epoch_previous) + + ' in {:.2f} seconds.\n'.format(t2 - t_start)) return (step, ave_loss, curr_loss, self.num_so_far, effective_epochs) def estimate_remaining_time(self, time_so_far, work_so_far, work_total): @@ -687,6 +687,10 @@ def mpi_make_predictions(conf, shot_list, loader, custom_path=None): model.reset_states() if g.task_index == 0: + # TODO(KGF): this appears to prepend a \n, resulting in: + # [2] loading from epoch 7 + # + # 128/862 [===>..........................] - ETA: 2:20 pbar = Progbar(len(shot_list)) shot_sublists = shot_list.sublists(conf['model']['pred_batch_size'], do_shuffle=False, equal_size=True) @@ -870,12 +874,13 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, cmp_fn = min while e < (num_epochs - 1): - g.write_unique("begin epoch {}".format(e)) + g.write_unique('\nBegin training from epoch {:.2f}/{}'.format( + e, num_epochs)) if g.task_index == 0: callbacks.on_epoch_begin(int(round(e))) mpi_model.set_lr(lr*lr_decay**e) - g.write_unique('\nEpoch {}/{}'.format(e, num_epochs)) + # KGF: core work of loop performed in next line (step, ave_loss, curr_loss, num_so_far, effective_epochs) = mpi_model.train_epoch() e = e_old + effective_epochs @@ -885,9 +890,16 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, specific_builder.save_model_weights(train_model, int(round(e))) epoch_logs = {} + g.write_unique('Begin evaluation of epoch {:.2f}/{}\n'.format( + e, num_epochs)) + # TODO(KGF): flush output/ MPI barrier? + # g.flush_all_inorder() + # TODO(KGF): is there a way to avoid Keras.Models.load_weights() + # repeated calls throughout mpi_make_pred*() fn calls? _, _, _, roc_area, loss = mpi_make_predictions_and_evaluate( conf, shot_list_validate, loader) + if conf['training']['ranking_difficulty_fac'] != 1.0: (_, _, _, roc_area_train, loss_train) = mpi_make_predictions_and_evaluate( @@ -905,15 +917,18 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, times = conf['callbacks']['monitor_times'] areas, _ = mpi_make_predictions_and_evaluate_multiple_times( conf, shot_list_validate, loader, times) - for roc, t in zip(areas, times): - g.write_unique('epoch {}, val_roc_{} = {}'.format( - int(round(e)), t, roc)) + epoch_str = 'epoch {}, '.format(int(round(e))) + g.write_unique(epoch_str + ' '.join( + ['val_roc_{} = {}'.format(t, roc) for t, roc in zip( + times, areas)] + ) + '\n') if shot_list_test is not None: areas, _ = mpi_make_predictions_and_evaluate_multiple_times( conf, shot_list_test, loader, times) - for roc, t in zip(areas, times): - g.write_unique('epoch {}, test_roc_{} = {}'.format( - int(round(e)), t, roc)) + g.write_unique(epoch_str + ' '.join( + ['test_roc_{} = {}'.format(t, roc) for t, roc in zip( + times, areas)] + ) + '\n') epoch_logs['val_roc'] = roc_area epoch_logs['val_loss'] = loss @@ -921,15 +936,16 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, best_so_far = cmp_fn(epoch_logs[conf['callbacks']['monitor']], best_so_far) stop_training = False + g.flush_all_inorder() if g.task_index == 0: - print('=========Summary======== for epoch{}'.format(step)) + print('=========Summary======== for epoch {:.2f}'.format(e)) print('Training Loss numpy: {:.3e}'.format(ave_loss)) print('Validation Loss: {:.3e}'.format(loss)) print('Validation ROC: {:.4f}'.format(roc_area)) if conf['training']['ranking_difficulty_fac'] != 1.0: print('Training Loss: {:.3e}'.format(loss_train)) print('Training ROC: {:.4f}'.format(roc_area_train)) - + print('======================== ') callbacks.on_epoch_end(int(round(e)), epoch_logs) if hasattr(mpi_model.model, 'stop_training'): stop_training = mpi_model.model.stop_training @@ -951,7 +967,10 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, tensorboard.on_epoch_end(val_generator, val_steps, int(round(e)), epoch_logs) stop_training = g.comm.bcast(stop_training, root=0) - g.write_unique("end epoch {}".format(e)) + g.write_unique('Finished evaluation of epoch {:.2f}/{}'.format( + e, num_epochs)) + # TODO(KGF): compare to old diagnostic: + # g.write_unique("end epoch {}".format(e_old)) if stop_training: g.write_unique("Stopping training due to early stopping") break From b036c0bb2c215750760d6f8784b09950ac4c2205 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 7 Nov 2019 14:31:42 -0500 Subject: [PATCH 174/272] Make it explicit that diagnostic within train_epoch() refers to local progress --- examples/mpi_learn.py | 9 +++++++-- plasma/models/mpi_runner.py | 14 +++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/examples/mpi_learn.py b/examples/mpi_learn.py index 59737219..f4d6da39 100644 --- a/examples/mpi_learn.py +++ b/examples/mpi_learn.py @@ -67,7 +67,6 @@ custom_path = sys.argv[1] g.print_unique("predicting using path {}".format(custom_path)) - ##################################################### # NORMALIZATION # ##################################################### @@ -101,10 +100,16 @@ # TRAINING # ##################################################### -# ensure training has a separate random seed for every worker +# Prevent Keras TF backend deprecation messages from mpi_train() from +# appearing jumbled with stdout, stderr msgs from above steps +g.comm.Barrier() +g.flush_all_inorder() + +# reminder: ensure training has a separate random seed for every worker if not only_predict: mpi_train(conf, shot_list_train, shot_list_validate, loader, shot_list_test=shot_list_test) +g.flush_all_inorder() ##################################################### # TESTING # diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 88d9a2f2..4a659da0 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -581,9 +581,14 @@ def train_epoch(self): epoch_previous = self.epoch self.epoch = effective_epochs g.write_unique( - '\nEpoch {:.2f} finished training ({:.2f} epochs passed)'.format( - 1.0 * self.epoch, self.epoch - epoch_previous) - + ' in {:.2f} seconds.\n'.format(t2 - t_start)) + # TODO(KGF): "a total of X epochs within this session" ? + '\nFinished training epoch {:.2f} '.format(self.epoch) + # TODO(KGF): "precisely/exactly X epochs just passed"? + + 'during this session ({:.2f} epochs passed)'.format( + self.epoch - epoch_previous) + # '\nEpoch {:.2f} finished training ({:.2f} epochs passed)'.format( + # 1.0 * self.epoch, self.epoch - epoch_previous) + + ' in {:.2f} seconds\n'.format(t2 - t_start)) return (step, ave_loss, curr_loss, self.num_so_far, effective_epochs) def estimate_remaining_time(self, time_so_far, work_so_far, work_total): @@ -884,7 +889,10 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, (step, ave_loss, curr_loss, num_so_far, effective_epochs) = mpi_model.train_epoch() e = e_old + effective_epochs + g.write_unique('Finished training of epoch {:.2f}/{}\n'.format( + e, num_epochs)) + # TODO(KGF): add diagnostic about "saving to epoch X"? loader.verbose = False # True during the first iteration if g.task_index == 0: specific_builder.save_model_weights(train_model, int(round(e))) From df88d1a3251a68c2b381df8dcb00b1a5aa7a8f69 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 7 Nov 2019 14:49:07 -0500 Subject: [PATCH 175/272] Use 1-based indexing for epochs --- plasma/models/builder.py | 4 +++- plasma/models/mpi_runner.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/plasma/models/builder.py b/plasma/models/builder.py index b2e29ed7..e47f6588 100644 --- a/plasma/models/builder.py +++ b/plasma/models/builder.py @@ -312,7 +312,9 @@ def load_model_weights(self, model, custom_path=None): epochs = self.get_all_saved_files() if len(epochs) == 0: g.write_all('no previous checkpoint found\n') - return -1 + # TODO(KGF): port indexing change (from "return -1") to parts + # of the code other than mpi_runner.py + return 0 else: max_epoch = max(epochs) g.write_all('loading from epoch {}\n'.format(max_epoch)) diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py index 4a659da0..30e0caed 100644 --- a/plasma/models/mpi_runner.py +++ b/plasma/models/mpi_runner.py @@ -819,7 +819,7 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader, # 2019-11-06 18:27:31.698908: I ... dynamic library libcublas.so.10 # which are NOT suppressed by set_verbosity. See top level __init__.py - # load the latest epoch we did. Returns -1 if none exist yet + # load the latest epoch we did. Returns 0 if none exist yet e = specific_builder.load_model_weights(train_model) e_old = e From 9935ff79eab0d6cf3674be2a24f5656301745921 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Wed, 13 Nov 2019 15:49:07 -0500 Subject: [PATCH 176/272] Restrict Conda TensorFlow dependency to <2.0.0 This version is now generally available on Anaconda Cloud and should be avoided. --- requirements-travis.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-travis.txt b/requirements-travis.txt index 77056dd9..69dfd817 100644 --- a/requirements-travis.txt +++ b/requirements-travis.txt @@ -4,4 +4,4 @@ flake8 h5py pyparsing pyyaml -tensorflow-gpu>=1.3 +tensorflow-gpu>=1.3,<2.0.0 From 3a3a9f27444ea77c17034fa8740c520bc7d870fb Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 21 Nov 2019 14:05:08 -0500 Subject: [PATCH 177/272] Print out total shot counts in preprocess.py --- plasma/preprocessor/preprocess.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plasma/preprocessor/preprocess.py b/plasma/preprocessor/preprocess.py index e93e7309..2c6384e4 100644 --- a/plasma/preprocessor/preprocess.py +++ b/plasma/preprocessor/preprocess.py @@ -257,6 +257,11 @@ def guarantee_preprocessed(conf, verbose=False): shot_list_train, shot_list_validate, shot_list_test = apply_bleed_in( conf, shot_list_train, shot_list_validate, shot_list_test) if verbose: + g.print_unique('total: {} shots, {} disruptive'.format( + len(shot_list_validate)+len(shot_list_train)+len(shot_list_test), + shot_list_validate.num_disruptive() + + shot_list_train.num_disruptive() + + shot_list_test.num_disruptive())) g.print_unique('validate: {} shots, {} disruptive'.format( len(shot_list_validate), shot_list_validate.num_disruptive())) g.print_unique('training: {} shots, {} disruptive'.format( From 09c4d6771c7883c195b3f1b26fb6155e88b4469e Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 21 Nov 2019 14:25:12 -0500 Subject: [PATCH 178/272] Always print # train shots before # validate shots in diagnostics Canonical order: train, validate, test --- plasma/models/runner.py | 4 ++-- plasma/models/shallow_runner.py | 6 +++--- plasma/models/torch_runner.py | 5 ++--- plasma/preprocessor/preprocess.py | 4 ++-- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/plasma/models/runner.py b/plasma/models/runner.py index 5b3f1d5a..29c42c3c 100644 --- a/plasma/models/runner.py +++ b/plasma/models/runner.py @@ -29,10 +29,10 @@ def train(conf, shot_list_train, shot_list_validate, loader, validation_losses = [] validation_roc = [] training_losses = [] - print('validate: {} shots, {} disruptive'.format( - len(shot_list_validate), shot_list_validate.num_disruptive())) print('training: {} shots, {} disruptive'.format( len(shot_list_train), shot_list_train.num_disruptive())) + print('validate: {} shots, {} disruptive'.format( + len(shot_list_validate), shot_list_validate.num_disruptive())) if backend == 'tf' or backend == 'tensorflow': first_time = "tensorflow" not in sys.modules diff --git a/plasma/models/shallow_runner.py b/plasma/models/shallow_runner.py index 5d459a43..cdfe2c32 100644 --- a/plasma/models/shallow_runner.py +++ b/plasma/models/shallow_runner.py @@ -324,12 +324,12 @@ def build_callbacks(conf): def train(conf, shot_list_train, shot_list_validate, loader, shot_list_test=None): np.random.seed(1) - print('validate: {} shots, {} disruptive'.format( - len(shot_list_validate), - shot_list_validate.num_disruptive())) print('training: {} shots, {} disruptive'.format( len(shot_list_train), shot_list_train.num_disruptive())) + print('validate: {} shots, {} disruptive'.format( + len(shot_list_validate), + shot_list_validate.num_disruptive())) num_samples = conf['model']['shallow_model']['num_samples'] feature_extractor = FeatureExtractor(loader) diff --git a/plasma/models/torch_runner.py b/plasma/models/torch_runner.py index 5c85e134..a4fde559 100644 --- a/plasma/models/torch_runner.py +++ b/plasma/models/torch_runner.py @@ -414,11 +414,10 @@ def train(conf, shot_list_train, shot_list_validate, loader): data_gen = partial( loader.training_batch_generator_full_shot_partial_reset, shot_list=shot_list_train)() - print('validate: {} shots, {} disruptive'.format( - len(shot_list_validate), shot_list_validate.num_disruptive())) print('training: {} shots, {} disruptive'.format( len(shot_list_train), shot_list_train.num_disruptive())) - + print('validate: {} shots, {} disruptive'.format( + len(shot_list_validate), shot_list_validate.num_disruptive())) loader.set_inference_mode(False) train_model = build_torch_model(conf) diff --git a/plasma/preprocessor/preprocess.py b/plasma/preprocessor/preprocess.py index 2c6384e4..deaccee3 100644 --- a/plasma/preprocessor/preprocess.py +++ b/plasma/preprocessor/preprocess.py @@ -262,10 +262,10 @@ def guarantee_preprocessed(conf, verbose=False): shot_list_validate.num_disruptive() + shot_list_train.num_disruptive() + shot_list_test.num_disruptive())) - g.print_unique('validate: {} shots, {} disruptive'.format( - len(shot_list_validate), shot_list_validate.num_disruptive())) g.print_unique('training: {} shots, {} disruptive'.format( len(shot_list_train), shot_list_train.num_disruptive())) + g.print_unique('validate: {} shots, {} disruptive'.format( + len(shot_list_validate), shot_list_validate.num_disruptive())) g.print_unique('testing: {} shots, {} disruptive'.format( len(shot_list_test), shot_list_test.num_disruptive())) g.print_unique("...done") From df7228ea17915ffb5d7c34f7449b421ab3768861 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 21 Nov 2019 14:40:33 -0500 Subject: [PATCH 179/272] Encapsulate printing of shot set sizes in new fn, in new file diagnostics.py --- plasma/models/runner.py | 6 ++---- plasma/models/shallow_runner.py | 2 ++ plasma/models/torch_runner.py | 6 ++---- plasma/preprocessor/preprocess.py | 16 ++++------------ plasma/utils/diagnostics.py | 28 ++++++++++++++++++++++++++++ 5 files changed, 38 insertions(+), 20 deletions(-) create mode 100644 plasma/utils/diagnostics.py diff --git a/plasma/models/runner.py b/plasma/models/runner.py index 29c42c3c..c2008817 100644 --- a/plasma/models/runner.py +++ b/plasma/models/runner.py @@ -1,6 +1,7 @@ from plasma.utils.state_reset import reset_states from plasma.utils.evaluation import get_loss_from_list from plasma.utils.performance import PerformanceAnalyzer +from plasma.utils.diagnostics import print_shot_list_sizes from plasma.models.loader import Loader, ProcessGenerator from plasma.conf import conf import pathos.multiprocessing as mp @@ -29,10 +30,7 @@ def train(conf, shot_list_train, shot_list_validate, loader, validation_losses = [] validation_roc = [] training_losses = [] - print('training: {} shots, {} disruptive'.format( - len(shot_list_train), shot_list_train.num_disruptive())) - print('validate: {} shots, {} disruptive'.format( - len(shot_list_validate), shot_list_validate.num_disruptive())) + print_shot_list_sizes(shot_list_train, shot_list_validate) if backend == 'tf' or backend == 'tensorflow': first_time = "tensorflow" not in sys.modules diff --git a/plasma/models/shallow_runner.py b/plasma/models/shallow_runner.py index cdfe2c32..0ea4962b 100644 --- a/plasma/models/shallow_runner.py +++ b/plasma/models/shallow_runner.py @@ -10,6 +10,7 @@ # from plasma.utils.state_reset import reset_states from plasma.utils.evaluation import get_loss_from_list from plasma.utils.performance import PerformanceAnalyzer +from plasma.utils.diagnostics import print_shot_list_sizes # from plasma.models.loader import Loader, ProcessGenerator # from plasma.conf import conf from sklearn.neural_network import MLPClassifier @@ -324,6 +325,7 @@ def build_callbacks(conf): def train(conf, shot_list_train, shot_list_validate, loader, shot_list_test=None): np.random.seed(1) + print_shot_list_sizes(shot_list_train, shot_list_validate) print('training: {} shots, {} disruptive'.format( len(shot_list_train), shot_list_train.num_disruptive())) diff --git a/plasma/models/torch_runner.py b/plasma/models/torch_runner.py index a4fde559..195b5275 100644 --- a/plasma/models/torch_runner.py +++ b/plasma/models/torch_runner.py @@ -5,6 +5,7 @@ from torch.autograd import Variable import torch.nn as nn import torch +from plasma.utils.diagnostics import print_shot_list_sizes from plasma.utils.downloading import makedirs_process_safe from plasma.utils.performance import PerformanceAnalyzer from plasma.utils.evaluation import get_loss_from_list @@ -414,10 +415,7 @@ def train(conf, shot_list_train, shot_list_validate, loader): data_gen = partial( loader.training_batch_generator_full_shot_partial_reset, shot_list=shot_list_train)() - print('training: {} shots, {} disruptive'.format( - len(shot_list_train), shot_list_train.num_disruptive())) - print('validate: {} shots, {} disruptive'.format( - len(shot_list_validate), shot_list_validate.num_disruptive())) + print_shot_list_sizes(shot_list_train, shot_list_validate) loader.set_inference_mode(False) train_model = build_torch_model(conf) diff --git a/plasma/preprocessor/preprocess.py b/plasma/preprocessor/preprocess.py index deaccee3..0e649c2d 100644 --- a/plasma/preprocessor/preprocess.py +++ b/plasma/preprocessor/preprocess.py @@ -1,6 +1,6 @@ ''' ######################################################### -This file containts classes to handle data processing +This file contains classes to handle data processing Author: Julian Kates-Harbeck, jkatesharbeck@g.harvard.edu @@ -19,6 +19,7 @@ import pathos.multiprocessing as mp from plasma.utils.processing import append_to_filename +from plasma.utils.diagnostics import print_shot_list_sizes from plasma.primitives.shots import ShotList from plasma.utils.downloading import mkdirdepth @@ -257,16 +258,7 @@ def guarantee_preprocessed(conf, verbose=False): shot_list_train, shot_list_validate, shot_list_test = apply_bleed_in( conf, shot_list_train, shot_list_validate, shot_list_test) if verbose: - g.print_unique('total: {} shots, {} disruptive'.format( - len(shot_list_validate)+len(shot_list_train)+len(shot_list_test), - shot_list_validate.num_disruptive() - + shot_list_train.num_disruptive() - + shot_list_test.num_disruptive())) - g.print_unique('training: {} shots, {} disruptive'.format( - len(shot_list_train), shot_list_train.num_disruptive())) - g.print_unique('validate: {} shots, {} disruptive'.format( - len(shot_list_validate), shot_list_validate.num_disruptive())) - g.print_unique('testing: {} shots, {} disruptive'.format( - len(shot_list_test), shot_list_test.num_disruptive())) + print_shot_list_sizes(shot_list_train, shot_list_validate, + shot_list_test) g.print_unique("...done") return shot_list_train, shot_list_validate, shot_list_test diff --git a/plasma/utils/diagnostics.py b/plasma/utils/diagnostics.py new file mode 100644 index 00000000..6f632887 --- /dev/null +++ b/plasma/utils/diagnostics.py @@ -0,0 +1,28 @@ +''' +######################################################### +This file contains fns for printing diagnostic messages +######################################################### +''' + +from __future__ import print_function +import plasma.global_vars as g + + +def print_shot_list_sizes(shot_list_train, shot_list_validate, + shot_list_test=None): + nshots = len(shot_list_train) + len(shot_list_validate) + nshots_disrupt = (shot_list_train.num_disruptive() + + shot_list_validate.num_disruptive()) + if shot_list_test is not None: + nshots += len(shot_list_test) + nshots_disrupt += shot_list_test.num_disruptive() + g.print_unique('total: {} shots, {} disruptive'.format(nshots, + nshots_disrupt) + g.print_unique('training: {} shots, {} disruptive'.format( + len(shot_list_train), shot_list_train.num_disruptive())) + g.print_unique('validate: {} shots, {} disruptive'.format( + len(shot_list_validate), shot_list_validate.num_disruptive())) + if shot_list_test is not None: + g.print_unique('testing: {} shots, {} disruptive'.format( + len(shot_list_test), shot_list_test.num_disruptive())) + return From cbc31b42aac55e1d2fce646925176a309e1ccb5f Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 21 Nov 2019 14:42:02 -0500 Subject: [PATCH 180/272] Add missing bracket --- plasma/utils/diagnostics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plasma/utils/diagnostics.py b/plasma/utils/diagnostics.py index 6f632887..d38d6e78 100644 --- a/plasma/utils/diagnostics.py +++ b/plasma/utils/diagnostics.py @@ -17,7 +17,7 @@ def print_shot_list_sizes(shot_list_train, shot_list_validate, nshots += len(shot_list_test) nshots_disrupt += shot_list_test.num_disruptive() g.print_unique('total: {} shots, {} disruptive'.format(nshots, - nshots_disrupt) + nshots_disrupt)) g.print_unique('training: {} shots, {} disruptive'.format( len(shot_list_train), shot_list_train.num_disruptive())) g.print_unique('validate: {} shots, {} disruptive'.format( From 366de2d8084e3bd5386c417790e5f4c2a143f98c Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 21 Nov 2019 14:11:17 -0600 Subject: [PATCH 181/272] Comment out unused fn --- plasma/utils/processing.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/plasma/utils/processing.py b/plasma/utils/processing.py index d6995bcb..12938c1b 100644 --- a/plasma/utils/processing.py +++ b/plasma/utils/processing.py @@ -90,15 +90,15 @@ def train_test_split_robust(x, frac, do_shuffle=False): return train, test -def train_test_split_all(x, frac, do_shuffle=True): - groups = [] - length = len(x[0]) - mask = np.array(range(length)) < frac*length - if do_shuffle: - np.random.shuffle(mask) - for item in x: - groups.append((item[mask], item[~mask])) - return groups +# def train_test_split_all(x, frac, do_shuffle=True): +# groups = [] +# length = len(x[0]) +# mask = np.array(range(length)) < frac*length +# if do_shuffle: +# np.random.shuffle(mask) +# for item in x: +# groups.append((item[mask], item[~mask])) +# return groups def concatenate_sublists(superlist): From e527c38d215610ce714135d8bb0d050ce8c13fbb Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 21 Nov 2019 15:21:11 -0500 Subject: [PATCH 182/272] Add details about how many omitted shots were disruptive to preprocess.py --- plasma/preprocessor/preprocess.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/plasma/preprocessor/preprocess.py b/plasma/preprocessor/preprocess.py index 0e649c2d..871a27fc 100644 --- a/plasma/preprocessor/preprocess.py +++ b/plasma/preprocessor/preprocess.py @@ -101,12 +101,16 @@ def preprocess_from_files(self, shot_files, use_shots): pool.close() pool.join() - print('Finished Preprocessing {} files in {} seconds'.format( + print('Finished preprocessing {} files in {} seconds'.format( len(shot_list_picked), time.time() - start_time)) + print('Using {}/{} disruptive shots'.format( + used_shots.num_disruptive(), len(used_shots))) print('Omitted {} shots of {} total.'.format( len(shot_list_picked) - len(used_shots), len(shot_list_picked))) - print('{}/{} disruptive shots'.format(used_shots.num_disruptive(), - len(used_shots))) + print('Omitted {} disruptive shots of {} total disruptive.'.format( + shot_list_picked.num_disruptive() - used_shots.num_disruptive, + shot_list_picked.num_disruptive())) + if len(used_shots) == 0: print("WARNING: All shots were omitted, please ensure raw data " " is complete and available at {}.".format( From 286dc8793feda0bd58ddd9c0cf7d2fc37ae7ed08 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 21 Nov 2019 15:32:28 -0500 Subject: [PATCH 183/272] Typo --- plasma/preprocessor/preprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plasma/preprocessor/preprocess.py b/plasma/preprocessor/preprocess.py index 871a27fc..eaf99604 100644 --- a/plasma/preprocessor/preprocess.py +++ b/plasma/preprocessor/preprocess.py @@ -108,7 +108,7 @@ def preprocess_from_files(self, shot_files, use_shots): print('Omitted {} shots of {} total.'.format( len(shot_list_picked) - len(used_shots), len(shot_list_picked))) print('Omitted {} disruptive shots of {} total disruptive.'.format( - shot_list_picked.num_disruptive() - used_shots.num_disruptive, + shot_list_picked.num_disruptive() - used_shots.num_disruptive(), shot_list_picked.num_disruptive())) if len(used_shots) == 0: From 300966252320fd792fc7616bc5962bdf69270139 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 21 Nov 2019 15:52:43 -0600 Subject: [PATCH 184/272] Remove stray diagnostic print --- plasma/primitives/shots.py | 1 - 1 file changed, 1 deletion(-) diff --git a/plasma/primitives/shots.py b/plasma/primitives/shots.py index cafb76ed..a9e95df9 100644 --- a/plasma/primitives/shots.py +++ b/plasma/primitives/shots.py @@ -121,7 +121,6 @@ def split_train_test(self, conf): shot_numbers_train = [shot.number for shot in shot_list_train] shot_numbers_test = [shot.number for shot in shot_list_test] - print(len(shot_numbers_train), len(shot_numbers_test)) # make sure we only use pre-filtered valid shots shots_train = self.filter_by_number(shot_numbers_train) shots_test = self.filter_by_number(shot_numbers_test) From 0e4715b78e5e73bb0de220c1e7818a0648d1ed47 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Thu, 21 Nov 2019 17:10:36 -0500 Subject: [PATCH 185/272] Reformat diagnostics --- plasma/preprocessor/preprocess.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/plasma/preprocessor/preprocess.py b/plasma/preprocessor/preprocess.py index eaf99604..18cf3114 100644 --- a/plasma/preprocessor/preprocess.py +++ b/plasma/preprocessor/preprocess.py @@ -101,15 +101,17 @@ def preprocess_from_files(self, shot_files, use_shots): pool.close() pool.join() - print('Finished preprocessing {} files in {} seconds'.format( + print('\nFinished preprocessing {} files in {} seconds'.format( len(shot_list_picked), time.time() - start_time)) - print('Using {}/{} disruptive shots'.format( - used_shots.num_disruptive(), len(used_shots))) - print('Omitted {} shots of {} total.'.format( + print('Using {} shots ({} disruptive shots)'.format( + len(used_shots), used_shots.num_disruptive())) + print('Omitted {} shots of {} total shots'.format( len(shot_list_picked) - len(used_shots), len(shot_list_picked))) - print('Omitted {} disruptive shots of {} total disruptive.'.format( - shot_list_picked.num_disruptive() - used_shots.num_disruptive(), - shot_list_picked.num_disruptive())) + print( + 'Omitted {} disruptive shots of {} total disruptive shots'.format( + shot_list_picked.num_disruptive() + - used_shots.num_disruptive(), + shot_list_picked.num_disruptive())) if len(used_shots) == 0: print("WARNING: All shots were omitted, please ensure raw data " From a99c01acf94d71a75d4249ba20a8d2cdbee94688 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Fri, 22 Nov 2019 10:25:44 -0600 Subject: [PATCH 186/272] Reduce number of lines --- plasma/primitives/data.py | 16 ++++------- plasma/primitives/shots.py | 57 +++++++++++++------------------------- 2 files changed, 25 insertions(+), 48 deletions(-) diff --git a/plasma/primitives/data.py b/plasma/primitives/data.py index 63d30abc..5d84fb0b 100644 --- a/plasma/primitives/data.py +++ b/plasma/primitives/data.py @@ -5,7 +5,6 @@ import re from scipy.interpolate import UnivariateSpline - from plasma.utils.processing import get_individual_shot_file from plasma.utils.downloading import get_missing_value_array from plasma.utils.hashing import myhash @@ -123,17 +122,14 @@ def load_data(self, prepath, shot, dtype='float32'): if self.is_ip: print('shot {} has no current'.format(shot.number)) else: - print( - 'Signal {}, shot {} contains no data'.format( - self.description, shot.number)) + print('Signal {}, shot {} contains no data'.format( + self.description, shot.number)) return None, None, False # make sure data doesn't contain nan if np.any(np.isnan(t)) or np.any(np.isnan(sig)): - print( - 'Signal {}, shot {} contains NAN'.format( - self.description, - shot.number)) + print('Signal {}, shot {} contains NAN'.format( + self.description, shot.number)) return None, None, False return t, sig, True @@ -278,8 +274,8 @@ def load_data(self, prepath, shot, dtype='float32'): return t, sig_interp, True def fetch_data(self, machine, shot_num, c): - time, data, mapping, success = self.fetch_data_basic(machine, shot_num, - c) + time, data, mapping, success = self.fetch_data_basic( + machine, shot_num, c) path = self.get_path(machine) mapping_path = self.get_mapping_path(machine) diff --git a/plasma/primitives/shots.py b/plasma/primitives/shots.py index a9e95df9..3079b8b2 100644 --- a/plasma/primitives/shots.py +++ b/plasma/primitives/shots.py @@ -13,7 +13,6 @@ import os.path import sys import random as rnd - import numpy as np from plasma.utils.processing import train_test_split, cut_and_resample_signal @@ -36,11 +35,9 @@ def __repr__(self): return self.__str__() def get_single_shot_numbers_and_disruption_times(self, full_path): - data = np.loadtxt( - full_path, ndmin=1, dtype={ - 'names': ( - 'num', 'disrupt_times'), 'formats': ( - 'i4', 'f4')}) + data = np.loadtxt(full_path, ndmin=1, + dtype={'names': ('num', 'disrupt_times'), + 'formats': ('i4', 'f4')}) shots = np.array(list(zip(*data))[0]) disrupt_times = np.array(list(zip(*data))[1]) return shots, disrupt_times @@ -77,21 +74,18 @@ def __init__(self, shots=None): assert(all([isinstance(shot, Shot) for shot in shots])) self.shots = [shot for shot in shots] - def load_from_shot_list_files_object( - self, shot_list_files_object, signals): + def load_from_shot_list_files_object(self, shot_list_files_object, + signals): machine = shot_list_files_object.machine shot_numbers, disruption_times = ( shot_list_files_object.get_shot_numbers_and_disruption_times()) for number, t in list(zip(shot_numbers, disruption_times)): - self.append( - Shot(number=number, t_disrupt=t, machine=machine, - signals=[s for s in signals if - s.is_defined_on_machine(machine)] - ) - ) - - def load_from_shot_list_files_objects( - self, shot_list_files_objects, signals): + self.append(Shot(number=number, t_disrupt=t, machine=machine, + signals=[s for s in signals if + s.is_defined_on_machine(machine)])) + + def load_from_shot_list_files_objects(self, shot_list_files_objects, + signals): for obj in shot_list_files_objects: self.load_from_shot_list_files_object(obj, signals) @@ -276,16 +270,9 @@ class Shot(object): property. ''' - def __init__( - self, - number=None, - machine=None, - signals=None, - signals_dict=None, - ttd=None, - valid=None, - is_disruptive=None, - t_disrupt=None): + def __init__(self, number=None, machine=None, signals=None, + signals_dict=None, ttd=None, valid=None, is_disruptive=None, + t_disrupt=None): ''' Shot objects contain following attributes: @@ -415,8 +402,7 @@ def get_signals_and_times_from_file(self, conf): if self.is_disruptive and self.t_disrupt > np.max(t): t_max_total = ( np.max(t) + signal.get_data_avail_tolerance( - self.machine) - ) + self.machine)) if (self.t_disrupt > t_max_total): print('Shot {}: disruption event '.format(self.number), 'is not contained in valid time region of ', @@ -425,8 +411,8 @@ def get_signals_and_times_from_file(self, conf): self.t_disrupt - np.max(t))) valid = False else: - t_max = np.max( - t) + signal.get_data_avail_tolerance(self.machine) + t_max = np.max(t) + signal.get_data_avail_tolerance( + self.machine) else: t_max = min(t_max, np.max(t)) @@ -449,13 +435,8 @@ def get_signals_and_times_from_file(self, conf): return time_arrays, signal_arrays, t_min, t_max, valid - def cut_and_resample_signals( - self, - time_arrays, - signal_arrays, - t_min, - t_max, - conf): + def cut_and_resample_signals(self, time_arrays, signal_arrays, t_min, + t_max, conf): dt = conf['data']['dt'] signals_dict = dict() From 31fafa34e0e4aeafc5ad04e40c7d5e61ecc65892 Mon Sep 17 00:00:00 2001 From: Kyle Gerard Felker Date: Fri, 22 Nov 2019 11:35:43 -0600 Subject: [PATCH 187/272] Drop Rick Zamora's ALCF notes into docs/ --- docs/ALCF.md | 366 +++++++++++++++++++++++++++++++++++++++++++++ examples/conf.yaml | 6 +- 2 files changed, 369 insertions(+), 3 deletions(-) create mode 100644 docs/ALCF.md diff --git a/docs/ALCF.md b/docs/ALCF.md new file mode 100644 index 00000000..63877423 --- /dev/null +++ b/docs/ALCF.md @@ -0,0 +1,366 @@ +# ALCF Theta `plasma-python` FRNN Notes + +**Author: Rick Zamora (rzamora@anl.gov)** + +This document is intended to act as a tutorial for running the [plasma-python](https://github.com/PPPLDeepLearning/plasma-python) implementation of the Fusion recurrent neural network (FRNN) on the ALCF Theta supercomputer (Cray XC40; Intel KNL processors). The steps followed in these notes are based on the Princeton [Tiger-GPU tutorial](https://github.com/PPPLDeepLearning/plasma-python/blob/master/docs/PrincetonUTutorial.md#location-of-the-data-on-tigress), hosted within the main GitHub repository for the project. + +## Environment Setup + + +Choose a *root* directory for FRNN-related installations on Theta: + +``` +export FRNN_ROOT= +cd $FRNN_ROOT +``` + +*Personal Note: Using FRNN_ROOT=/home/zamora/ESP* + +Create a simple directory structure allowing experimental *builds* of the `plasma-python` python code/library: + +``` +mkdir build +mkdir build/miniconda-3.6-4.5.4 +cd build/miniconda-3.6-4.5.4 +``` + +### Custom Miniconda Environment Setup + +Copy miniconda installation script to working directory (and install): + +``` +cp /lus/theta-fs0/projects/fusiondl_aesp/FRNN/rzamora/scripts/install_miniconda-3.6-4.5.4.sh . +./install_miniconda-3.6-4.5.4.sh +``` + +The `install_miniconda-3.6-4.5.4.sh` script will install `miniconda-4.5.4` (using `Python-3.6`), as well as `Tensorflow-1.12.0` and `Keras 2.2.4`. + + +Update your environment variables to use miniconda: + +``` +export PATH=${FRNN_ROOT}/build/miniconda-3.6-4.5.4/miniconda3/4.5.4/bin:$PATH +export PYTHONPATH=${FRNN_ROOT}/build/miniconda-3.6-4.5.4/miniconda3/4.5.4/lib/python3.6/site-packages/:$PYTHONPATH +``` + +Note that the previous lines (as well as the definition of `FRNN_ROOT`) can be appended to your `$HOME/.bashrc` file if you want to use this environment on Theta by default. + + +## Installing `plasma-python` + +Here, we assume the installation is within the custom miniconda environment installed in the previous steps. We also assume the following commands have already been executed: + +``` +export FRNN_ROOT= +export PATH=${FRNN_ROOT}/build/miniconda-3.6-4.5.4/miniconda3/4.5.4/bin:$PATH +export PYTHONPATH=${FRNN_ROOT}/build/miniconda-3.6-4.5.4/miniconda3/4.5.4/lib/python3.6/site-packages/:$PYTHONPATH +``` + +*Personal Note: Using `export FRNN_ROOT=/lus/theta-fs0/projects/fusiondl_aesp/zamora/FRNN_project`* + +If the environment is set up correctly, installation of `plasma-python` is straightforward: + +``` +cd ${FRNN_ROOT}/build/miniconda-3.6-4.5.4 +git clone https://github.com/PPPLDeepLearning/plasma-python.git +cd plasma-python +python setup.py build +python setup.py install +``` + +## Data Access + +Sample data and metadata is available in `/lus/theta-fs0/projects/FRNN/tigress/alexeys/signal_data` and `/lus/theta-fs0/projects/FRNN/tigress/alexeys/shot_lists`, respectively. It is recommended that users create their own symbolic links to these directories. I recommend that you do this within a directory called `/lus/theta-fs0/projects/fusiondl_aesp//`. For example: + +``` +ln -s /lus/theta-fs0/projects/fusiondl_aesp/FRNN/tigress/alexeys/shot_lists  /lus/theta-fs0/projects/fusiondl_aesp//shot_lists +ln -s /lus/theta-fs0/projects/fusiondl_aesp/FRNN/tigress/alexeys/signal_data  /lus/theta-fs0/projects/fusiondl_aesp//signal_data +``` + +For the examples included in `plasma-python`, there is a configuration file that specifies the root directory of the raw data. Change the `fs_path: '/tigress'` line in `examples/conf.yaml` to reflect the following: + +``` +fs_path: '/lus/theta-fs0/projects/fusiondl_aesp' +``` + +Its also a good idea to change `num_gpus: 4` to `num_gpus: 1`. I am also using the `jet_data_0D` dataset: + +``` +paths: + data: jet_data_0D +``` + + +### Data Preprocessing + +#### The SLOW Way (On Theta) + +Theta is KNL-based, and is **not** the best resource for processing many text files in python. However, the preprocessing step *can* be used by using the following steps (although it may need to be repeated many times to get through the whole dataset in a 60-minute debug queues): + +``` +cd ${FRNN_ROOT}/build/miniconda-3.6-4.5.4/plasma-python/examples +cp /lus/theta-fs0/projects/fusiondl_aesp/FRNN/rzamora/scripts/submit_guarantee_preprocessed.sh . +``` + +Modify the paths defined in `submit_guarantee_preprocessed.sh` to match your environment. + +Note that the preprocessing module will use Pathos multiprocessing (not MPI/mpi4py). Therefore, the script will see every compute core (all 256 per node) as an available resource. Since the LUSTRE file system is unlikely to perform well with 256 processes (on the same node) opening/closing/creating files at once, it might improve performance if you make a slight change to line 85 in the `vi ~/plasma-python/plasma/preprocessor/preprocess.py` file: + +``` +line 85: use_cores = min( , max(1,mp.cpu_count()-2) ) +``` + +After optionally re-building and installing plasm-python with this change, submit the preprocessing job: + +``` +qsub submit_guarantee_preprocessed.sh +``` + +#### The FAST Way (On Cooley) + +You will fine it much less painful to preprocess the data on Cooley, because the Haswell processors are much better suited for this... Log onto the ALCF Cooley Machine: + +``` +ssh @cooley.alcf.anl.gov +``` + +Copy my `cooley_preprocess` example directory to whatever directory you choose to work in: + +``` +cp -r /lus/theta-fs0/projects/fusiondl_aesp/FRNN/rzamora/scripts/cooley_preprocess . +cd cooley_preprocess +``` + +This directory has a Singularity image with everything you need to run your code on Cooley. Assuming you have created symbolic links to the `shot_lists` and `signal_data` directories in `/lus/theta-fs0/projects/fusiondl_aesp//`, you can just submit the included `COBALT` script (to specify the data you want to process, just modify the included `conf.yaml` file): + +``` +qsub submit.sh +``` + +For me, this finishes in less than 10 minutes, and creates 5523 `.npz` files in the `/lus/theta-fs0/projects/fusiondl_aesp//processed_shots/` directory. The output file of the COBALT submission ends with the following message: + +``` +5522/5523Finished Preprocessing 5523 files in 406.94421911239624 seconds +Omitted 5523 shots of 5523 total. +0/0 disruptive shots +WARNING: All shots were omitted, please ensure raw data is complete and available at /lus/theta-fs0/projects/fusiondl_aesp/zamora/signal_data/. +4327 1196 +``` + + +# Notes on Revisiting Pre-Processes + +## Preprocessing Information + +To understand what might be going wrong with the preprocessing step, let's investigate what the code is actually doing. + +**Step 1** Call `guarentee_preprocessed( conf )`, which is defined in `plasma/preprocessor/preprocess.py`. This function first initializes a `Preprocessor()` object (whose class definition is in the same file), and then checks if the preprocessing was already done (by looking for a file). The preprocessor object is called `pp`. + +**Step 2** Assuming preprocessing is needed, we call `pp.clean_shot_lists()`, which loops through each file in the `shot_lists` directory and calls `self.clean_shot_list()` (not plural) for each text-file item. I do not believe this function is doing any thing when I run it, because all the shot list files have been "cleaned." The cleaning of a shot-list file just means the data is corrected to have two columns, and the file is renamed (to have "clear" in the name). + +**Step 3** We call `pp.preprocess_all()`, which parses some of the config file, and ultimately calls `self.preprocess_from_files(shot_files_all,use_shots)` (where I believe `shot_files_all` is the output directory, and `use_shots` is the number of shots to use). + +**Step 4** The `preprocess_from_files()` function is used to do the actual preprocessing. It does this by creating a multiprocessing pool, and mapping the processes to the `self.preprocess_single_file` function (note that the code for `ShotList` class is in `plasma/primitives/shots.py`, and the preprocessing code is still in `plasma/preprocessor/preprocess.py`). + +**Important:** It looks like the code uses the path definitions in `data/shot_lists/signals.py` to define the location/path of signal data. I believe that some of the signal data is missing, which is causing every "shot" to be labeled as incomplete (and consequently thrown out). + +### Possible Issues + +From the preprocessing output, it is clear that the *Signal Radiated Power Core* data was not downloaded correctly. According to the `data/shot_lists/signals.py` file, the data *should* be in `/lus/theta-fs0/projects/fusiondl_aesp//signal_data/jet/ppf/bolo/kb5h/channel14`. However, the only subdirectory of `~/jet/ppf/` is `~/jet/ppf/efit` + +Another possible issue is that the `data/shot_lists/signals.py` file specifies the **name** of the directory containing the *Radiated Power* data incorrectly (*I THINK*). Instead of the following line: + +`pradtot = Signal("Radiated Power",['jpf/db/b5r-ptot>out'],[jet])` + +We might need this: + +`pradtot = Signal("Radiated Power",['jpf/db/b5r-ptot\>out'],[jet])` + +The issue has to do with the `>` character in the directory name (without the proper `\` escape character, python may be looking in the wrong path). **NOTE: I need to confirm that there is actually an issue with the way the code is actually using the string.** + + +## Singularity/Docker Notes + +Recall that the data preprocessing step was PAINFULLY slow on Theta, and so I decided to use Cooley. To simplify the process of using Cooley, I created a Docker image with the necessary environment. **Personal Note:** I performed this work on my local machine (Mac) in `/Users/rzamora/container-recipes`. + + +In order to use a Docker image within a Singularity container (required on ALCF machines), it is useful to build the image on your local machine and push it to "Docker Hub": + + +**Step 1:** Install Docker if you don't have it. [Docker-Mac](https://www.docker.com/docker-mac) works well for Mac. + +**Step 2:** Build a Docker image using the recipe discussed below. + +``` +export IMAGENAME="test_image" +export RECIPENAME="Docker.centos7-cuda-tf1.12.0" +docker build -t $IMAGENAME -f $RECIPENAME . +``` + +You can check that the image is functional by starting an interactive shell session, and checking that the necessary python modules are available. For example (using `-it` for an interactive session): + +``` +docker run --rm -it -v $PWD:/tmp -w /tmp $IMAGENAME:latest bash +# python -c "import keras; import plasma; print(plasma.__file__)" +``` + +Note that the `plasma-python` source code will be located in `/root/plasma-python/` for the recipe described below. + +**Step 3:** Push the image to [Docker Hub](https://hub.docker.com/). + +Using your docker-hub username: + +``` +docker login --username= +``` + +Then, "tag" the image using the `IMAGE ID` value displayed with `docker image ls`: + +``` +docker tag /: