From f010521197400e425fb6b38ab804d7fde7636944 Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Wed, 7 Nov 2018 19:18:11 +0800 Subject: [PATCH 001/173] Add requirement.txt --- requirement.txt | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 requirement.txt diff --git a/requirement.txt b/requirement.txt new file mode 100644 index 0000000..3bb0756 --- /dev/null +++ b/requirement.txt @@ -0,0 +1,6 @@ +docopt +hdfs +scipy +sklearn +pandas +mpi4py From 874a41170926cdad2eb0a59725656eb2c8d91479 Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Wed, 7 Nov 2018 19:27:30 +0800 Subject: [PATCH 002/173] change requirement.txt --- requirement.txt | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/requirement.txt b/requirement.txt index 3bb0756..a24f3ed 100644 --- a/requirement.txt +++ b/requirement.txt @@ -1,6 +1,7 @@ -docopt -hdfs -scipy -sklearn -pandas -mpi4py +docopt>=0.6.2 +hdfs>=2.1.0 +scipy>=1.0.0 +sklearn>=0.19.1 +pandas>=0.22.0 +mpi4py>=3.0.0 +tensorflow>=1.10.0 From 99e153e66694e3aa3b4460e6804f18c18e1c5613 Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Wed, 7 Nov 2018 19:31:37 +0800 Subject: [PATCH 003/173] Add numpy to requirement.txt --- requirement.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirement.txt b/requirement.txt index a24f3ed..2fa693c 100644 --- a/requirement.txt +++ b/requirement.txt @@ -1,5 +1,6 @@ docopt>=0.6.2 hdfs>=2.1.0 +numpy>=1.14.0 scipy>=1.0.0 sklearn>=0.19.1 pandas>=0.22.0 From bc0f82697062c6ada6636feb297eba1f6f748bd6 Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Fri, 9 Nov 2018 15:52:49 +0800 Subject: [PATCH 004/173] make channel pruned modle be able to convert to tflite --- learners/channel_pruning/learner.py | 35 ++++++++--------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/learners/channel_pruning/learner.py b/learners/channel_pruning/learner.py index f44f88e..f1e7d6b 100644 --- a/learners/channel_pruning/learner.py +++ b/learners/channel_pruning/learner.py @@ -49,6 +49,10 @@ 'cp_prune_list_file', 'ratio.list', 'the prune list file which contains the compression ratio of each convolution layers') +tf.app.flags.DEFINE_string( + 'cp_channel_pruned_path', + './models/pruned_model.ckpt', + 'channel pruned model\'s save path') tf.app.flags.DEFINE_string( 'cp_best_path', './models/best_model.ckpt', @@ -116,30 +120,6 @@ def __init__(self, sm_writer, model_helper): self.__build(is_train=True) self.__build(is_train=False) - channel_pruned_path = './models/pruned_model.ckpt' - best_model_path = './models/best_model.ckpt' - if FLAGS.enbl_multi_gpu: - self.parent_path = '' - if self.mpi_comm.rank == 0: - self.parent_path = '/opt/ml/disk/' + \ - ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) - pathlib.Path(self.parent_path).mkdir(parents=True, exist_ok=True) - channel_pruned_path = self.parent_path + '/' + channel_pruned_path - best_model_path = self.parent_path + '/' + best_model_path - - channel_pruned_path = self.mpi_comm.bcast(channel_pruned_path, root=0) - best_model_path = self.mpi_comm.bcast(best_model_path, root=0) - self.parent_path = self.mpi_comm.bcast(self.parent_path, root=0) - - tf.app.flags.DEFINE_string( - 'cp_channel_pruned_path', - channel_pruned_path, - 'channel pruned model\'s save path') - tf.app.flags.DEFINE_string( - 'cp_best_model_path', - best_model_path, - 'channel best model\'s save path') - def train(self): """Train the pruned model""" # download pre-trained model @@ -312,7 +292,9 @@ def __build_pruned_evaluate_model(self, path=None): self.saver_eval = tf.train.import_meta_graph(path + '.meta') self.saver_eval.restore(self.sess_eval, path) eval_logits = tf.get_collection('logits')[0] + tf.add_to_collection('logits_final', eval_logits) eval_images = tf.get_collection('eval_images')[0] + tf.add_to_collection('images_final', eval_images) eval_labels = tf.get_collection('eval_labels')[0] mem_images = tf.get_collection('mem_images')[0] mem_labels = tf.get_collection('mem_labels')[0] @@ -497,7 +479,7 @@ def __save_best_pruned_model(self): def __save_in_progress_pruned_model(self): """ save a in progress training model with a max evaluation result""" - self.max_save_path = self.saver_eval.save(self.sess_eval, FLAGS.cp_best_model_path) + self.max_save_path = self.saver_eval.save(self.sess_eval, FLAGS.cp_best_path) tf.logging.info('model saved best model to ' + self.max_save_path) def __save_model(self): @@ -712,7 +694,8 @@ def __prune_rl(self): # pylint: disable=too-many-locals accuracy: {} and pruned ratio: {}""".format(self.bestinfo[0], self.bestinfo[1], self.bestinfo[2])) with self.pruner.model.g.as_default(): - self.__save_best_pruned_model() + self.__save_in_progress_pruned_model() + #self.__save_best_pruned_model() tf.logging.info('automatic channl pruning time cost: {}s'.format(timer() - start)) From 1cd3e9f44a48c01a951490e4c001f6a4e4094984 Mon Sep 17 00:00:00 2001 From: Jiaxiang Wu Date: Mon, 12 Nov 2018 14:25:31 +0800 Subject: [PATCH 005/173] Add source files for documentations (#40) * add source files for documentation * add Python package dependencies * add README.md --- README.md | 2 +- docs/README.md | 13 ++ docs/doc-requirements.txt | 4 + docs/docs/MathJax.js | 53 +++++ docs/docs/automl_based_methods.md | 3 + docs/docs/cp_learner.md | 69 +++++++ docs/docs/dcp_learner.md | 85 ++++++++ docs/docs/distillation.md | 28 +++ docs/docs/faq.md | 5 + docs/docs/index.md | 73 +++++++ docs/docs/installation.md | 98 +++++++++ docs/docs/multi_gpu_training.md | 111 ++++++++++ docs/docs/nuq_learner.md | 109 ++++++++++ docs/docs/performance.md | 46 +++++ docs/docs/pics/dcp_learner.png | Bin 0 -> 102436 bytes docs/docs/pics/deep_compression_algor.png | Bin 0 -> 111446 bytes docs/{ => docs/pics}/framework_design.png | Bin docs/docs/pics/rl_workflow.png | Bin 0 -> 157308 bytes docs/docs/pics/train_n_inference.png | Bin 0 -> 59205 bytes docs/docs/pics/wsl_pr_schedule.png | Bin 0 -> 89327 bytes docs/docs/pre_trained_models.md | 23 +++ docs/docs/reference.md | 12 ++ docs/docs/reinforcement_learning.md | 50 +++++ docs/docs/test_cases.md | 163 +++++++++++++++ docs/docs/tutorial.md | 241 ++++++++++++++++++++++ docs/docs/uq_learner.md | 167 +++++++++++++++ docs/docs/ws_learner.md | 96 +++++++++ docs/mkdocs.yml | 30 +++ 28 files changed, 1480 insertions(+), 1 deletion(-) create mode 100644 docs/README.md create mode 100644 docs/doc-requirements.txt create mode 100644 docs/docs/MathJax.js create mode 100644 docs/docs/automl_based_methods.md create mode 100644 docs/docs/cp_learner.md create mode 100644 docs/docs/dcp_learner.md create mode 100644 docs/docs/distillation.md create mode 100644 docs/docs/faq.md create mode 100644 docs/docs/index.md create mode 100644 docs/docs/installation.md create mode 100644 docs/docs/multi_gpu_training.md create mode 100644 docs/docs/nuq_learner.md create mode 100644 docs/docs/performance.md create mode 100644 docs/docs/pics/dcp_learner.png create mode 100644 docs/docs/pics/deep_compression_algor.png rename docs/{ => docs/pics}/framework_design.png (100%) create mode 100644 docs/docs/pics/rl_workflow.png create mode 100644 docs/docs/pics/train_n_inference.png create mode 100644 docs/docs/pics/wsl_pr_schedule.png create mode 100644 docs/docs/pre_trained_models.md create mode 100644 docs/docs/reference.md create mode 100644 docs/docs/reinforcement_learning.md create mode 100644 docs/docs/test_cases.md create mode 100644 docs/docs/tutorial.md create mode 100644 docs/docs/uq_learner.md create mode 100644 docs/docs/ws_learner.md create mode 100644 docs/mkdocs.yml diff --git a/README.md b/README.md index c6a1bac..2de0582 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ For general discussions about PocketFlow development and directions please refer The proposed framework mainly consists of two categories of algorithm components, *i.e.* learners and hyper-parameter optimizers, as depicted in the figure below. Given an uncompressed original model, the learner module generates a candidate compressed model using some randomly chosen hyper-parameter combination. The candidate model's accuracy and computation efficiency is then evaluated and used by hyper-parameter optimizer module as the feedback signal to determine the next hyper-parameter combination to be explored by the learner module. After a few iterations, the best one of all the candidate models is output as the final compressed model. -![Framework Design](docs/framework_design.png) +![Framework Design](docs/docs/pics/framework_design.png) ## Learners diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..7330c34 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,13 @@ +# How to Build the Documentation Site + +1. Install mkdocs and other dependencies: + +``` bash +$ pip install -r doc-requirements.txt +``` + +2. Build a directory named "site", which contains all files needed for the documentation website: + +``` bash +$ mkdocs build --clean +``` diff --git a/docs/doc-requirements.txt b/docs/doc-requirements.txt new file mode 100644 index 0000000..aa4ce36 --- /dev/null +++ b/docs/doc-requirements.txt @@ -0,0 +1,4 @@ +mkdocs +markdown +python-markdown-math +pymdown-extensions diff --git a/docs/docs/MathJax.js b/docs/docs/MathJax.js new file mode 100644 index 0000000..35e1994 --- /dev/null +++ b/docs/docs/MathJax.js @@ -0,0 +1,53 @@ +(function () { + var newMathJax = 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js'; + var oldMathJax = 'cdn.mathjax.org/mathjax/latest/MathJax.js'; + + var replaceScript = function (script, src) { + // + // Make redirected script + // + var newScript = document.createElement('script'); + newScript.src = newMathJax + src.replace(/.*?(\?|$)/, '$1'); + // + // Move onload and onerror handlers to new script + // + newScript.onload = script.onload; + newScript.onerror = script.onerror; + script.onload = script.onerror = null; + // + // Move any content (old-style configuration scripts) + // + while (script.firstChild) newScript.appendChild(script.firstChild); + // + // Copy script id + // + if (script.id != null) newScript.id = script.id; + // + // Replace original script with new one + // + script.parentNode.replaceChild(newScript, script); + // + // Issue a console warning + // + console.warn('WARNING: cdn.mathjax.org has been retired. Check https://www.mathjax.org/cdn-shutting-down/ for migration tips.') + } + + if (document.currentScript) { + var script = document.currentScript; + replaceScript(script, script.src); + } else { + // + // Look for current script by searching for one with the right source + // + var n = oldMathJax.length; + var scripts = document.getElementsByTagName('script'); + for (var i = 0; i < scripts.length; i++) { + var script = scripts[i]; + var src = (script.src || '').replace(/.*?:\/\//,''); + if (src.substr(0, n) === oldMathJax) { + replaceScript(script, src); + break; + } + } + } +})(); \ No newline at end of file diff --git a/docs/docs/automl_based_methods.md b/docs/docs/automl_based_methods.md new file mode 100644 index 0000000..2a8672e --- /dev/null +++ b/docs/docs/automl_based_methods.md @@ -0,0 +1,3 @@ +# AutoML-based Methods + +Under construction ... diff --git a/docs/docs/cp_learner.md b/docs/docs/cp_learner.md new file mode 100644 index 0000000..2719fc7 --- /dev/null +++ b/docs/docs/cp_learner.md @@ -0,0 +1,69 @@ +# Channel Pruning + +## Introduction + +Channel pruning is a kind of structural model compression approach which can not only compress the model size, but accelerate the inference speed directly. PocketFlow uses the channel pruning algorithm proposed in (He et al., 2017) to pruning each channel of convolution layers with a certain ratio, and for details please refer to the [channel pruning paper](https://arxiv.org/abs/1707.06168). For better performance and more robust, we modify some parts of the algorithm to achieve better result. + +In order to achieve a better performance, PocketFlow can take advantages of reinforcement learning to search a better compression ratio (He et al., 2018). User can also use the distilling (Hinton et al., 2015) and group tuning function to improve the accuracy after compression. Group tuning means setting a certain number of layers as group and then pruning and finetuning/retraining each group sequentially. For example we can set each 3 layers as a group and then pruning the first 3 layers. After that finetune/retraine the whole model and prune the next 3 layers and so on. Distilling and group tuning are experimentally proved as effective approaches to achieve higher accuracy at a certain compression ratio in most situations. + +## Pruning Option + +The code of channel pruning are located at directory `./learners/channel_pruning`. To use channel pruning. users can set `--learners` to `channel`. The Channel pruning supports 3 kinds of pruning setup by `cp_prune_option` option. + +### Uniform Channel Pruning + +One is the uniform layer pruning, which means the user can set each convolution layer pruned with an uniform pruning ratio by `--cp_prune_option=uniform` and set the ratio (eg. making the ratio 0.5) by `--cp_uniform_preserve_ratio=0.5`. Note that for a layer, if both of pruning ratio of the layer and its previous layer are 0.5, the real preserved FLOPs are 1/4 of original FLOPs. Because channel pruning only prune the c_out channels of the convolution and c_in channels of the next convolution, if both c_in and c_out channels are pruned by 0.5, it will preserve only 1/4 of original computation cost. For a layer by layer convolution networks without residual blocks, if the user set `cp_uniform_preserve_ratio` to `0.5`, the whole model will be the 0.25 computation of the original model. However for the residual networks, some convolutions can only prune their c_in or c_out channels, which means the total preseved computation ratio may be much greater than 0.25. + +**Example:** + +``` bash +$ ./scripts/run_seven.sh nets/resnet_at_cifar10_run.py \ + --learner channel \ + --batch_size_eval 64 \ + --cp_uniform_preserve_ratio 0.5 \ + --cp_prune_option uniform \ + --resnet_size 20 +``` + +### List Channel Pruning + +Another pruning option is pruning the corresponding layer with ratios listed in a named `ratio.list` file, the file name of which can be set by `--cp_prune_list_file` option. the ratio value must be separated by a comma. User can set `--cp_prune_option=list` to prune the model by list ratios. + +**Example:** +Add list `1.0, 0.1875, 0.1875, 0.1875, 0.1875, 0.1875, 0.1875, 0.1875, 1.0, 0.25, 1.0, 0.25, 0.21875, 0.21875, 0.21875, 1.0, 0.5625, 1.0, 0.546875, 0.546875, 0.546875, 1` in `./ratio.list` + +``` bash +$ ./scripts/run_seven.sh nets/resnet_at_cifar10_run.py \ + --learner channel \ + --batch_size_eval 64 \ + --cp_prune_option list \ + --cp_prune_list_file ./ratio.list \ + --resnet_size 20 +``` + +### Automatic Channel Pruning + +The last one pruning option is searching better pruning ratios by reinforcement learning and you only need to give a value which represents what the ratio of total FLOPs/Computation you wants the compressed model preserve. You can set `--cp_prune_option=auto` and set a preserve ratio number such as `--cp_preserve_ratio=0.5`. User can also use `cp_nb_rlouts_min` to control reinforcement learning warm up iterations, which means the RL agent start to learn after the iterations, the default value is `50`. User can also use `cp_nb_rlouts_min` to control the total iteration RL agent to search, the default value is `200`. If the user want to control other parameters of the agents, please refer to the reinforcement component page. + +**Example:** + +``` bash +$ ./scripts/run_seven.sh nets/resnet_at_cifar10_run.py \ + --learner channel \ + --batch_size_eval 64 \ + --cp_preserve_ratio 0.5 \ + --cp_prune_option auto \ + --resnet_size 20 +``` + +## Channel pruning parameters + +The implementation of the channel pruning use Lasso algorithm to do channel selection and linear regression to do feature map reconstruction. During these two phases, sampling is done on the feature map to reduce computation cost. The users can use `--cp_nb_points_per_layer` to set how many sampling points on each layer are taken, the default value is `10`. For some dataset, if the images contain too many zero pixels (eg. black color), the value should be greater. The users can also set using how many batches to do channel selection and feature reconstruction by `cp_nb_batches`, the default value is `60`. Small value of `cp_nb_batches` may cause over-fitting and large value may slow down the solving speed, so a good value depends on the nets and dataset. For more practical usage, user may consider make the channel number of each layer is the quadruple for fast inference of mobile devices. In this case, user can set `--cp_quadruple` to `True` to make the compressed model have a quadruple number of channels. + +## Distilling + +Distilling is an effective approach to improve the final accuracy of compressed model with PocketFlow in most situations of classification. User can set `--enbl_dst=True` to enable distillling. + +## Group Tuning + +As introduced above, group tuning was proposed by the PocketFlow team and finding it is very useful to improve the performance of model compression. In PocketFlow, users can set `--cp_finetune=True` to enable group finetuning and set the group number by `--cp_list_group`, the default value is `1000`. There is a trade-off between the small value and large value, because if the value is `1`, Pocketflow will prune convolution and finetune/retrain by each layer, which may have better effect but be more time-consuming. If we set the value large, the function will be less effective. User can also set the number of iterations to finetune by setting `cp_nb_iters_ft_ratio` which mean the ratio the total iterations to be used in finetuning. The learning rate of finetuning can be set by `cp_lrn_rate_ft`. diff --git a/docs/docs/dcp_learner.md b/docs/docs/dcp_learner.md new file mode 100644 index 0000000..895bc52 --- /dev/null +++ b/docs/docs/dcp_learner.md @@ -0,0 +1,85 @@ +# Discrimination-aware Channel Pruning + +## Introduction + +Discrimination-aware channel pruning (DCP, Zhuang et al., 2018) introduces a group of additional discriminative losses into the network to be pruned, to find out which channels are really contributing to the discriminative power and should be preserved. After channel pruning, the number of input channels of each convolutional layer is reduced, so that the model becomes smaller and the inference speed can be improved. + +## Algorithm Description + +For a convolutional layer, we denote its input feature map as $\mathbf{X} \in \mathbb{R}^{N \times c_{i} \times h_{i} \times w_{i}}$, where $N$ is the batch size, $c_{i}$ is the number of inputs channels, and $h_{i}$ and $w_{i}$ are the spatial height and width. The convolutional kernel is denoted as $\mathbf{W} \in \mathbb{R}^{c_{o} \times c_{i} \times k \times k}$, where $c_{o}$ is the number of output channels and $k$ is the kernel size. The resulting output feature map is given by $\mathbf{Y} = f \left( \mathbf{X}; \mathbf{W} \right)$, where $f \left( \cdot \right)$ represents the convolutional operation. + +The idea of channel pruning is to impose the sparsity constraint on the convolutional kernel, so that some of its input channels only contains all-zero weights and can be safely removed. For instance, if the convolutional kernel satisifies: + +$$ +\left\| \left\| \mathbf{W}_{:, j, :, :} \right\|_{F}^{2} \right\|_{0} = c'_{i}, +$$ + +where $c'_{i} \lt c_{i}$, then the convolutional layer simplified to with $c'_{i}$ input channels only, and the computational complexity is reduced by a ratio of $\frac{c_{i} - c'_{i}}{c_{i}}$. + +In order to reduce the performance degradation caused by channel pruning, the DCP algorithm introduces a novel channel selection algorithm by incorporating additional discrimination-aware and reconstruction loss terms, as shown below. + +![DCP Learner](pics/dcp_learner.png) +**Source:** Zhuang et al., *Discrimination-aware Channel Pruning for Deep Neural Networks*. NIPS '18. + +The network is evenly divided into $\left( P + 1 \right)$ blocks. For each of the first $P$ blocks, an extra branch is derived from the output feature map of this block's last layer. The output feature map is then passed through batch normalization & ReLU & average pooling & softmax layers to produce predictions, from which a discrimination-aware loss is constructed, denoted as $L_{p}$. For the last block, the final loss of whole network, denoted as $L$, is used as its discrimination-aware loss. Additionally, for each layer in the channel pruned network, a reconstruction loss is introduced to force it to re-produce the corresponding output feature map in the original network. We denote the $q$-th layer's reconstruction loss as $L_{q}^{( R )}$. + +Based on a pre-trained model, the DCP algorithm performs channel pruning with $\left( P + 1 \right)$ stages. During the $p$-th stage, the network is fine-tuned with the $p$-th discrimination-aware loss $L_{p}$ plus the final loss $L$. After the block-wise fine-tuning, we sequentially perform channel pruning for each convolutional layer within the block. For channel pruning, we compute each input channel's gradients *w.r.t.* the reconstruction loss $L_{q}^{( R )}$ plus the discrimination-aware loss $L_{p}$, and remove the input channel with the minimal Frobenius norm of gradients. After that, this layer is fine-tuned with the remaining input channels only to (partially) recover the discriminative power. We repeat this process until the target pruning ratio is reached. + +After all convolutional layers have been pruned, the resulting network can be further fine-tuned for a few epochs to further reduce the performance loss. + +## Hyper-parameters + +Below is the full list of hyper-parameters used in the discrimination-aware channel pruning learner: + +| Name | Description | +|:-----|:------------| +| `dcp_save_path` | model's save path | +| `dcp_save_path_eval` | model's save path for evaluation | +| `dcp_prune_ratio` | target pruning ratio | +| `dcp_nb_stages` | number of channel pruning stages | +| `dcp_lrn_rate_adam` | Adam's learning rate for block-wise & layer-wise fine-tuning | +| `dcp_nb_iters_block` | number of iterations for block-wise fine-tuning | +| `dcp_nb_iters_layer` | number of iterations for layer-wise fine-tuning | + +Here, we provide detailed description (and some analysis) for above hyper-parameters: + +* `dcp_save_path`: save path for model created in the training graph. The resulting checkpoint files can be used to resume training from a previous run and compute model's loss function's value and some other evaluation metrics. +* `dcp_save_path_eval`: save path for model created in the evaluation graph. The resulting checkpoint files can be used to export GraphDef & TensorFlow Lite model files. +* `dcp_prune_ratio`: target pruning ratio for input channels of each convolutional layer. The larger `dcp_prune_ratio` is, the more input channels will be pruned. If `dcp_prune_ratio` equals 0, then no input channels will be pruned and model remains the same; if `dcp_prune_ratio` equals 1, then all input channels will be pruned. +* `dcp_nb_stages`: number of channel pruning stages / number of discrimination-aware losses. The training process of DCP algorithm is divided into multiple stages. For each discrimination-aware loss, a channel pruning stage is involved to select channels within corresponding layers. The final classification loss corresponds to a pseudo channel pruning stage, which is not counted in `dcp_nb_stages`.The larger `dcp_nb_stages` is, the slower the training process will be. +* `dcp_lrn_rate_adam`: Adam's learning rate for block-wise & layer-wise fine-tuning. If `dcp_lrn_rate_adam` is too large, then the fine-tuning process may become unstable; if `dcp_lrn_rate_adam` is too small, then the fine-tuning process may take long time to converge. +* `dcp_nb_iters_block`: number of iterations for block-wise fine-tuning. This should be set to some value that the block-wise fine-tuning can almost converge and the loss function's value does not decrease much even if more iterations are used. +* `dcp_nb_iters_layer`: number of iterations for layer-wise fine-tuning. This should be set to some value that the layer-wise fine-tuning can almost converge and the loss function's value does not decrease much even if more iterations are used. + +## Usage Examples + +In this section, we provide some usage examples to demonstrate how to use `DisChnPrunedLearner` under different execution modes and hyper-parameter combinations: + +To compress a ResNet-20 model for CIFAR-10 classification task in the local mode, use: + +``` bash +# set the target pruning ratio to 0.75 +./scripts/run_local.sh nets/resnet_at_cifar10_run.py \ + --learner dis-chn-pruned \ + --dcp_prune_ratio 0.75 +``` + +To compress a ResNet-34 model for ILSVRC-12 classification task in the docker mode with 4 GPUs, use: + +``` bash +# set the number of channel pruning stages to 4 +./scripts/run_docker.sh nets/resnet_at_ilsvrc12_run.py -n=4 \ + --learner dis-chn-pruned \ + --resnet_size 34 \ + --dcp_nb_stages 4 +``` + +To compress a MobileNet-v2 model for ILSVRC-12 classification task in the seven mode with 8 GPUs, use: + +``` bash +# enable training with distillation loss +./scripts/run_seven.sh nets/mobilenet_at_ilsvrc12_run.py -n=8 \ + --learner dis-chn-pruned \ + --mobilenet_version 2 \ + --enbl_dst +``` diff --git a/docs/docs/distillation.md b/docs/docs/distillation.md new file mode 100644 index 0000000..5db8487 --- /dev/null +++ b/docs/docs/distillation.md @@ -0,0 +1,28 @@ +# Distillation + +Distillation (Hinton et al., 2015) is a kind of model compression approaches in which a pre-trained large model teaches a smaller model to achieve the similar prediction performance. +It is often named as the "teacher-student" training, where the large model is the teacher and the smaller model is the student. + +With distillation, knowledge can be transferred from the teacher model to the student by minimizing a loss function to recover the distribution of class probabilities predicted by the teacher model. +In most situations, the probability of the correct class predicted by the teacher model is very high, and probabilities of other classes are close to 0, which may not be able to provide extra information beyond ground-truth labels. +To overcome this issue, a commonly-used solution is to raise the temperature of the final softmax function until the cumbersome model produces a suitably soft set of targets. The soften probability $q_i$ of class $i$ is calculated from the logit $z_i$: + +$$ +q_i = \frac{\exp \left( z_i / T \right)}{\sum_j{\exp \left( z_j / T \right)}} +$$ + +where $T$ is the temperature. +As $T$ grows, the probability distribution is more smooth, providing more information as to which classes the cumbersome model more similar to the predicted class. +It is better to include the standard loss ($T = 1$) between the predicted class probabilities and ground-truth labels. +The overall loss function is given by: + +$$ +L \left( x; W \right) = H \left( y, \sigma \left( z_s; T = 1 \right) \right) + \alpha \cdot H \left( \sigma \left( z_t; T = \tau \right), \sigma \left( z_s, T = \tau \right) \right) +$$ + +where $x$ is the input, $W$ are parameters of the distilled small model and $y$ is ground-truth labels, $\sigma$ is the softmax parameterized by temperature $T$, $H$ is the cross-entropy loss, and $\alpha$ is the coefficient of distillation loss. +The coefficient $\alpha$ can be set by `--loss_w_dst` and the temperature $T$ can be set by `--tempr_dst`. + +## Combination with Other Model Compression Approaches + +Other model model compression techniques, such as channel pruning, weight pruning, and quantization, can be augmented with distillation. To enable the distillation loss, simply append the `--enbl_dst` argument when starting the program. diff --git a/docs/docs/faq.md b/docs/docs/faq.md new file mode 100644 index 0000000..93d34f8 --- /dev/null +++ b/docs/docs/faq.md @@ -0,0 +1,5 @@ +# Frequently Asked Questions + +**Q: Under construction ...** + +A: Under construction ... \ No newline at end of file diff --git a/docs/docs/index.md b/docs/docs/index.md new file mode 100644 index 0000000..d3f6707 --- /dev/null +++ b/docs/docs/index.md @@ -0,0 +1,73 @@ +# PocketFlow + +PocketFlow is an open-source framework for compressing and accelerating deep learning models with minimal human effort. Deep learning is widely used in various areas, such as computer vision, speech recognition, and natural language translation. However, deep learning models are often computational expensive, which limits further applications on mobile devices with limited computational resources. + +PocketFlow aims at providing an easy-to-use toolkit for developers to improve the inference efficiency with little or no performance degradation. Developers only needs to specify the desired compression and/or acceleration ratios and then PocketFlow will automatically choose proper hyper-parameters to generate a highly efficient compressed model for deployment. + +## Framework + +The proposed framework mainly consists of two categories of algorithm components, *i.e.* learners and hyper-parameter optimizers, as depicted in the figure below. Given an uncompressed original model, the learner module generates a candidate compressed model using some randomly chosen hyper-parameter combination. The candidate model's accuracy and computation efficiency is then evaluated and used by hyper-parameter optimizer module as the feedback signal to determine the next hyper-parameter combination to be explored by the learner module. After a few iterations, the best one of all the candidate models is output as the final compressed model. + +![Framework Design](pics/framework_design.png) + +## Learners + +A learner refers to some model compression algorithm augmented with several training techniques as shown in the figure above. Below is a list of model compression algorithms supported in PocketFlow: + +| Name | Description | +|:-----|:------------| +| `ChannelPrunedLearner` | channel pruning with LASSO-based channel selection (He et al., 2017) | +| `DisChnPrunedLearner` | discrimination-aware channel pruning (Zhuang et al., 2018) | +| `WeightSparseLearner` | weight sparsification with dynamic pruning schedule (Zhu & Gupta, 2017) | +| `UniformQuantLearner` | weight quantization with uniform reconstruction levels (Jacob et al., 2018) | +| `UniformQuantTFLearner` | weight quantization with uniform reconstruction levels and TensorFlow APIs | +| `NonUniformQuantLearner` | weight quantization with non-uniform reconstruction levels (Han et al., 2016) | + +All the above model compression algorithms can trained with fast fine-tuning, which is to directly derive a compressed model from the original one by applying either pruning masks or quantization functions. The resulting model can be fine-tuned with a few iterations to recover the accuracy to some extent. Alternatively, the compressed model can be re-trained with the full training data, which leads to higher accuracy but usually takes longer to complete. + +To further reduce the compressed model's performance degradation, we adopt network distillation to augment its training process with an extra loss term, using the original uncompressed model's outputs as soft labels. Additionally, multi-GPU distributed training is enabled for all learners to speed-up the time-consuming training process. + +## Hyper-parameter Optimizers + +For model compression algorithms, there are several hyper-parameters that may have a large impact on the final compressed model's performance. It can be quite difficult to manually determine proper values for these hyper-parameters, especially for developers that are not very familiar with algorithm details. Recently, several AutoML systems, *e.g.* [Cloud AutoML](https://cloud.google.com/automl/) from Google, have been developed to train high-quality machine learning models with minimal human effort. Particularly, the AMC algorithm (He et al., 2018) presents promising results for adopting reinforcement learning for automated model compression with channel pruning and fine-grained pruning. + +In PocketFlow, we introduce the hyper-parameter optimizer module to iteratively search for the optimal hyper-parameter setting. We provide several implementations of hyper-parameter optimizer, based on models including Gaussian Processes (GP, Mockus, 1975), Tree-structured Parzen Estimator (TPE, Bergstra et al., 2013), and Deterministic Deep Policy Gradients (DDPG, Lillicrap et al., 2016). The hyper-parameter setting is optimized through an iterative process. In each iteration, the hyper-parameter optimizer chooses a combination of hyper-parameter values, and the learner generates a candidate model with fast fast-tuning. The candidate model is evaluated to calculate the reward of the current hyper-parameter setting. After that, the hyper-parameter optimizer updates its model to improve its estimation on the hyper-parameter space. Finally, when the best candidate model (and corresponding hyper-parameter setting) is selected after some iterations, this model can be re-trained with full data to further reduce the performance loss. + +## Performance + +In this section, we present some of our results for applying various model compression methods for ResNet and MobileNet models on the ImageNet classification task, including channel pruning, weight sparsification, and uniform quantization. +For complete evaluation results, please refer to [here](https://pocketflow.github.io/performance/). + +### Channel Pruning + +We adopt the DDPG algorithm as the RL agent to find the optimal layer-wise pruning ratios, and use group fine-tuning to further improve the compressed model's accuracy: + +| Model | Pruning Ratio | Uniform | RL-based | RL-based + Group Fine-tuning | +|:------------:|:-------------:|:-------:|:-------------:|:----------------------------:| +| MobileNet-v1 | 50% | 66.5% | 67.8% (+1.3%) | 67.9% (+1.4%) | +| MobileNet-v1 | 60% | 66.2% | 66.9% (+0.7%) | 67.0% (+0.8%) | +| MobileNet-v1 | 70% | 64.4% | 64.5% (+0.1%) | 64.8% (+0.4%) | +| Mobilenet-v1 | 80% | 61.4% | 61.4% (+0.0%) | 62.2% (+0.8%) | + +### Weight Sparsification + +Comparing with the original algorithm (Zhu & Gupta, 2017) which uses the same sparsity for all layers, we incorporate the DDPG algorithm to iteratively search for the optimal sparsity of each layer, which leads to the increased accuracy: + +| Model | Sparsity | (Zhu & Gupta, 2017) | RL-based | +|:------------:|:--------:|:-------------------:|:-----------------------:| +| MobileNet-v1 | 50% | 69.5% | 70.5% (+1.0%) | +| MobileNet-v1 | 75% | 67.7% | 68.5% (+0.8%) | +| MobileNet-v1 | 90% | 61.8% | 63.4% (+1.6%) | +| MobileNet-v1 | 95% | 53.6% | 56.8% (+3.2%) | + +### Uniform Quantization + +We show that models with 32-bit floating-point number weights can be safely quantized into their 8-bit counterpart without accuracy loss (sometimes even better!). +The resulting model can be deployed on mobile devices for faster inference (Device: XiaoMi 8 with a Snapdragon 845 CPU): + +| Model | Acc. (32-bit) | Acc. (8-bit) | Time (32-bit) | Time (8-bit) | +|:------------:|:-------------:|:---------------:|:-------------:|:--------------------:| +| MobileNet-v1 | 70.89% | 71.29% (+0.40%) | 124.53 | 56.12 (2.22$\times$) | +| MobileNet-v2 | 71.84% | 72.26% (+0.42%) | 120.59 | 49.04 (2.46$\times$) | + +* All the reported time are in milliseconds. diff --git a/docs/docs/installation.md b/docs/docs/installation.md new file mode 100644 index 0000000..0799266 --- /dev/null +++ b/docs/docs/installation.md @@ -0,0 +1,98 @@ +# Installation + +PocketFlow is developed and tested on Linux, using Python 3.6 and TensorFlow 1.10.0. We support the following three execution modes for PocketFlow: + +* Local mode: run PocketFlow on the local machine. +* Docker mode: run PocketFlow within a docker image. +* Seven mode: run PocketFlow on the seven cluster (only available within Tencent). + +## Clone PocketFlow + +To make a local copy of the PocketFlow repository, use: +``` bash +$ git clone https://github.com/Tencent/PocketFlow.git +``` + +## Create a Path Configuration File + +PocketFlow requires a path configuration file, named `path.conf`, to setup directory paths to data sets and pre-trained models under different execution modes, as well as HDFS / HTTP connection parameters. + +We have provided a template file to help you create your own path configuration file. You can find it in the PocketFlow repository, named `path.conf.template`, which contains more detailed descriptions on how to customize path configurations. For instance, if you want to use CIFAR-10 and ImageNet data sets stored on the local machine, then the path configuration file should look like this: + +``` bash +# data files +data_hdfs_host = None +data_dir_local_cifar10 = /home/user_name/datasets/cifar-10-batches-bin # this line has been edited! +data_dir_hdfs_cifar10 = None +data_dir_seven_cifar10 = None +data_dir_docker_cifar10 = /opt/ml/data # DO NOT EDIT +data_dir_local_ilsvrc12 = /home/user_name/datasets/imagenet_tfrecord # this line has been edited! +data_dir_hdfs_ilsvrc12 = None +data_dir_seven_ilsvrc12 = None +data_dir_docker_ilsvrc12 = /opt/ml/data # DO NOT EDIT + +# model files +model_http_url = https://api.ai.tencent.com/pocketflow +``` + +In short, you need to replace "None" in the template file with the actual path (or HDFS / HTTP connection parameters) if available, or leave it unchanged otherwise. + +## Prepare for the Local Mode + +We recommend to use Anaconda as the Python environment, which has many essential packages built-in. The Anaconda installer can be downloaded from [here](https://www.anaconda.com/download/#linux). To install, use the following command: + +``` bash +# install Anaconda; replace the installer's file name if needed +$ bash Anaconda3-5.2.0-Linux-x86_64.sh + +# activate Anaconda's Python path +$ source ~/.bashrc +``` + +For Anaconda 5.3.0 or later, the default Python version is 3.7, which does not support installing TensorFlow with pip directly. Therefore, you need to manually switch to Python 3.6 once Anaconda is installed: + +``` bash +# install Python 3.6 +$ conda install python=3.6 +``` + +To install TensorFlow, you may refer to TensorFlow's official [documentation](https://www.tensorflow.org/install/pip) for detailed instructions. Specially, if GPU-based training is required, then you need to follow the [GPU support guide](https://www.tensorflow.org/install/gpu) to set up a CUDA-enabled GPU card in prior to installation. After that, install TensorFlow with: + +``` bash +# TensorFlow with GPU support; use if GPU is not available +$ pip install tensorflow-gpu + +# verify the install +$ python -c "import tensorflow as tf; print(tf.__version__)" +``` + +To run PocketFlow in the local mode, *e.g.* to train a full-precision ResNet-20 model for the CIFAR-10 classification task, use the following command: + +``` bash +$ ./scripts/run_local.sh nets/resnet_at_cifar10_run.py +``` + +## Prepare for the Docker Mode + +Docker offers an alternative way to run PocketFlow within an isolated container, so that your local Python environment remains untouched. We recommend you to use the [horovod](https://github.com/uber/horovod) docker provided by Uber, which enables multi-GPU distributed training for TensorFlow with only a few lines modification. Once docker is installed, the docker image can be obtained via: + +``` bash +# obtain the docker image +$ docker pull uber/horovod +``` + +To run PocketFlow in the docker mode, *e.g.* to train a full-precision ResNet-20 model for the CIFAR-10 classification task, use the following command: + +``` bash +$ ./scripts/run_docker.sh nets/resnet_at_cifar10_run.py +``` + +## Prepare for the Seven Mode + +Seven is a distributed learning platform built for both CPU and GPU clusters. Users can submit tasks to the seven cluster, using built-in data sets and docker images seamlessly. + +To run PocketFlow in the seven mode, *e.g.* to train a full-precision ResNet-20 model for the CIFAR-10 classification task, use the following command: + +``` bash +$ ./scripts/run_seven.sh nets/resnet_at_cifar10_run.py +``` diff --git a/docs/docs/multi_gpu_training.md b/docs/docs/multi_gpu_training.md new file mode 100644 index 0000000..1efc11c --- /dev/null +++ b/docs/docs/multi_gpu_training.md @@ -0,0 +1,111 @@ +# Multi-GPU Training + +Due to the high computational complexity, it often takes hours or even days to fully train deep learning models using a single GPU. +In PocketFlow, we adopt multi-GPU training to speed-up this time-consuming training process. +Our implementation is compatible with: + +* [Horovod](https://github.com/uber/horovod): a distributed training framework for TensorFlow, Keras, and PyTorch. +* TF-Plus: an optimized framework for TensorFlow-based distributed training (only available within Tencent). + +We have provide a wrapper class, `MultiGpuWrapper`, to seamlessly switch between the above two frameworks. +It will sequentially check whether Horovod and TF-Plus can be used, and use the first available one as the underlying framework for multi-GPU training. + +The main reason that using Horovod or TF-Plus instead TensorFlow's original distributed training routine is that these framewors provide many easy-to-use APIs and require far less code changes to change from single-GPU to multi-GPU training, as we shall see later. + +## From Single-GPU to Multi-GPU + +To extend a single-GPU based training script to the multi-GPU scenario, at most 7 steps are needed: + +* Import the Horovod or TF-Plus module. + +``` Python +from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw +``` + +* Initialize the multi-GPU training framework, as early as possible. + +``` Python +mgw.init() +``` + +* For each worker, create a session with a distinct GPU device. + +``` Python +config = tf.ConfigProto() +config.gpu_options.visible_device_list = str(mgw.local_rank()) +sess = tf.Session(config=config) +``` + +* (Optional) Let each worker use a distinct subset of training data. + +``` Python +filenames = tf.data.Dataset.list_files(file_pattern, shuffle=True) +filenames = filenames.shard(mgw.size(), mgw.rank()) +``` + +* Wrapper the optimizer for distributed gradient communication. + +``` Python +optimizer = tf.train.AdamOptimizer(learning_rate=lrn_rate) +optimizer = mgw.DistributedOptimizer(optimizer) +train_op = optimizer.minimize(loss) +``` + +* Synchronize master's parameters to all the other workers. + +``` Python +bcast_op = mgw.broadcast_global_variables(0) +sess.run(tf.global_variables_initializer()) +sess.run(bcast_op) +``` + +* (Optional) Save checkpoint files at the master node periodically. + +``` Python +if mgw.rank() == 0: + saver.save(sess, save_path, global_step) +``` + +## Usage Example + +Here, we provide a code snippet to demonstrate how to use multi-GPU training to speed-up training. +Please note that many implementation details are omitted for clarity. + +``` Python +import tensorflow as tf +from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw + +# initialization +mgw.init() + +# create the training graph +with tf.Graph().as_default(): + # create a TensorFlow session + config = tf.ConfigProto() + config.gpu_options.visible_device_list = str(mgw.local_rank()) + sess = tf.Session(config=config) + + # use tf.data.Dataset() to traverse images and labels + filenames = tf.data.Dataset.list_files(file_pattern, shuffle=True) + filenames = filenames.shard(mgw.size(), mgw.rank()) + images, labels = get_images_n_labels(filenames) + + # define the network and its loss function + logits = forward_pass(images) + loss = calc_loss(labels, logits) + + # create an optimizer and setup training-related operations + global_step = tf.train.get_or_create_global_step() + optimizer = tf.train.AdamOptimizer(learning_rate=lrn_rate) + optimizer = mgw.DistributedOptimizer(optimizer) + train_op = optimizer.minimize(loss, global_step=global_step) + bcast_op = mgw.broadcast_global_variables(0) + +# multi-GPU training +sess.run(tf.global_variables_initializer()) +sess.run(bcast_op) +for idx_iter in range(nb_iters): + sess.run(train_op) + if mgw.rank() == 0 and (idx_iter + 1) % save_step == 0: + saver.save(sess, save_path, global_step) +``` diff --git a/docs/docs/nuq_learner.md b/docs/docs/nuq_learner.md new file mode 100644 index 0000000..8f22cfc --- /dev/null +++ b/docs/docs/nuq_learner.md @@ -0,0 +1,109 @@ +# Non-Uniform Quantization Learner +This document describes how to set up the Non-Uniform Quantization Learner in PocketFlow. In non-uniform quantization, the quantization points are not distributed evenly, and can be optimized via the back-propagation of the network gradients. +Consequently, with the same number of bits, non-uniform quantization is more expressive to approximate the original full-precision network comparing to uniform quantization. + +Following a similar pattern in the previous sections, we first show how to configure the Non-Uniform Quantization Learner, followed by the algorithms used in the learner. + +### Prepare the Model +Again, users should first get the model prepared. Users can either use the pre-built models in PocketFlow, or develop their custom models according to [TODO](???). + +### Configure the Learner +To configure the learner, users can pass the options via the TensorFlow flag interface. The available options are as follows: + +| Options | Default Value | Description | +| :-------- | :--------:| :-- | +| `--nuql_opt_mode` | weight| variables to optimize: ['weights', 'clusters', 'both'] | +| `--nuql_init_style` | quantile | the initialization of quantization points: ['quantile', 'uniform'] | +| `--nuql_weight_bits` | 4 | the number of bits for weight | +| `--nuql_activation_bits` | 32 | the number of bits for activation， by default it remains full precision | +| `--nuql_save_quant_mode_path` | *TODO* | the save path for quantized models | +| `--nuql_use_buckets` | False | use bucketing or not | +| `--nuql_bucket_type` | channel | two bucket type available: ['split', 'channel'] | +| `--nuql_bucket_size` | 256 | quantize the first and last layers of the network or not | +| `--nuql_enbl_rl_agent` | False | enable reinforcement learning to learn the optimal bit allocation or not | +| `--nuql_quantize_all_layers` | False | quantize the first and last layers of the network or not | +| `--nuql_quant_epoch` | 60 | the number of epochs for fine-tuning | + +Note that since non-uniform quantization cannot be accelerated directly, by default we do not quantize the activations. + +### Examples +Once the model is built, the Non-Uniform Quantization Learner can be easily triggered by passing the Uniform Quantization Learner in the command line as follows: +```bash +# quantize resnet-20 on CIFAR-10 +# you can also configure the +sh ./scripts/run_local.sh nets/resnet_at_cifar10_run.py \ +--data_disk local \ +--data_dir_local ${PF_CIFAR10_LOCAL} \ +--learner=non-uniform \ +--nuql_weight_bits=4 \ +--nuql_activation_bits=4 \ + +# quantize the resnet-18 on ILSVRC-12 +sh ./scripts/run_local.sh nets/resnet_at_ilsvrc12_run.py \ +--learner=uniform \ +--data_disk local \ +--data_dir_local ${PF_ILSVRC12_LOCAL} \ +--nuql_weight_bits=8 \ +--nuql_activation_bits=8 \ +--nuql_use_buckets=True \ +--nuql_bucket_type=channel +``` + +To enable the RL agent, one can follow similar patterns as those in the Uniform Quantization Learner: +```bash +# quantize mobilenet-v1 on ILSVRC-12 +sh ./scripts/run_local.sh nets/mobilnet_at_ilsvrc12_run.py \ +--data_disk local \ +--data_dir_local ${PF_CIFAR10_LOCAL} \ +--learner=uniform \ +--nuql_enbl_rl_agent=True \ +--nuql_equivalent_bits=4 \ +--nuql_tune_global_steps=1200 +``` + +### Performance +Here we list some of the performance on Cifar-10 using the Non-Uniform Quantization Learner and the built-in models in PocketFlow. The options not displayed remain the default values. + + +| Model | Weight Bit| Activation Bit | Acc | +| :--------: |:--------:| :--: | :--:| +| ResNet-20 | 32 | 32 | 91.96 | +| ResNet-20 | 2 | 4 | 90.31 | +| ResNet-20 | 4 | 8 | 91.70 | + + +| Model | Weight Bit| Bucketing | Acc | +| :--------: | :--: |:--------:| :--: | +| ResNet-20 | 2 | channel | 90.90 | +| ResNet-20 | 4 | channel | 91.97 | +| ResNet-20 | 2 | split | 90.02 | +| ResNet-20 | 4 | split | 91.56 | + + +| Model | Weight Bit| RL search | Acc | +| :--------: | :--: |:--------:| :--: | +| ResNet-20 | 2 | FALSE | 90.31 | +| ResNet-20 | 4 | FALSE | 91.70 | +| ResNet-20 | 2 | TRUE| 90.60 | +| ResNet-20 | 4 | TRUE | 91.79 | + +## Algorithm +Non-Uniform Quantization Learner adopts a similar training and evaluation procedure to the Uniform Quantization. In the training process, the quantized weights are forwarded. In the backward pass, the full precision weights are updated via the STE estimator. The major difference from uniform quantization is that, the location of quantization points are not evenly distributed, but can be optimized and initialized differently. In the following, we introduce the scheme to update and initialize the quantization points. + +### Optimization the quantization points +Unlike uniform quantization, non-uniform quantization can optimize the location of quantization points dynamically during the training of the network, and thereon leads to less quantization loss. The location of quantization points can be updated by summing the gradients of weights that fall into the point ([Han et.al 2015](https://arxiv.org/abs/1510.00149)), i.e.,: +$$ +\frac{\partial \mathcal{L}}{\partial c_k} = \sum_{i,j}\frac{\partial\mathcal{L}}{\partial w_{ij}}\frac{\partial{w_{ij}}}{\partial c_k}=\sum_{ij}\frac{\partial\mathcal{L}}{\partial{w_{ij}}}1(I_{ij}=k) +$$ + +The following figure taken from [Han et.al 2015](https://arxiv.org/abs/1510.00149) shows the process of updating the clusters: + +![Deep Compression Algor](pics/deep_compression_algor.png) + +### Initialization of quantization points +Aside from optimizing the quantization points, another helpful strategy is to properly initialize the quantization points according to the distribution of weights. PocketFlow currently supports two kinds of initialization: uniform initialization and quantile initialization. Comparing to uniform initialization, quantile initialization uses the quantiles of weights as the initial locations of quantization points. Quantile initialization considers the distribution of weights and can usually lead to better performance. + + + +## References +Han S, Mao H, and Dally W J. Deep compression: Compressing deep neural networks with pruning, trained quantization and huffman coding. [arXiv:1510.00149, 2015](https://arxiv.org/abs/1510.00149) diff --git a/docs/docs/performance.md b/docs/docs/performance.md new file mode 100644 index 0000000..39a1a75 --- /dev/null +++ b/docs/docs/performance.md @@ -0,0 +1,46 @@ +# Performance + +In this documentation, we present evaluation results for applying various model compression methods for ResNet and MobileNet models on the ImageNet classification task, including channel pruning, weight sparsification, and uniform quantization. + +We adopt `ChannelPrunedLearner` to shrink the number of channels for convolutional layers to reduce the computation complexity. +Instead of using the same pruning ratio for all layers, we utilize the DDPG algorithm as the RL agent to iteratively search for the optimal pruning ratio of each layer. +After obtaining the optimal pruning ratios, group fine-tuning is adopted to further improve the compressed model's accuracy, as demonstrated below: + +| Model | Pruning Ratio | Uniform | RL-based | RL-based + Group Fine-tuning | +|:------------:|:-------------:|:-------:|:-------------:|:----------------------------:| +| MobileNet-v1 | 50% | 66.5% | 67.8% (+1.3%) | 67.9% (+1.4%) | +| MobileNet-v1 | 60% | 66.2% | 66.9% (+0.7%) | 67.0% (+0.8%) | +| MobileNet-v1 | 70% | 64.4% | 64.5% (+0.1%) | 64.8% (+0.4%) | +| Mobilenet-v1 | 80% | 61.4% | 61.4% (+0.0%) | 62.2% (+0.8%) | + +We adopt `WeightSparseLearner` to introduce the sparsity constraint so that a large portion of model weights can be removed, which leads to smaller model and lower FLOPs for inference. +Comparing with the original algorithm proposed in (Zhu & Gupta, 2017), we also incorporate network distillation and reinforcement learning algorithms to further improve the compressed model's accuracy, as shown in the table below: + +| Model | Sparsity | (Zhu & Gupta, 2017) | RL-based | +|:------------:|:--------:|:-------------------:|:-------------:| +| MobileNet-v1 | 50% | 69.5% | 70.5% (+1.0%) | +| MobileNet-v1 | 75% | 67.7% | 68.5% (+0.8%) | +| MobileNet-v1 | 90% | 61.8% | 63.4% (+1.6%) | +| MobileNet-v1 | 95% | 53.6% | 56.8% (+3.2%) | + +We adopt `UniformQuantTFLearner` to uniformly quantize model weights from 32-bit floating-point numbers to 8-bit fixed-point numbers. +The resulting model can be converted into the TensorFlow Lite format for deployment on mobile devices. +In the following two tables, we show that 8-bit quantized models can be as accurate as (or even better than) the original 32-bit ones, and the inference time can be significantly reduced after quantization. + +| Model | Top-1 Acc. (32-bit) | Top-5 Acc. (32-bit) | Top-1 Acc. (8-bit) | Top-5 Acc. (8-bit) | +|:------------:|:-------------------:|:-------------------:|:------------------:|:------------------:| +| ResNet-18 | 70.28% | 89.38% | 70.31% (+0.03%) | 89.40% (+0.02%) | +| ResNet-50 | 75.97% | 92.88% | 76.01% (+0.04%) | 92.87% (-0.01%) | +| MobileNet-v1 | 70.89% | 89.56% | 71.29% (+0.40%) | 89.79% (+0.23%) | +| MobileNet-v2 | 71.84% | 90.60% | 72.26% (+0.42%) | 90.77% (+0.17%) | + +| Model | Hardware | CPU | Time (32-bit) | Time (8-bit) | Speed-up | +|:------------:|:-----------:|:--------------:|:-------------:|:------------:|:------------:| +| MobileNet-v1 | XiaoMi 8 SE | Snapdragon 710 | 156.33 | 62.60 | 2.50$\times$ | +| MobileNet-v1 | XiaoMI 8 | Snapdragon 845 | 124.53 | 56.12 | 2.22$\times$ | +| MobileNet-v1 | Huawei P20 | Kirin 970 | 152.54 | 68.43 | 2.23$\times$ | +| MobileNet-v2 | XiaoMi 8 SE | Snapdragon 710 | 153.18 | 57.55 | 2.66$\times$ | +| MobileNet-v2 | XiaoMi 8 | Snapdragon 845 | 120.59 | 49.04 | 2.46$\times$ | +| MobileNet-v2 | Huawei P20 | Kirin 970 | 226.61 | 61.38 | 3.69$\times$ | + +* All the reported time are in milliseconds. diff --git a/docs/docs/pics/dcp_learner.png b/docs/docs/pics/dcp_learner.png new file mode 100644 index 0000000000000000000000000000000000000000..cff9ad822301a56ec3aa0cb5a20fd4fb31503899 GIT binary patch literal 102436 zcma&Oby$>Z*FFp?N=c`5Bc*gV(jg7f3@|hdjg%lLLxa)-C?MV4NP{#;mvpx@$afFA z_q+FV{J!^n{%|~CX70G|wa&H9bDbB^m&!7aQJ(FuH4>WNgz`N{^R9d2a$m>7V#${`pN)FSi|$Fk)w(A1CaIHK&K0CXu%9Z6j?7p|;9|tDuGEY}2ku-)_&X zm=c*P$tszu&_MG4xSXml*Be$d-5h?=yB~IGqdPFWFlMCVE_%oPghHGjg$nB_N+5O+ zQ)P-Pr!~HLL2|a7X0`D7?8@da`?G=_?o<|WC|YT4No#p!;CIR{yQ=&?pRk5gNW|$m zindwud|Ldzi2dO08N(m}uIcS^+`68GY-TxVAEocYE%FL>`ZdR)a=qSBs;JaF*=**U ziiT%Z9F_z%n(spztuoJT-@=?S8#6CCGe;*qj3dJ)UFQ59qA5e1C-BvJwPLDfAG5_K zuuD|CNXaWHYl~~uRXAJm0W+SqDtcMVK5i=lTkzf8DND}|tY&-=3qE&0HbmtDt^vzx zqU?xRv58cX`hDZ0NKd(3uPg6&A8S1)6O1Ba+BE+xWW1-Kc|Ei-TAs7~zxcTofnVQoR%0GdRT|W-b_`+WC3fk8rnDZjQqihK?Gg8{Gu0;A zR6q0Iq~>)`yLFV@b?a5Kx(L9c3Y-j=ZHSBQj&BLujg#g5(B9M_LSL>W(Qjm!6RYVK zYMWi!Uv|@;xHv@OUWq%^=X1{ zg`K~Z^R}bHqrNgBzkfwS&E4PXEWv@J1eKcS`2rCx$`?Vl9r6o;hEmJ|fzZc&3q!H? zZ7y9wDa&5!@di+AiJidOX~iaVi)yyEd(fYk#yf8Vd0kPc+^9#2&FR(#jSB?I8N4vb z^J7(to9kaP=j=S0G6c<-iv~BIFHSDLy3lyuRIK%Kio2|_vnowcXJ?AW6bX)Eyc|`D zkYPq%Nms(SXsk{0)=A6=V*HzKCEDdPR^CqP&6$<8ZeFhN8)MO{;YbZIakGP5CVBPR z@$S{}k+7ExGS#At{hKH2*|LqFnb{Y!4Z7TAfCV5E7CE2OcRWy1@2HO!DeLEI|_jaP%t_Dt2wj5(XrRV1wsI zz0#N9=g#KEJA|c#c;I#>#sOJMIEc@_ghk8JCPnBSzvsc3lN#-R2om*}1=2oH7q&TI z5RC&rE7@m0n$N9#tv=4!F*|>#s8zxpowau4HX-N_q!MS{QNm0)h7yRxx%vzcc(okgRA+;5EQQh(_)phDyP6t^L*As{pWM zIVH+4vaXO721m~`Gw-I$H@RM&&WzKQNa{v=P;sb($JR=PF1$SNVni!aXXH z5q|dVHKCo$QlA0Tjm$7XZ>zD00S|ooZYk`+J)j)717QKbMp;@oE>Yc~6V*77DiT5F zO9d{%(qV>?)CS4z-{(vCY$A&|LmDPd0{^@FEL*Z|jY78_uOt(BdtG2Al-e+szf^E? zj8@W7p+6jQ=7`^++YX%do()&8%u2tc%gmUa~mw+iE1hEFksh8h{_-0CT`yZC=NR!G#(r+xS^Kset_J<`6)!iswMxZ{5ZO*S1<#I}RFP`PF4 z#UR7Rp-L|mQO^gwm3BcqZ4>IS7uZ1PV8N5j{!c8!vqS>9l;G|a1IQSr0$eb-#JokY zhJOI@eV8_Z7~?=X8t?}4UHSPFdhnd33e-Wndwm2zCVm{WToNhar>qwrsh?PD?}z8E z_B>jG%O0SrAu)84r zaMc|0liRaqBQf>oB!zJ6PUHsnn)*)Ew{rKV13ReyI}*DtDSol!H>7|C71r@A?92t6 zWjmzvg;E0%JzraMtH*NFRqkOUWhC^SXMnZe*R9uCR7tnaRQgY7a^*Uwz3jbDwMjqNuZWq5+hg#v;8Y z^JTs5r~e{QLO5+Cfo1`?R6W%i7Bq0AtxU&=`{7b$bKYiB>5U|jC_llaU*3eb1%*BWNa0HtV!&3UmY69H~(;QkJTkD^00j{ZaWncns?J$ zHF#*)?4MBmOYrQcicC{m>SL0Fj0qbq(*N@u!%h%4j2B+tqN=miMhPsLqSKP0Ex0@{ z0hoykDaZE9&E>FhWj4IBBxm~85MQfPhK}}77&;*;-&Le}+nS%T=G7^P#)1d_40Kg@ zmF+=%`PXkyv4%_D3^pRHG-R`yH`Xf7QJ?s1f`_g#WJmwA=B0mGDwsUMPqkh^jW{i) z>~O6sDs;h>)-PPEXAZchki5A46abz4j;TR=FYIG2hofih-U1uRP6ZDBSAMWI_CjniST}P2a{IcjYk29)jtvNr&$CXcxtX;s; zU<}ew(RP*^iuHAXsu>nDG_O=$Bu2^GiLR)qwA2B(xi6QpZKl}y!TzAhgf;ow@$&raco0BIm+8j8DH;U7>a8ZrnAFC_!&F#`lzkhBgVV zHy3#i_h6*9EQKS_W?JvA&lyMdHL7esLdf9jH6+;}wC)5%THg($BQ6Opvg&?9@Zt?w zR2FVX@?)U|90^n6?T#rlOXryd$k|EtQVVYuwW}ryYDDS;%S_on7ncO6ul#tisl$Dk z%0NL@Vr-uZic#W&H4ivo*rkRhHWqh=I}C__vGDQxJLkvIg6YnUAwjQwF=h;G*nMVg zYObc0Jzjq&HLGDN(XLkJ1ns;Jl;RzBndLD8=tkS+pNk^t?U*9mJkQ$^!XPdDI&JQJ zcpB;_(=~HtGFecmQ@;uJ<%~x?ivkCBsq?l!$=*CcdpcH&rvb}X;A^MII zFZgqLbxdJ~s+)DRAtDXHxQJkEuxM?}u(-P;cfx?Id`cwJ7KM$OZ)6i{(aM8<-^Spu zkTjuQj~u4XZ!1#m3SOi~t?Y67DeMTH#>E*g!g4)U7xf@Mk=2Ld z79v$~)!q%r0F7&x7_g8`aj~wu8l@+fWj$i?78mx@FAtNCd7WZ`1BAyxd!ox+&BKH= z{o6C#w460?iB0Eo&_(~lR0n|KMN}^B2!*)Co78KcnP@UjroR1tw$T1|VgYL3Ci3F- zz-Iz?c4+|rN@<2&SQP-=+@Tr(;HMhdp(#xgLhu48*Hszk8Sf7{-|shii&}XJ8YVYA z(43wadB4Qs!Y34oE$P}JkFF4W#y_M;otsBAb^IZk$iY5RMndgj2u&GDp0!3TJ$1{% zueu8qeBx(G)0zU==?RVY(e%c(Z!8%Zm`b_9#3DrFBojsp_lq0Spf6a7H~}t>QcC2i z{7!%&A2(7(;QrKVY7qLsy<8u~-kg)~wY78x52QySBzO?5Xl>qcU633sFYb1@oEn8L zZo$@)@h%h*Y&vvlo{9#>CZ?G>MhXfWgxN{#Ew0VYegPW($!N>ExGz9Q!-@}n!93rl zxm8{~aWi6()qijTFaH1+xz@`R_2#aNOn{OV>1)L9cXp7CvTsm0JC57v61p{<31k%y zw7n?42>)v(+L7?GMfRNU~$|G=e@IGiO#JR29gLdmeJcf*yEc@fI8Yyj2x-IufIgZW7Msoi`@seRymp zkbFA(Uv!>+xHP|k^Y6c7yrO1l=LbR9_N+7+P1e93@Uu2bo_0? zv?WxsB&%k~W-8Rn0<`j0vc$yQf8x4R0zx1_n%addK?8L$AhbrBS(Ytyr{TS4sR4UB zsYzo+WrB}YxzN{$1gY$*l`=p$ON(^v;@alIg6k4^8uxk(8iK+!KG3auPCr)crBk$3 z{gNmSzzHi3SaYlKH1Aq{zg(elp-ChDo{vCMoH|OtS6U^S0$vi2CP2kNSi}X3ezH^% z8Od%8x1YHB88cFZcA2%`DYEewwvK*u?caAPMenmHH**p$1`gdijLY z;Rfs#p6A)4``F;X3annF_9i=>jXbV?oAXYIG{|v@ z-uFJTyDJS1(#G(9^x93mrDL4}_6*NzMh3m6b zrTVZhCu1;><}00xA3#g{Kjju6`UOB&n5A~g+T^(?eck6XZyo8IR(#TD1oeb!U%@8W zI($G)7TV!F9gCh}lw3;%k{tRcLTaO(eECI=CgRcpF}Q-Si$6pKen(={IImCBF+kLo z=D=AN?(Jfr=9gr>Re#FyP;tyw`|(~{d{SWGblMoPT<#R(_}oi-S=gA(ar<>DSB)Zf zUdV)T6T3)~y!Vn+fvs&A*VH}|XVgdIk3C&9R8R4zNcC7vQ2@w2b8k5JwHUHzNq|?& zz=YRx2zj$f2(AIDgKsBZWWdv~9~Yct|AZvgILUR=03?g@S22;c>`2rQY4CSnU#X}$ zY5|BJASU7{O_Tv7o}O!Ihf9Au4m47u>hW3onagf!F!-VJ*K^(c=*H~|i;J&toq`1) z98=amnR0>Bl={J6@(Rsn-H4^9H6U7r`m-IW?`igT>cDp&eg-SCIgzY|OUA`*MSBM_&l0kJN z3JP1zP&^v=c#hcpJW|Olz0Q<|z78p)bYbnRp7Wh~yo{kH1xoar=k_eNKsllK*^5L4 z*K%`mAAG-%O+oj@nPq$OIDNCmE|t$l`6R(QU;7Kdd*vmZe|K*?I8PvMbbG$8sA}4~ zGQ%iyPOO;!%w!QY}ToL{|P zXAiK{fsGZf;B3Z*p=$?8uW2mXo|a6@GdoymB56BG%njyD^nMxt_&8$1Zx6Xqk6lBUXofg@cNC=rIvl+VE5K@|3a@-h$r;mkZ-ttZ|L| z7#{}%sGz%~$ZM!u-k$jSvFiZCXDYluAkHx!TCROCbmhrdBAY<`ulxu4?zi>PBWLs{ zkV$8Xid`~~PZN~rPkyn*(8|3%FoTPu5Mq1fS5Vfv2}?>K_ButDY?<8(b%SpI$PyTX zpa+XG2v(!m&>mvP3oXa7CFZU>R1+0nSd;A&3#wN-*h4KVdetwM_jUe+K;O`b39*OG z`DRAWjA8DFRXw8In!_(W`hmI}yXSRE`OVFTa#II;9>euzg?o)S4RmHVuFyH4qi1Oi z>~r!-UeNel@-xiBw+)Vut^NXpPe4hD83Sl7^ni9AQa;)8&sG%9HY`n%+K1H{SsbPX z{+UIc0uZ-{dTaEHPF-JWvL?J_d#1y^?KG!!@oKb86eG!HzX_yUol(Hm1WZU3ZftUv zNqb;1M0}_=7h}MGii0k{0%Eu?ZBBg>rue!F< zPOf;%RoruvKmxFq#jlZkq3!GgNgpi8tMS04Z&!Iql5p=jkig2Ng2bsNWOc2S@6{J5 z|B7v8*sjdhG4FEfu+6>gccsRNmknhPW?bxi|6s$a-b)TYq0XPRn11%=vodFCy)u2- zeC1E!&vQ5df#Z2Ni57`-P^_p_gE{3?l-1gKaOT#MhRdmebe*wcQem#sF}~g7P|o5s zb6m(;V{>>gXFm^)7p@v2BpqkQolMH!E*iKo=t_F^K-ykP^>GdTr7W?!Kz1D= z3IUusD8L|L<-xR>f8vyq3Lm`M6_qVmJ-&wEU)S^!{JNV6d^oqk%303fhkky@JW0x z9J=GZ0(HUYy=)SOAbcGW1H#vIBG|aceam@?^`_{QYd&Q`e8+_eHY`mBsyOwa#y5PC zjb7EJYdO9(UksT0*)vo~=DWjL@{Zm-$bF+GH=wp@8p9bAAts#*UW|it$f0)KjOCu( zDve6T593rV?SF?OK-c(J9L}G#r8Z*PajmQS>}3L_SPJP68vyXSaU?p!W^`2yLwc4U zBN6D0-8uvdZv{>?}1w_;)$-O0(5YFVgMnef3{DgR= z1^^3^5KSamZr5I-1%~^pdtUDmLpXUy8kcQp>Ds-X72~9R3{O%W^)?y=3qEP3H{ibf z6;|g%GyJCV-ja|TE4d*}g5F#Jz{om(_EqZCFm?Gg-4JlfoOGsn(C|vUWEhxcd(sa| z-0T$mU|k`k;?)aBR?j6xRoJWB8#~Jej=37kEpKIs_m#h+77L(0NKH?Uv`_X&{cE

&%;?*Pj6hb>G6nxu3_vM-f^S)!7bLR%jZ8XPvsgsddoW!i|RYyb;jgtBX!$9kqrNz+nh2~+=uX! z7N&j;oK;V9^Es{Ocn99;ZdT(e)1TRda%Zj;qdUxmm%SQxRrL?xA4SkJH)8sfpvcY=dGH#k=h8tCs0`2v)fy+nGrQ?Ygc{ zIWE6zuFJWHqx5)MD;a(FisB)$VH~4I&1ZH$EqD?s%2OMfGi4S|W~;MZ0OKTWG&cLFLN zSd?rkLNbvi>gW7Kj%`7G=TT+%r8iZ*b;TVZLl8T1BfZkPU1C+nR=df^r@*x$vaB3q z9%Hy_(%Ru;IZOjN3n!-Q8h8`G5}sh=rzO0jUPx}QjMeS*uzJ{pZ_jiHl^t;W+}ap* zJ-aHaEHCXAWObePdPdjJyCMo$PLT;I+Jt!3tAqp?Vti@iUS7QkQKG^8$kifkyACs@ z2GbGyqJv+@=;qbQu6S4}1&%4VhynZM_R)Sp(!St$qERg_-)}7lWy?>`nj-D2nlma+ zn}K2Mg184d-2WoQCiNlG;MC@S8uJ*w@Qx}kj@UGm#a<;#z zj$lJ^Co(KopYPif(jDH9Qe}(jrcK9*iAplq$!*BX=gTK~)*q<6k+{r!X69EWGZwFz zi~-(b*)@clRU;}VPe{cJ7b32*}FPi3Kgj2AJ$br3Zg~2r}^)M94EOM z$g}f0Gt^_0vu^3fva>wLRxkDvQ(40G7jk&9fCukm6ZvqzUkEtl{qiY_kNDai?y43Z zi1$xkhjYSzv>bovs9uLXQ;xVva({nX6+VIaTT_yt87M%36FOxWe!9~MpU|iX#^u%8 zi8Fskq8-G8+)kx2Bu|WbO#hg(Y3q;e@=tk9m!K5j~T`{_&bu`1;!Md~|TzYecHt)^UgIFGOe<}XbY%2fXqs2gVgRe@bJZG)zuAyqDc-ma$y51-EUbRCf;VxQiI1wOk;7R3m0wo-;57v)j0U)6Ud-2Ghv|4My z_QwU!CGyHGA62OYr5`4Mq-c{y&j}+1$%5-oAwRur3VzMLN-p(Sj_n#Q#qUTYTfB(H zaxmQmbYm9p2QCG=PX{peq~#Zq6=QaQp}q~)!^eJvaxt0W9%a2`mdiEhybOiy1amfA zSRv&H*xjY(KuuK(9#y#YSB~_3bAPqYZ2@md^pTrXARpR_PdE>NKTt#t4vvFvnn>g2 z&hS+JlTv*#*fnNVzZY&~pKYrRh0a?O3dg<_oi8;tM1B{&MzdYT&3*&_b5npM3f$Dz z@0)5J`YbsYeOxG7KmdNE{E>RML;GO()cNxC^#!II+Z7S$>42>aWfv}0sZ;@c3=$Oi zCW5=S9T( zx@c0%C+JCDA9qXgbY{?c`r2%Phs8y6Yvvm8jl90cpNb5n`^WR4A2_8sxk!c#ZgBg% zw&|20_!#O2*!>eWsSHnYkR*}@EDn#^!a+AJKDu%q%qLUq{7Q-g(+8zLa zNB|S$ZIl%c)JIMjAD!dz*Chqh%H__XmB7#kh@+uWi&#EpgUC!0852XfPoQ{wlP%{D zr8ycCz(iH|9C>B33=q={-&H(f%Rf6Nk^iy$d8{2{sV{i{>WLj7^Cd|CssT~eVFpTv zkvLEhV!>tkXM{%m$&s5@&WAk>wq~nj62mC{WlCW?ZLvP!^j#X7iv35p#EBf{A*Ttk zM7O}?W*(ortI*qW8A5V_vQoo3>HcwDFG~)7zl_l&nMcn3GgMvf3ihyXA_+z+)P4ZC zi!_|<&P;5dn31%(a_XOXnI{o0fCfn=Jp%P%u@tRa+*7hw-bDg~12WS~`UUf&6u=r3%2wN6Q{K2?Zq$irzNNS9Le6;h?PVhz;HRO8CL>$!4k%zj8gx@*Le_r?<;$ z;k%b8(KgOdsg{OC3g32Y1gSwX+assvO9PN@+jPF|^X@B4_wCRGo}2R5SA#n7ViASQ zF9Zv)!7`|hE64LSQa1%@H=AriH61AMuY$E=cNdBG+4Gn!9s9(#y4+~bT_IfV*SiWV194+#UNJ+&}91fDsLq1-^ zvd(8c3y%aZUJ^!Ny42a36!aEip1LXc@XG+UJn`OP4i;EyVEs(xh;3JF%~c2V+@&p= zKC%|n&N5);w{?q7kB9O_Y!u}wjT-wn>9x;AdhNO`wtd~ihi)a#{Gl5mdZF62>Ftay z&ICy6&6rZ8l`jVFz^n%t(*_tfjqw?CSrEd8D=xA~b@xR>-7vFFQmPCnrsi)wjVs)m z6Asn~{pvbdQ?*1F(aQj3IyGF?p|AC9HHjxpfA;LLg>t`b8DH|%*{w>w(V}t%3ZUHL zjBfnu260niV=+XSbiDT&Qa(Oz#Eldc7DQu~zs>MVk)|eW1b83`ATxP2^={&aU z>o|`x<$gT(JeDgvKwR6vM9Ti0=h2)EvDIGxdfcZOjLy11DOjxUd%u$gt*hh4ku5Eu zn?nLTt_+TVr9N_$FMK3|ZUt^6v?JphnSw9iUmWo5xqUKcBbVegfbn2%5Y!sqyXVDk zE%KIl9hk3Pb^KOnnu3(?H@AiuWOMJvj5@9sh&kP|SuR^fzR$w{q*D(Q0eZl}A0*2x zSm2->rf;2D9lCt^cD!X|j|lwy$?B_(Z`WMUi`|I-R4k~~N+dtQ(!Sn1oaIalpLlo8 zNe%~O0V2`*IUf(}ZZq%a%($DqP9=`9X7`(t;v-Np9{$*Ki~lrN?p#_*yg<%r&(__q zzi<`+g*{MvNUj7=!_c7Rac_~Bg;M)OS!K$u!V?4TBX1JBJNOZSacPjzTMZ~pzWZgL zk;>Fn1MXD5Hx==}wk~Q5ht^I-Ja!v3ykSk~#Cy&K->UPY zPB&f9w;6b6i@HXwUN8D_>EFcy|>PQ=%hs zpK`LlK2P45+Z1T`tja!VOEd167weMDhuS`?w?`sP zd0psqZ!Qo;n4&X3G}3aKBm!U*IJFxh8#2cbn_oi&{Rx9IpPPvBQGr6lO7SvfeI|z{ z>UR0Fgm3D8Rl+E2SGUwO=(Ljd2(gf#%Z0OnZuv6mAOjbzDWL5Y#EraOTj}e`)_Uh$ zYG9<+u7epY#kt$)7=^rkGwpm!Omcnsl0B4jiUG$Ps8|wmlK3_KWb2B*j5Z8asPV0o zzI@)JO;*R3*$LWvrA2)k8NN&haFYJC7PDsA4de~y(H77+l(Q$bFoEa5q~dTYq7Lp* z{fp1h%-Go0?>s^w^M`c}kg;IY^IdIryHB34J+7fW1d*bA(vzWq`=fh(14G1qS!6s0 zMjWOH$B%ja7`XG1+UG0xm-Dpd4o;Im)2EmGkul-wQ3qQbJAjKI@@|n5Zl89w!uvXS zt$XDIDBUv~cWcw$AiTzPmQ}P9s_fF!#G~yb=N{Hn%urhPBp~o%`NjbQA21inj^eJu zT_R5iD|l1?P%sF;XK=Bj&q*oy_9aYLg|bT>s2;DK_BgA0g8OXtJQsk#vM=b3gOx*l z0OPWJA8!&MxBqUxe%AI_L)HeYYR1i}JkKdz8duNucA@oa;y*WT$?1JkFtx2lnc%%^ zP^8!Zs09~{#8-Pkvci_U-$N+7eMLl{Imq2Vy2@N2ryx;$3URC;3z+4Yv)Kkcf~@U3 z@Js`K3EGEIA9O(s+%`y3_)epSUt=8#JJ{^%(@6VUO7!Ms#@Uth?!!f3YQl?bQiU#7 zV5*AM4YApU&*L+q#ePM;f0B#CX%;W^t*Y|D)U%71HZY1z>(5_!jbEe^^SZCzd`ND0 z^SqTk#;zSs_)($-+wJ&Cj`<_p+<$(kuxDO#88SL4v@=$5f!2|D0xy71_|Fo%1U{@c z@m?o|jIG;xVm|Fpzki_@wUE0fT`qp-WdWKv5#Asvm@e*J)9J~QfDa~=FN+p*rhFaz z;P~1IT?kU+G-L}KZzyUG1PJObXYLpv72w?O77giCwg#LRh#PTL+N0xQAsh!k@s~>u za}WI#0}lj+nOTuif30b0sT7uQhA72~! zB~AD_V#j-%HKWA5r_M56gs9NN#?~Fx{4h^ zOZDAzwR7LK-Yjn}M~g!U@O5Yhd?j9VhF;>v(3;?tG5h{x*=|!|qjeuDppgTHDqwaT zO5gcGj2^ZVy|Mr_MrEId`9WtFkSO1w>^-zAU$M#c0C#QG)L@YvFXf#2i^bR@6^ne z`9!m$HyXxxj}9(nEFLjT%Pw1*eFO;8Irb{?eJqDdPye3c%WZCNSd_78#=*fur-)30LQOt=2|TlvR`H2t0pWAjQ? zKzsT?&@P6rceCJMxrCm>ru+gtI_S(tE@OR&q>@juau)sLqUz)}Ko*y2n@WK^Ps6C)W0VI%2!l_*RZqJl-sUb{mAS09U(FemT z!D-9E(ip%?V^bboS-q)=^?P5+A4?oAIz+2tNm&16ojpO@MS8}t%$-x(y^eU$yVhu7 zz7J3`0IMG-F)%tEl4G1V+87<@g1D8MiDW7DWCFt{DlG70*O^Id9}f9N?NCiKO7jJf$nr~_Cz|g2f4a`}}$4_rCjE-pLQr{7*q(+VAmzf@31I@KmHabp|8iN*)r$t1C6 z>clAg8C>q+qRe`vZZP?M6^cJ8Ax0`3QFxU+5gX2bw_OXEMF91D$2jyi=%gncgq9yO zPPso2e5Hc|cM<;+O5T|cwbCT<(11(r_B_ySUKYhtaG46Lo~c*(CY|h5b8h5fU+P0;raIp-f0sQ#dG}_rxfnS#jlaL~#b19tawtb`Hs#K*sLOWuwOPiu0(vMYesKDPR9A4Lll>j3MFxi!S zZhzBP% zi^NJ^|MvB?V1u7go19N{_r2O@$d8WsPVv^FaJ(XyML!@nBdts6_r3~oe6XR_G!a-d zw(x7icVMd?4_*5ofE03Shk4M_-T_Ur7%)mf(nqtP%&m%1YE)KZDhutO6TKvKGu%92 z<5;^SiBx(~wF-CMSx(Q&iIpjw{=nkE5wu)OR-Mskcu1TpXE+1QB*HGHI_CG^CSM-q z(?6rmkieUa%y9&U%QO#{;og#^t|Uzoe6X)!Jz(4EFR>Zx$lZhVFMTM3jOWnjr<}GB z*$f!Cp1;iwmASYG4~I$zD7|EI$Jp;`O_ZzoqS?=m=YGRU6c_iKV@#~Jvlk@91d32N zh_|39c_F|%#q?xL0sE%|rtd#_E5J9bA|EOMepO6HroI-LVpj>+m7kW-ukX&JS;fkB2pS;F8&5Eb~W4dGhJ){ z0qwVZxQd`K&o6L8%Y-H!H}$|4Q?~#$dnJMcZ(UemyD_@KW$y@&(vMT1{FtoH3s%hwRaSo;UHvgHOTb6fdKN zzk!=-Tl3i=^a%@sC%*L0WJ-&!`lNF^b=}7j4v)9nb#K^v(V^2wJw@`|b3ceMF0{2SrVzAmQu0a?l7%^rdLp z|Ge(5GozHfDLkf?jjezZ0nGGcW>p;h&UAJ3hRfK=Ff>*`t)N?}%lRxLF}UaA?~HW( zDS=SR!~4b0;WbWy5@8l!8h#`o~SBKLa9 z_Svhqs;{nDqF)+FUut}Q+SjV`CH3nOj#Cn(#l_%7QlQ4JSJHFTDMa@%MvuH^m$IU# zM9axzT0ejrHq`Va|7;PH?7E-Zd^i*w`#?sFv_)I;axo=5o6AnAU)L5kg?Z+uKjd+{ zCi1Ve_%RE}*TCNTYivA-+Yh~lX>%s6il6U_R-~{+w&ovE)FkoD6aUj@f)c`)Tz!wZ z*Tqgr?&(_J9Ka)905x>Cz2|Xxw1_X|yPv*zjpW-A+K+DTbBf;bhJ?CV$6>;^PtX)B zDQ3zWoV}TcXj8>rVS^2wawQule!&eOYwf$HVbs~vB?lkYv-dWOj@x?Cza^CuFCm?gZb$q6+LM(F!dOz0x*W?SAhV zv_3D>Skd@MG8?=kmFRo(4(_;XWf^z?f-x_=%>jicj0CgpTNg{WCwiejm&E3N0lScA z!A@m45jS$QD8Ow{|8~WqStz$w z?jq=44OmZa-5GM22ELV*{s`K(-uL30{)IH>B^t?2^UgS6rGI^R|1+?yr|XNX$FTy8 zaiT%`K78*14;DDUeInE_m!XPFBEN-}a!ONu^g9!!H8t|Am&{Z? zcq?cS~vlku)i$% zt(9<{ackK=*DhcdAP}&n9>5#>(!DWSdTQ~?ZVSq4g!294VM!9>0M{J1fIy_w(GSAy zI?|tczUokw*6WKlA=7@stKjnU_7S~v6!PgsW>)z;A9F?5ElUI_^<{f3mb&0=FtKQrdsqDmuc2wZeW8j(p$j$GfTz-l5P*v0ul{tpOf0lqb zvd!vz@dfkySm35&d|HS_7(@%cSHf^kwUdU5U!_r9sVmCp!3#}F| z9wNm15&f(WE4d1@)yUWN=^cZ2@kN$MU2{ql&nN!{5>9_W0_dT1`_Bd4`L0Y}xA5C} z{kXt9;=J|8*m52x``~VzcBj4n!`cDt1TaQSy;164ANl{^h95gvJU1GxUjK`Kr<>?q zFL?rVhXa)A4maig-bqjZIOPDF{d{cTEJEHWTxg8I3g(Qah5~-DMHvmVNQr>;tRyYl zvF-8Uv>pD?^ol7Rb>^`xv`p!a#C<*s0l9T$&Kk6&w!S;d{i1SFkOhH^%Ga)Tlg zjmqKO+SS4%)^~W{cbhkb!zkW_IW6o_Dw(nxqsRk|=Y#ehPMs@=O1psw!f3U&_4NI| z;YL}nU1e8A{Q#2AwRD|o#XE8Ge_GDJstOJ|P+L6^MLZ6iD!km-c*yPSU$lFE#A9n$ zm@rBoD98MfW|Pcg@#LTI3fMXR&p#{>>g{`i1Ma+*VYi>_sq@VGZ5seaNax=u3uZMx zSslxHlDl~y@D+Ig8OS(MafJdG75w?~qpAJfvl(G8RISsJzOfWej`XBP$SMxJ=_Sg@ z&m&^~@ZKad9$`e|J)L4;{uV=}$HRAY#l!X)we`|Bz8bBuvmM7t;hKt`q2db@$6N14 z8tp`IdK5r*dHM-I1cE0r{znvEGIx5P1C;e z<%*IQn5YKC9w(;@b+_x4zSe9!yoW+c|D=k5>BC%)y=X?KkD0@@$8zAbc-T*Z{4{6% zh3f;1Sk(gL`b)Li0AuRC`!isoso1+5YmL)&0Ge)rf^f=q58n^=<_9 zxy=;fM$kdrCKQvy88dS95<3TvJ@4DKR^Ww3)|dj^O#i)DVl+I1-s&YDv0C@-xnAWQ z-a{_w1Ud9?S23y@hWTAMAb!;I|Z@d*8&;IFn>^$*mcJ(VA zW@Javjei7uH#Z>gMe~iiKrKVuE7k7@ugJd|64MR=EN_NyFM6O^)=zVVl0$b)Z%^#T z@I7S}wsv#@2?Xw58msVN%5uRsez;dUWPkGivG$g6QLbzI@UmM8X#^cghHelLW9T8J zQ|a!IRzzinA%-4OVL-Y&Rl23SOS+^);C~Ic*4}HcXFu=r;r-(Lbe#LTuR8NMkCUly zF>%=G(iC9`hu3RqnmBS#xjO+QXoZh1s!NV5CIPwaI3wa)fi!u#Nw?C5XJEvkbI z)}sq_xd9VbE{72|DXbP5lK0P!hQ*gzfcaD0uG~(^&*Eag!$Jam0u38ajNUIRO-m&d| z+sm@Q*=X#R%SS5b+PAq{@rR#sGF926ghY6vNIRgNk2^gL4(8P24! zL6rrn$JGFs7z)btCzc+^+LZS1H^=Bs*4J*>tn8?U&57Ww@AKL`1WzEZKU}@sgdkKC zH0EC2s&x5gl2z+b6LGJ2P+t-twwVFc7Z-%V>xsTWfI|?~`h7rhNt&Kf9Ct04qnMYG zEEngXwb#wZV*cjx0o`?zx+bBs@|*o8y#=GNDqr;vJeBrXABOBh;Z?~oHYW3rgtbpI%M*K(EcZ&uvTwu{au zPVAcxAc}=})~L-xQsX+N$Q391)D!@hC^@fT-pbq|X#;E-|Ejpb$$|fwL8NJ;_v#9u zrzzzUn=Dm$7SvOjyjbzS@`s!v^nMov=ZZZd6$q?6K<2R9v(lsPyd{ZW@V63gB>4X*nNHS(0D>O0Q~+67l{i@UOYBNpr@$4L#+Ze|ZkbfbgfcCH-l_ffoT_6Sh6 zb!Obbr*G4Kzbt&izk~4Q-++~2QyDYs0?SS;HNI4Q(N%c%WLAsO?~>fwQ+DhOWIV6* z`@+q~NkYPn^}k*_ z(gXaJ22jkE2Xbm}S^u_>{|CyF1S9xEw5b3Cyu}sgMfm98+{w1B+|4b<$lZPB8LJWB zG`mr|lUh)m=s6)y0)=&(A8@s3K6vT4`+|4EF9QJ(^;Cfys_o8dW!ojmnc!V^l4qwZ z7@*hsrU8sT-r{r9xwkqkI?U?c*_Rx@&c+?w6B#cQ29Df+0+qIe<>{(&A0IemsKyV_x%q$>Mvv)UbS`IpeV^)gU75#jRP8(6RhC>x{9}ruT^Ow zWZZ5(pY?V?eLrBgTIIMHQZJ7a}-V}uZx7$j;2FO&3yXP(Zw#S9<`Ir zo{ap8S?a6bp1@@!o@7z)Ctv$NOwe(v)fttaf8WOPGiu{$XHqv8DZk4-VryY5jjDx{ z=sk(CdG3df=z*7y`irh9r;AU2RN>hbrfg>jI?}E(5@<4OT@nTeNnmRIug4JVUEiaf z)w-!4x34d6u$NUF;TtitJbrCe3M%hF(vr1ZMSraO={6u`V(cm_+8)_0xqizlhjiW} zf&(&*-KKg!`kn_@x$|4xozmbCyU54jf)KvA^zm<|D12g&lee{Kl@hHM^vr%(T=XQ` zP4|nh+*ZzPkd-=sz%0vvfG6=xz{|6Q;U4qC)k3$18H7;MLc2nqPV*u`np)p0QH&(j z(Y~HE^OGV`y#JyEIHjKP6T07y6DPGNXzkh))~m~<{KJ~w)?M_|Z{3dN4=}wrFQ;+O znC!wpmS%(8??G8(G{b!~ttnnzh%TB)Xk{8kX`7x4dhXCUut@dGd|@t%SG$={ukC|4 zhmmbu)hPJc|M#IGG;YYTnIyhl7Edu9edEcKVU@yoWYApLNY z#z^hEW>C3b@|-jAE2nb3JZjBRW*qtpzr%g~2DF7|NwS&u%l4P)k}1B!9J|0{B)9he zI5bgtwUqnn%FM^=(hjK{{ zyRrWWdApsYKIqJ!^Wctj@~?B*qr+83>c)I<%`#%;qw~@9|Hdx@gUAS5-J0|CK2|1S zSiCA4KTbP&a886fZ*03HNL|MobM8|3os0o~aHtc5WtX2LRji2~#JDNAwZ}pvL3Gf3r z;pE!iNX8UxdFiiA=eO`!mW6`EP`0Fgtio?4c1jll6b4$tK5+P_HZSkpQU`7Jp9@`` znlX+wr6W>poFL~_cN3??{ziYt&>o=g(x-C5{NZL1BS^4|Qj-JGPaH`cC=LXC5I1q` zYf&B}RO7Ghog4ZQ?1-X+I%@fF%uHpB;88sbm-46mOes*h_CLJx16yJVS%~#E0m`lP z+E7YImgPG@ghhxW!aC(+0dg9Go}(^~S|uq|HYl^YL+1(YF-z}`>XS(Xy6_*10QoVs zOo`lC7Bd>oY5%mBb@_ zk-&;JSSC)#kP#VBeN>Sv*jEbct9SWTJpN>ev?9?)UpP@U7WcBdPJeLVmL1`h|7`#brC#!oq;uF6g z*?+FjL#x98dm{->uTc*2)KvNO*tOnX{cfP91WzSjy_wcx#gn>I=`l5q38*P%BOnuAPn?!>;8?N> zewA!sU^GLCbh4~jZldYj z9@;6H8*{Ur{>*H_B0QfzBAL-P=qIeRMGl}`Cq$9|Hi`4|B`giQt4NNJ>-OTRkCW5o zx3Cxf%j;*57$+~YeBm<<+4x$i`mwF@4Tse#@yv>#X19~cC3=}AQ$;%R|8a<3C3NXD zR~3fglD){6<37!z|8_o~K-*EFRAM{cZ0Me8qk^hr!JjveFYAZ6PDIf=O-cY=^B>3O z%LA)I&gE4(n;#`c`>xY5sRcRFV*it-5LK&Hx0J0Zw7ODXIdZ|W$GD#t2!RtO)_(~B zT9bxMqnLc48_r#FY*{RGvUu@d1lBr|i{?|V;lB8H-ryC zm%7if;VCWc@dRe*@a6NK;w`3e2^r*?C%m=}?GJ^d)9ChAuZz*SCExOmYh{7(NyqIi&hAL96VeGCF&NPLRk?GvJB#eNcV}pY+gV#R}8D$66Gcv zwBB-MXU1XH%WSs0(&J(GRcN_bQU8F`;4M9o>-(X@YY#F%)*1E*M{W`F&D9WO>+bXXzn{D@0Sp{+*y&2>3u=-|1qm;+vr=#z08q;r_LkI#VxZXg~KwO0#>sW0aXo_1%Z}Ek8D-mk6iZ0 z{^o0cLA3;j@Hx#&GdX3Tg4hllUs~k(&siF&miXY66PysKxBjX-k8r=J-*IKfsNH+b zj7};3#a3ZLvlV)UyGR|7mWR}&bDGnID%l?HKB@xD+Khm|JY{sw<2Y{dPwH>i@3b&1 z%xY@$D<3anW?H2xlZ*)jsW5zYBT-BqG>oA}K0y+nRpUez@3(=J14D4OA!8fl+``=Y zYZ9YgWJ+##eqmlSEBb^eJ$jUTDB`%%W=y-#&^INRjK-3s!{7jA8TMro{g(;1UO$#Xu0Ki3^XRYXd47AdB@`T5np~ zOrBguoZllush!ZAb4o#ow4IiU6p9+OOzF`p^gDAiMU-r0bf^DXX3VV`C5hDtP{Sov zv5#k-mb*Xe4> zG-qY3R-+ok3Z-X8*oNu#^YOu{(c3jwAIwq%MB+DEBlEvVOG&P-Y?0}BIlma78J;Y+ z%&Vm33}U_2rYIE*PVH^f8pTed)1DPYxykht&e12AuaM!a-_tGwQAi~lOu1!&zQKW- zDX|M>oBBepWukVCRQ@-dRJ-RB6of-(oi;bX3N%K(uqSqZKlXIaq)xd6GE6gE$I%8M zA>+z$=(MmnY)I5FCCX|91KPs#zn#P${{Da@9ML_X9KYsfASmImQBBf5J@REL=Y1Y? z1=P^z--o3I6jL;G5;^U{%5AlZ&UHr>~dGbvmtopB`{*+y8%kaG(8T!!`bw z2D-4h@4MncZjb+J;6OlybJ2j%8Z5~WpFl@AA0Rec+8rlM|K}$Crp5(Q@;Wkq;_fI%a&}}YkxWOW zv&h)Ko_wuXv7DkLgR|jy;akDIuu<%2-w(H9e@-Dn@zb)?GcboZ3 z!?gyNl|-LS-~#-9ttSK8mT(j%P9A9CEGzx@~?@U zkobcq;1-@me|o1^hd?r|)!{2+6hv{In(Lr%HQ!+A$7owUaomz4nFu`3^oo&rc5*UJIT8eO_xLsO~hc z8w_Q9RxuekbTo_$-Zf6d9~JP17EUPKM46uw}95uo6+`oS#Y z5Lq{vF&k2-$f%-Maf{%eZz^q?v54w5^3UgVw#{Vzxf?~M@$q(RD6xV57Ndt|$?y4` z{Eq1)YWl=?mRrfO-YmcVN-BSPYr_!Pv!LzD$0`S9ok5zF)d^XADXNrA; z6=ciYJvsnf<$o=d9^m!Hy_C=S@iZ=#=nJwSsdUdy#O=Cm0Dr;Cue0?Y=yw;)xpHcC z!##bmpT?5G9-gmVXg=qT+XeC_2>Q?<{5ufK7pHJ!^ia+zM8l$(R^FbQRHa%t4jwl zZgFXYPgg;rkzExxaT^W#_RU;_Rv8=q_$N}GtTJwg%bJ3>S{T8-zHxU-%%G_-P_Eb# zW(JFX)OjPDxx06vIIPT4gjWtcAZ77AN+)+CL$y zGc4|c(-yFkECCP+jPo5^7oR*c&E*$-=94hu^>Iz4TLJGCp1!KT51tpU8^=y^#RxWn zVu$X1xsJ+q}jwZXB&!JXB;sx`X`*f~zm$Xq!Bv{KpOEoabSq+XLS z!o|{@8%UPTCZL187e@kk_7)nghDFausmC=9oTxYi2ydPoatm4Max>jS1V;cSRG-SJ7ip~cnYP5Sv8=eiFs0(HbmLR2$*ttCweUJR1jUx6<`6i zQUj5jm!~Li?(W&0Nn)J-_!hXuK0deX3)P!PgjR02m5Ua24;YcfLkTzn(}@skZdlb= zL+cUka<(qvL@Tcq;#46n)uR>Rm#z7HXataf*ic!}s6>?dX480IYSGnsKe%;*(1`4OxCkXzqx^fq8?3VNC95SFZtfvnIEu z@T^psNslB{aBk*j8B~C5=|q5_f$xWVcSzLYqFI6ED7k9-ufYWek+lzYv91pdf0+E5 z?61y>jpNokpH;QYx%;EgOe35+B-E$6z*<-E(5-iep<~P21mDD$x0O+xLbtPAhf0kb>cdRPOZi*Ea!K*u7l8@TyfMjU=yiPxrSK;*8V zwKfd03U1@45334_)l+j92?jUp32KMxGJTLTLgCEq`T?seCvOV~>2XpG#!SMd13T!; z>=g5fXr6^`YEd2E4UFTh?uJ5!W;U*d-Gr_Pbr_EAq~IIZ4OGukxNGK$56?w+d0SXK zOtQWPFff;|+(I+AL)0h0ITNXWDB*Bm+7I*N=L)n|;XS6oc^)UnIH>WY0m$Wh_;q#E zUbXM_8>UjL58GJEhSD1oH?+=2`LaF5E+qTAZw#5(kMk8I zTq{47yVW=7lD{hc0P&FeJMxp$`jcPaz7RH}G5-=iK zv%Y)N)`p)lN-MzaqU={>m-f$XOC0+8>$wZEFDDMDJW(D&-*>vR0Avs_A%G?QYobN- zKbk^HK02Bg?-4A}b!8;ZcLo`8vSM6*x)PyZaIyq))?+$_6Em0;+k%WBBddxoT1lUp zb(IsOV~C!&sCsmMR0%GuY159cZJ~Lmw--qp_WI?1=_MQc1OD1kf!o)loXvG=sDMzU z6_TIuxm2|_*)VVkVI}f3BLDt?;~&vZu{{O7I2wRc)%vGc%jNkLK;0mdsaakZsQ_vS zUWgS&N@PcKcS9NsG|ms{D#1b#^5u_b$vf;#Nj~m;kg^VoPjXyU3D<9=^b{#;A9(Bi zRa>phLAXkxVJFQ5YAGJhQhwFC~9*2(VU z=~1-DKJ@P?9v3@4Sha8AOeH5oIQk*sem{a2G?lO4Gl~mOB@s!ZEVXx4{z^3>KaN{<7A)jZWU=Wqr1%f>SWe>HvdJ%0 zXFn0_3+PUUZGgrQ5U}X}qP2sgC5x*EHZoeHk%_kbRX#JzcHMS#5)0dxdY!1nGWb_9 z&Nl_~>!$;bq;ry-1lIns;z2q2UL90g484ZXD^#|0ql{yz5dA<&!$^3GJ;Ik=Un86^ zc}<~ITa-3hf?HizN7@GiKC7aUDE*xRq(jS|H&TKNZG5N{r-bHcA4;7O_vfd06*~2( zfu*GZc|}fdZwP=naR*Z3Wsb^((%pk}GSVL-AXHu?Lp8aFy7zB1%R2qPOS(C zfU@EwoKM&@2HEE$LHAaX>}Xv`CCskL-X+vD(KX|vsrtB@DwY3sk_>F%5ZZMku`v%8 z_DKmACmA-#1U(e)FwEdeeeOpE%5ivZer59GH63!t83I>cR@CT67BRiDx>JLP69sP^ z&(3r6w!2+HCxR-{IaMN+e48Q- zAywF6QI}BLrWu8L4nhtiI0cbmJ*iO_eKuY9rbetXEE0hilG_T4YO=>SsUI)`I76Ww zWpozha60TgNYzKaJ97&>s^6JE#avZD4ba%iy5R2?1U}W;m2AxHVIx@u;}cLZ6Y>L; z+G@M>q;EMzhiv;371=9xMXTE(1P0lAl;;MN^C1%=Yq5 zGF)_7aHTCIGF@hBH-mP19&Qp?Y-tpOWa58LaAtX|=!REV7FjIMrZg=^g6*kyQ$ayf zK_fT%`8(PWQ?!v-5(fO;q2HuHa%Q_MMd%Y`l;dGGq^mWBKV+XRwPLPm3HnBO3+vaA zLSSs{1lje26*w2vWOm&6Jp7CvGV8|(rPkZ5rL7ptCd@LOO`WrG65S+rbLzf-A_>&! zZG;pj`aMV{1hw5NDId>$wY+VN4|sas*h)KAY*$Ombs1R&7qLMs_Scmrth za2BiG=~L>%qc7-?#mqlGrn!gD=&&2#On|uW0LV1!6{y77;DDi zrm}+>wJJ>`H{!tF8a@=XR19sqdHVhgHc}i1vr9JWU1qnpd@_Ew%Vk7qBjJ9q5ZeYT6*O<6HWqU^U&J+86t%5@&uG-n1r=6?C6j?#iYSyR zy*BqvMVF_BDaJp~5bcu*CP>D(*yUIV%q3SKBH}B^EJY&=%gA?ywjg32;Q@t)JCyyc zCsr)(GEq9sV?Q3mPNUWBWV=_RpAA%O2RXqRZ~y?1izjauSk~BhxVm`An0Hk!dM^H* z`6vXn=y+}Ffd?w=QtxU8ieY zXdOj-dLjd77b;O%#BD-T`Y?m4Zb%O;q{qO;G^C&7e9UDo7a6osV5%=|NEWduQc@MQ zd4CHQx{u_-w;)KB*dj9KWQ}qlAyWpu15ONtH-%^C-7J3nJXgX0;@oE8${a4?8>r1A z2H1Sbn?8C-Q1_7828>T9v&toy?@~#W%ymUU*LT=nbn)p<^pez1>jD&ZY^(Vptr#2I zHst{WTDUYqV;A}(Mp^N=Q28IH(3ft;$as3LFVp-jMuKA@SRq;Ton%s^5(@Ey;EEC4 zkDDv4{U{9*W|SAF5JtZwUF`o37P82WHtI+NbFw0vEd?wwu*mIoA*l;dusP{oj5>}C z%sJgb9uddUo&W$vK(U)&Xa}sceJ>P`8@|oXDoBtYWyPcFU+L^lE#kU`;KcWtndcl` z;v#R%rzjmq>Bl+oLh@Ra56EVWy^wMJNO(E%=%>st-U#at^WPXFxm{?>3al@Pzo%{G zbsjgl`VV{M3c0at>{nw#hAFvB&1v#(ej`X7%!dr^(q$wRA+A3Zq*q@cM7~53$d?LI zi&CLxzlL)TQ&mgsq$Noq?^WUe^br$eVwH~3hUxG%1&_$>}x z=W}W4d_d!CxodYU}!7fPyGC5c=lHkng;!*-Jd%)z zZITx!!ReAI@{&r_$Wa-c#<5_fPz#Jbv>vE~g|wrjIMGIVC?Rsoyk#b6wrjHCpeK|w zYj6n)X*YNPsZ*cTXmvd@b9`7vURkZ-mp4?jqQ)^9;5wgivHKy88g%Nd;n`0P!ofpn zTkV;IL8ap3voX$69LB4}$o$Kf>(p)X8H)cfZ~0*fYLNf(9{}A@czpa?R_nL1$QPvV z5D7w?jWqo3dpYyV;t~$_A9C;#2?%YYKQ+U#5f1tH30g@L;dY7frs_Irod!ds=Vay( z>>>U5lSUGJWcJGnMEFLK7MO37ed4g_(-;J4h)W&{1~$q_De*+;5E*w>CNeW^B{jom zFiNqKq8cz{{xfOz)$qPWtsZhOOv(AAiMH?OVrKbplAAh%kROZQR>~0`Q5ZeceIhSi=I*} z0v?i9P@OK641vI*wZO>r<-&h1ri$ufRcw=xBhn``C9&PfLpE~Q8%7m0xu-rLL=&T@ zmf`qFr`EcR>^Z$@G-R!)*#f?_!7s+5<>T+h`&D_FjUs3%>P^$R21O9#tH_a^i1|{K zaiyfvoV@Q&SK+}N^VQwhu2w*coe%UK^Q&dX&oQPU9DB3#ohMz}2yTsRf?BUCKRA-> z-`vnTMOzgutz}0pbts1c6>xzmINLF$c`9*REoW$l_&t8y=k++`366KY>SaAT#8Qft z7InFOh8_aPVo;h+@kj}_tpa8SZLa3L-4tK>5GQrk#~^BRnQNh0p|?wf0ts%H8;P3Y_{N(fBe98{hWazW%E+S-B*EUz*9!#w zGj}p$-_PQjGaNH$n|5jmlB`}5>2C4iT`iiAvq~^y=t2?+l~GPKydtyW7crm z((|FfWg>A=M1P5H^a(N%OKWZ2v{)XniGLgK@eb&9r)%u^FM#neC0z!<|Dc3i z@gBV${CrfWWg;BxZuklo0!|@)-m)i=i!j=dGaQq_(AAv6Nw`5rWF#hvkzmCz;B)jn zO)5t+GwH!$QLSzxTPDm5_^;``;bn&Il3+0!c@u+Wd@!wnQnpSbZ|}T!ntGm;+!orU zt*b9Q5t09GrYfx`;0yVeu5c2jNfH0`136lY0mjT`3!X8gxHSPj6DhQo)+Hl46UouR+>W}+?GoCLXG`B} z2x&yE-VyyEX;9T#E|U~1RRCON^Y_yWY<*_K6`O|_TMxxmf5cT1Tyj|9#r6K1{KEPA zS<#)QTLl|so@UIa-QsWuIpZVc)46rQ36oKhXE>g*DuMT3GkVn|Y?FzU?lrxpq-2GD zJ0)1hc?kgD-r8#f=>*eNi3IX@WnV6ike~@vA3|%fP&h5dMFZN_+D(9dF=vXex49<* z1^sVbG<-13=DvLR7K(@+J|cWQNHrEnoN!+@jIl2#LdZ*P7QdrFJq^Jf;z%dv><^hh zS&@qdf%A;Tw1yzLTtZh&((_pQYB9}K)|V!W+hMIk<(k7(r#=G^NMQp)=^`5GOUWlS zRYM$cC?W6Yp(0QX5ClA<+a0#!p{pdjv9OO;RiHqPC)kcecJ+#q&&t=qy2Oo6&WPl; zKVvC%1h({}VPt&%NkhQP*1M9LGb<1TyQZ6rEn)f!vqR7lHqmv^K{{cXA7;!+9Cfb) z7AlGT3GWNpS3D%{Br+bi7F~^ElvvymDV$HBpGB%}Y)?WC&EE^0SWe~_WEtl?qx{$Y z-p^0{CAhh2?RQ1oF2=dp3ly0Qw*lU_FC@lW93KzwkZ$ynA8%$@5IVKG<6p^~`H91$ z;pTh&1%kh&u`#7`1JZHj9&I0~pp(UlOAx?&a$ZwZ9Dzu0en9#%Z51>!1Itfi*DX0o z8yhY}A_;8w`b{9Vq=%AZ`q01-G<9J~42J7Ta3x-qDSHtrjyBsgLA|9PWk=qp$l!#} zvV=W`D$39k8Rx~pYB9{#65OVQx%AWVO^J=wEy@s2RQ?*@^Dot9iIp@rEv>7mN8^`= zBwVLvIhrEN_0x~GE&lTS**V{d`0_UtzO)c~5#pov;kT0_MW-bY9fT>o7$4|HBKrV)?IHrBTZM7(xhN&yQb0SYUpp31%0yA}v2h^wB6Vw?}&#_GNho4T=wndiE5 z5?dzFRAX&_Srdsz;I}R$k?ZDUdKjMgKXb zHiNj%%tVP|Rb^P-XevZo$|~?3M>~{HZ}r1@hg&u%uJS)EMn=FpT4dMu7od2|P&?7ANZ=WPEe+)T6m!(#>WV=tlPg9{32* zt?{h69au%~_3{lo+`0kB#pl6Azld|NbN`6DBw4`DKvyT6g>k67-v!4;qV(yzA!RUM z$#MtF``wUItj1DyoGAij4zyUAN~ky{yg%3!-xMfZUoQ3j@7@a7DC zt-A=~E@6FOR}*6Bf|o9K0R7C*H6;2H=hHTz@4Ar4tbdss-&l+%3WS(p;Y2PtO>at%Z%co6`A*m*6w(7jNz!42BGP~c<7mTb ziluUg%r>OMmB~IqM2ZuoKxaO{rc5p@SD*%pF;Bw$;a6jXqd6XK@Sr+a`-`LPZ5Wc< zTSd`XZNMl03X0?E72gRwt$(qz$`!>sPFaHt?Ao|dhf$F)6dG2@{h?g}QtxIDYYM$C z6Lv);=4lmMO#FMO?)PSk&|PpAFrNvNfZ}V{7f7&f6*#!uCwc+-ij``O3^(IMYZ74C zy1PIa6>Rk)n`ubA;bm^L;if$-rm5hU768N5MM~uNv!*UYZQR)AS2FZ=J^F}5+3y4+ zl|m%x2IZe&1;tb06iJ*<4dbMeo5i7PU_40)p)8KqCDxjt#IAUKq$VzjHN! z#B^vG7fJm3<}YJq&fAM7RouFQ(=zoKm{zP`u~@1SqdnJCH$q?Fy|%f14wg1Zss*~- z#PWHF(<2(h_55ePU7g8lt&uaN2s-%ktRy$g-N~=T+h=>ne{MCgh_L{Ao(dFk@n3t`{j`K>G^RPt_#-}bg zi(*q}$CS`%oT-o8Gz|+F6b*M$n;~IDP(Ni@34(v=1A&yS5scv*$gej?yyz3`_}I{* zpNn-UPR+dRwrg;AHui{=-;kG=D!2F%sC=Ro0#UW*BAWfzygKt=>T$$05P-!U!zuf` z{k?t`=MTdymcsMlA_@J1@zbA=p7d7TdC&QNdi%?>9O-)DkXdP!7tTjF*ZNIp|G6T> z$=DthSHyY3kF*3Wq~@(cmP(OiEiDC?Sr>M`1!;HDcZ)?kbl!)^#z3XA7OK1cMDU*G0vvz^UL#v?f(2YRy2lO-=5$F&cyzCX!7?i8G$7`fEJy z%?NF4@G0Wy*Agk86~M9#zq+Q=ay!AUyMeTG`T~|k2W3fp0b0#ZuwT^kJFMXyv|C_m7Y;OD}m2zxP5Em_ikb554^d~^q?DtAO#zoQfx{F`{&qCyOyTI)g#z$sd42@X+?yG4=bu8_y zqF-M%4dl9g@+1rMqmyR0{=Bk>nP!VTMeNjG$DO3p)dCb|jT~rAMKXG{DgJXvC2f5{ zf#aqSA226iQA*(;15Yqrh%!~zO>4YvY6LUDT;_Fg7c>0M7PFV*Pl)LkP z=|fXTxndo2`;7JAUinS&tcpWLA))1;M!xyDki{9Hhx!6LywI9H$ZXU2m&xTfY&VDU z7lEG#;xw6`1>O-cU6oI<+-f_;W1rC91L?~Z_Ec0tvO@Ndy85PE8}x~-<*+R?xrc7$ z@%D}6mGaxhfM82kf9S()%Bu#H`FW0NhjSr&B&YTTfQ{on@x5?jT!=CPXiPND-lp_MwoL zn<PsLJ#5b6r0L%7JOz|dc5 z#&4Uuhs;IR!mn0_ctH#3WX16JoqPhl$I1*=lo7@RRr5K5Zk$%#zdCj)z;#3BKONUw zTKog1R(kDHU*_Z4)!Bh2j+4A?WH|h56)f+idW24XP+W zzvXXgN&>{NF{Bnd8X3NdcS9YR9eS%-3PbVAWKIaeg=(j%g84d$`{I=`2WHa_M3Aen|x!w($5s`IU_#u^|iPA$;efb4}A!gFW zg0)ulpYMowJ{GzOlp*#0 zU$0fVel*{O32}iIIUO4Q%fRPsxfDJ?Y#i38UO=@}VA=O>3T}6C*k0wHVIS0BO<;P| zDdmq4Dvygab90d4LJ|SZL*vo1Fo{)>-m$5SC)!+e>07=ra+G&o_jpI`D=+5Dueb|b zl2L9sQtKxE1Muv>dgj|4HqCC=#h1d|UwDAaXX0Ghl=Se48j5a3HTu|Sfd1c_Q_ECK3>gQt7TT~8H6$)famk5h!1wqA@mNM%U z_;C`U<}km4M|QqgP-S9p#%2~0kYRTKNot4damQAB< z#F?y~)$Bx+IFz9!Iy3k%>@5_sk_99hS9*}Wk8KsZ-es;g;Pv%FiDaJYtzA)5Ud=wJ z)4tz$c=>QG)Qqd}hL;PWbiisSGszLC!x%`=h^6iOV@qbL#n(4$pX^mj_3{&Csvw(l z?6x%Pi4Wo(gB%H8RX`la+!eHQQpA6yP+fYdV^3nyf35| z1P*`Rpq;B5Y<1>f?Vp;-cEQ=c&io=|P*J6Ls@iJO*){uxZ;H36~~8avla7YYd5^ZYbh! z&~+aQZ^81K?2hHpP8ebZUifT0R9{I)m7_MMG2auKw_NSHTDv(eV7qydVLhfjfssrUdH#XIr$*C!u4P`?HADsw^JVS9-Dy zxfVUHf3Xmqd&Y36{|cC#M5{ddQ@dg8l9eLg?P?-QGaXygw(j`rwtZ(LUS4fv&Bm}P z8NAe4y2x{kHBROW|FcP9;gz^8`C-BJN7;`_xqoTXN(Nt?Y+`l(;6Ked5>r5+>5$#| zNKYrG?R(-_zqIXkrSP{xw~qSmM3IXXVJO>flIJ?@lSbG0kokWoYM5faT`!r+&`W&q z{-uw~Gre%#-iFs6#^!MpQ_M#y6RFNJ^U3i&@3ADpmPd4zY<-A)?UbP(9AXYu*}P39 z7tQ!10wV*j4dI$IpF$COh@{siZ|Faf$f&`&5mg%?9%~3GhWUUrYAOFWF1a!tZrD&X z)I;ey8kZz!hXq%zV2EvbrDmUo-BL0r)rs?sA#CE^AdzK0iX|lCdoVi6T4p!^3p0g2 zQ+_Cq@4~B5N4jQF()^MrRVu`#Bi+Gv>Z`J>(qTQ~Tttr&MsgSBrVK3dM1w%2L?JiP zLNI6ORBJptmkFp7-+y>XnqD|yC(b^EH`A(aE1u=C0_gZ14Ir7d=ZTnJ`|Oob^e8sv zb~yxmlPvU6NT%o^H=6|=OAQLE$nuq1_-8N+{*Iu3Oi`=M*VzDRTbb$CS6!j;x_$Cd!WG{jq zww}UNQWOiUuBoVXDeLnD31_ToO$CT0s1muvI~OSH(+kXfKU)G7-V!ENZ2K6VD&KPP z6=;aJ&WRPa!3 zq4BFUj{za5;=$dqPJBe_g_O-!y4*`0Z)i0d;gYt2s(mt|(9f{YrUGBO1ud?@xx~W7c@c76Uq7-Te`Q;2W(mLr=urH~S{KWM$xbW?oBKXl86bw6B3+DltNU;vj zxLa9K3A>2PAwl!4pj|wPjljMSF^mU50E+Syt;9`1hGsAIZk|LKFzd?25GY4_Ws@%d zX$vAU{WegKZbNVz)Nr%jE;G{KjP(3IxNlZl+VO0~#=dxrlz&5?c`#o)KcEs!Hi61{ z7fElLT)xj7SuV`g#ZcpMtKwlY#dMiephhb3P1_>s)Q)kKn^^Xh42pX13*Wbz2`st` z_a$5X@&&Hte1Ftk7vXSAO{wI}wCf0JSS-V|%Z(KTQdD}@UdY`u3XC-EBsB(PN?+VP zyO3OkvcVdv!&c#DlO)C@#qO`D+AnduSbY*oJUcywK1;d(%dw)Izr$_taQyjCrs^C>hKmPPJ9qS0(U)Jz9rrgRy#O>xUxZsmLB*Egkxqs;%yEjM|kxi@iYvzgFr4G08 zdt|Itr5x9ATN4={>@66>qaO);D=7}lY8nJ0V}R*oRo1so6-t8W$4EvqcdG*oj#}7c zoN1~UAz~*EsT2V$NySupkdo^&(weG`l8#Nmvvn#50H5p>xI7w}&j?*h_r1H6x|xG) zWb3pH~@b;-?{~o zA@I>U|0$4T4)$jxJVxyG2SeV~pM}b9>-1Rz+Turc-pBpvnP(M5%GE62Ts&dKp&jMF zx+Qg%NKiEL=ZFEbnfD`K7u)c{D*{7BpA{>F2|FCWQ$7pxG&(`H<$pOv3Z#m#br{!% zV^_+-qR-ifu4*Z2R;Fo=J5ViVv}HLSyRO4;O*gB zlO@8$V>#7C`?%7&4SwDU+mGPSJyc|9*O|rF4NQ@^fD7r5vP+TpbB+z-5n;xqXMuU= zxD$$0NTv+ww2*8Y-P{jVv?WfklftM{iq5Ph9d5lz= z>+0JDv6tug@<9{&%C2)Hf~PHMWC$+jvb1}nr_~;;;{WPccf8+IwDi=LULb&|Nmz(z zj0MF~t4`C??C$Sly)HCEy;M8M{v*0{$oPmpgX|aQoaOL>@bTwrjydOIeL&GyF@M4m zi$kd>sB(XK&xguPWbD=JohAEDn2m4T`Ks)DTx^kluU%~s&hWp0Z_M|=yfzZxxAZt! zAZ;nkglPH|P8R+6(NB;mc(@l_6o&Clff11%z>||?K&D4l6xvIl23OpVXf_7xyUocU zJ3di+EIg5cHX2nnsK|>D!hgXFh~s$(ZFf?T4o&wrmwAcstyEJa>NGSNj&_d)Y*!;> zDV4L?y5I(b7sDutQaK3`^j#qL>Ck&$Th2)yL2T>9zAOC0J&&o3FFcFyW99?v!qu%S zn{OVp4?3%H&QaA0WPK&n%@9jB&v{~Cmc=Gw4%!@XZ7N$y+V)%( zDt(ShB$iu9Qt#-_zZKcRw*@B=T#t!D{jKq!>^QZiHrml3{P3t0+2L!Y3W?=-oU zWA120&k*a?yQFp&xSkIwCNa@XSKohlu4cH!TsMG9H{&tJ>V#oGf#lh21b*!Ktta`i zl@w!h(NkAm^&7vDx?3tO6Ft7sX?Bv)4KKC+*!?*>N3Y*hrhJB5OMoe8r25(r|IqYD z(+l_eSU&*-D_$PY9qnkf4VbMF_PTCco4I14&SA{c7ky^T*Q5ElXg!{Cj;o1fj^7&; z#;UWNx$1gH1(6@$oX@d_pjdxCuFCPERvsSn;Fh>(cv!ZmN_BasdF!WvTtk+d8T+>h zsSgr|SWfX=;qu_7Foau?T7znhPfW+tn5$=!H`>WbjC=W`h4Bd!Wnz>gAc=Aff9%8u zt%KsSgtys>itDPz{=WbcV{_Sv2(2p&0JLN)kZ9L*Uw#~VEf3{eD6SN@JL5!zZ`0+G zh%)AtF?`Dq?X^&gXo-EqKtxs!uZOy6i3kYwv`|jbE9E=JjQ3=oHNHD zGd@GIu4F#GrsVBb&yOb-%)^-;nMO%=gmX84aZx%4itR|sxK-ZP)3l8%;`9g7iY+<1 zU5uQ_`1Qy-Z%g6J=L5~;k5#nKz~LE~_{fV&JU{H2iTN*GV=vrwv?-`0k-Hi5%C@_z zwufvXGK!Nh@qIO7EH*2BWijht?Rr23HQp23xd2cE67>O@KV(HrfoH6v-Qc9M6w%+B zJ&@AQai>=(4ETvIGM_nZlkFzUQ(vwO>6Uh|1gOaAe`!D4@$1%x-a$#!zL>`7dj!*< zG7ZjGWJ+O*njnai$Bd3TgFN?U_i2>H#^?GW**C^exQ~ycF;kcqK&33`0%v%{b!*UxukenUt#z zN(mca?vi|dkaj9X{KxcPaVbUVdMl{cvrPdFx|=&x zPg(9=QJb)Y+RRgU_9OnqlKoL$&$nji_H z6D>sV4ADuV_voDs|{m(}DA?SG+POLpi2ISrt$Wf0g*OCYc26Hp>3`G((C2kWtW=Qmr0N&-K`LY99;>|EM`e_DMiBqZc0a1J_n+u68{sB#?F!En^fk3Jz2N3?G zLb+qT+OS?e@?McBZ^Wb%=3QQmY2S=ehTR(NPvq|lN#UMSsLd;leWZ(pY3W(9; zL|+k{j*P9Sz>L8%lYPsTS-oUo6s@5M;}@V)Apv4cP`7~XgG_uu08B~xwzBkP_&9N~ zcpma{F9mZX`_?D<$W7cZSGkxEL@iX(|9YAdczL6h7 z!C6MeLzmAaB@o7vq4l_Ev8~IVoLl{PJ85z!gczcVcyo;&haVkdj13jc zCjsL9gYBOmPrb?nT0E(z)j-IUK&J2@9v1;2aG-J46m7cVEf&Sz*|zQ`ytg^C{5Ri+E74eOZ{E*`sp%9R1uq@?#o|@!wcLx zL5=N?&x%uZQaxR@X?o!_EyYh_QqJm=42#QpB~xfy31n!S0JDQ2Mi(sdP^!7|?D^X* zXSYE|tk*Pijey5S41~q9~Z5aqbM_-XCW_t zwlvN%L_Hfn99o_BJ9!ib5_LDOY0YbCn2x+ECm5N*53a3SjNVgqw||MZl)mi`D7BhB zXX_T!n2PT{vot0Be^@9ZfWAeLLJkKs_WwUj;qWPaG*u9jr*&&C9cYL~lDH@BCKWwSfQ|*d@y0Q;gU##{NL}lf@+YY$H<;zcG84w% z^N_T>i_wCh@LMY)u*G8L**7xduvReYlfVm2PcbH1$(k6Lsv zdQ1J|k^GyC2=#{MlRQ?uzV5~}4Yj$Fq20(3ko3I&Rwp#;M2v6ucG`}foY*AYl`JEw zfeZJIbI`!6dpOowW7Os1&tY!mC1-u#R&W=#edLQ1TU&#-4xSywP8FUm=w$cvk5upB zY*i~9HZ9Tq{Fj+;Rpfti01==ss8G%$vaU!p^B}Rmm{sOdLQ|%wR~HPhtfc@dZ?3*F z{j3jbhu9degx+e0BJAuSNo6N)zkl!jW+C0E-H-LkVqjsa%r)ymfIq=v9YCCnGFMdcPTXpg<^wAW zTPl0sKkfZ+R!ZGb^XuKNEx$~Gs}OUExS zMP(5B(QZDh#{;_@-Oc-cPM3e(eiz!F#{UTg{S-_(6JnZh$KI9?{&-A2JvRBSga}Nk zsQqUa57X?;dhtF8KHs?9#u@PQ(SIeYPihqe67X4Riq8oH_Byk-8bXRUZM+xCVZZ0P zLtWJEPh$F?veFIa%9qb}9Xstheg1_HjjZKLK9&5$)n_;7M*S3Ne43wGJEi|57O3|iu#{AXazsxf@BGkkFseid zRYE*7H%fTjjgZv_HDUwuVEo@fzYkVz&)o7)WATSCoA3&`qJHhngZb?*9_hfo;Q{!{ zxedGLn$6HYH6Yk|hjZw@Gs4FJ_C?Tl>~y9t@63((y&G?AVRTxV( zpABDQ1{D@bc(shlBK>U*I$GY>x=G&d=XUCX$#o;+=JZaOH~~9Q5Z3%vRtRgNkRi|B zSvC>ezu~fNw4nUKYox4tuBBWT!Bwo(+8H{eb?L;f$~wkaw8?twQm@4J4|>&- zYwd``K64I!2Ivz66GZtJbIm8@m z-Kv?E@Gu*)P_f&64@xV`J1uk@y6wZB17N>?4HZof7nIp&LQ)`(Z2=(FO@An1o8~~# zs&oWko0*XQy-02-ft-M)P#Q7KX92JzV7Q8X4>RvzMT*Sy1-n$g}c9?k*84-G~cd0F0)fXRnMEO`^k7PP~OG$S8D* zhW`HZ+y<#FJ%}R|>(P;Y3IAc~pKwuPC}P3@DblZPFh>yr*Ys2JTIY#)rE7;S=bJ&& zQ37Dq@a;>$!3(Q;tafwu5m1N$yxAt3@wbpQWy$hIU1_~nMnZ8hcNh=``SK1Ycu?08 znj{>1V8BEmB?HtGpFNqx9hW=PY%YlT#ZGmA32Yjnqsdz5fQ>hjYeEzr6%p`yn^eV4 zXTYJk%F?^mW_B8|Rt4sR0oW>6o^^yww>zCnC#*JUl5~^f&KO&6uZv`3O-+jAJR+F1 zqKb$LdUY-75UpBzIlQXUiK%70#;X$m=lBLeKC~dk37D^yv6qMeE7R6drQLHL)G|Ip z=at65E4j(wkotL=;wALtI}@lYu+?KH=k}N7xbObXU!*K7XW?U^3rT`J$LdVoR}xGP z>Oils89i}y+2)3lQN6@$b4G#quf3ZIRST^*Cm~t-Yxj?RVcxBOJybrrJE0tH&arL0 z)fZ?fqszP_l^vVq4*c|&~Ygq(C^E!mGH%tZgT&C zT%nb-0t&s*n3A z8j(Ux)P+xd1KeMB)(SrKmy%x4r0>6kaX$lCqwGIHwP_tAW57Qn*tL@AtoY_V4HyDbrnQ} zhgrX4TD_4n-uXAeLK|;21E#7O!TnNwFkBy8Y7_sY`j~ZeH%kswNKFW-)wL*R0yHbw z*|aPV?`HjpvA`~C^7YZ;t@p`GVJoO0+q;nz(|WiT<;-dN=ee;)ixQ;6j^Vh;b-)dO znV*B8WvS5@I(scW{F6OsI7C$pAZHt;xC5Dp?Fy^FRvhg>cAfi*8)BZOSsgNr1#@Ok zd;MIXYr;^nYhIX0V~T2oeX0h?C;Z^pjI_Z0`1KF&+TzY9k7m+-0Bc?^2INapM`(8c zX@V|v{#^lZfmcSWKoj6Y`Y`=cr0nF1Ls0*;vmPGkF~*Fo3)3n%l+DT??V${}@cp67 zgw*Vx{BSH`(sh*J73n_Z3F#r#tvB=wn)+w&%Ox?<$0JMHemc+|r0G&%!Me2wi1h=C zQK1*nM80O5L6AH32Qcj~BnrXzI7!m)ld2NOryHM}yNZr(G)0S;cpIgpW)#>Q+QH54 z#ZB;0$YcC6O_X#xNbZ#>Fd~+EngJ|C;HfYP7dQLME9XupY@J`TKAbpg1uF33t@Bi% zV4ywr%t%Sxqp(QVENIE$)rK<^_8tweE>9eWwmF$K>N6TCD^4W$@&F4>9hQZ17}H8X z12;;wY?3b4q*+JbFe>9}8_~ha)h| z8``H0LdFBr#Fn9V)q+M$<@I;}X+YDmL@3rK-~Pl3sL8!YM}Zpm+@{r{#5(>MWMoB#;4Bd=^dFzEpE$Oiw zBKvZ33de~;TODm|ugF7ynag>v6Ct{);A*N`2en$k?)#MQ-#mZx^`0rwN6pW2@84kXDLo1K6wC8G2-g(r z`Ac4gm^Uxqe0u%{`?=G-$o2*2@eUWi}j*HI&})r(TM(wwcbJ>b9gG^e<3 zFIzuX9ev!b4s^NI&rR~Wa&hY=TmHk=E0w4GCEdd*_Ln1x>oMRrIJRk|D zx8>=Yw$)!49@P&IseY^f*ycPq@kXD~ioCFk2#mItAEQe!Q|AG8n(2lTZSiSOViTwJ z#i!`vl!DB@@2=TEi=;n7n-?kq!mm^D14BX8GsVP=B-H9PQ%YThzLd`_EoLk*yRXb2 zRdJ=iubb+{MeGXTHiUUi|KkImV8e<9{jEe8T^ncX-7g+elO=vj%P(WDEj3|AKIpSp z;Zpu-G&8i8$kVh%4`_Q{MQW&XrtgbixO)=S^`hTiiOKC;JXQxgb|;$O4B1*n-b(Vz zysizc?x&+npTQ5{6|bI1zhptqoslyeqi>?A%y*GqpvTZ_4jmha%Ul z6L1&)z61bh2f#ru!p1sRn#Iy4U!T3O6DmE|c^oXcZz3+v)@XQWM}Ap=NV^}Bq1dZv zz2=+V3F(c$t!PmTZLXxD9in_V%lk3MO|XN4G`xa4S3)LV^_@=Lb)_8p?mJH9rkJrQ zw#7yL&P4c@%JJE!^fm=s+fq2on!qc3%)5(WPZl8!XgRkO&6bwZ!{ex8Km3j}HYz(1 zV1pm~T30cQgnV*@$T|+QW`h{4FGD^+H_m7S&08hpV1om^HlY(J3BEav4Zas6uqA^O zpQIb&(GQnve%p4LR+&_(+U^3=A~v5b`RJMf$F+b1hS)zB7&6SVB@#?;WIFZAq0HLw z!ULn%-C{gXD2OhfU!zK>1yZm=@9p3VN@USH(t6jPeK~AoX~k$nFiV9x&t3rAi0v?u zC_S!QC`u`OOYjvya&A2wXf5zhB4{;6}sz-gr8(|mnBMs`a- z$V5``;>)u`fE7Si?5}cq?JCi|MF#Nmf||a&v21!VPc43-sMi)P^*-zP`qoLa5zo&h zdo=K)M*l>zD&lazZ{|4tm{Iq~kYk}`F$1>zSj9JiKfj**yR38(Qjd8xGAapdEJKNHkIghv_D)(KfNB zWYWofI$nGJ`i<6`H?xao_WCJz7xjY}EQ?p(6+|@N` z*WoDL%6Py|ek05~M5$BbC6(lxHk@WXfjuEP&hJH@ zt(~-7?>q|(Qc0{@JGw(lS&%gaQ6P0m$ePHMKuh2(%0IlblI^`#;X*eDleVcMXBTwN z@okFxpt_`!UXZC0EGXr0*?Pb$J8c(xM(2tA#4)58p|8g(-lKu|668s?;{&-F* zTG$_FcH`J9=U}IJi}r=B6zfpgrIn6mdS6V$^kEE&z>3#yS@a{RNlnNP@2sKygq82$ zg_X?an_ompInIRg7ndK1EkEP2STTQWbJn(Eb8A8t~f+t|o=@((|5V6BQ{bM?x zD6S#9<^@E8O^DPOi7ij9yo?97^>gBsW~6g)rVbJlN#P-K06l!0dIaE}(e+FD_f^Tk z@oJKZ!QnM>T2ReoV12j0rVid~PH$c#2Tv9CsFS$*`MW7d@&l@b=-g%Y#o^;~C?m^T z5-_jf$_LRjUQ7`rwMLC%)Y3-gGX&WQ{fnM!%o*SG;Oem5EFIXysSF`)?A!MkNGR7* z^7zBL7k(xZ1PQK7<>+*SU?ln)d^nWDlZq)~#GX@8+N*RZ+OFAFuwy`dkmP!fX z57mT)O2rpdYMSePO?lSCe3%NXr3ek?*~Th>pmj?0X?ASCl!89(#K+w0i6HMb<`;cH zU9>-*zu{huRk>z0-#{gNCU>dr@lt#Pr8KZAicoHqoDB}%SLip8yYyvT;~c3vlT?Cn zV~J%<7*A~oINC^H&$g)z1(U#0B@1_P=-FpRCl$)}l0a4RHgu$uZLqa2Ckp}l*!WgU?6}e%ez6JH88uR zhYAwLzE%m>{097EgfTXgT>5(`nNN={N#@;-=^*{VwI>H1LaIBI-A60@8II73Y5?y^ z-V#BD;AJINaZkj%wFX}4RM{fj53?oFF@$yy#=^O;=h^z1l_L_92(**O0p@SRrBYT7 z{Vu(Gz;R*UUktbvJ2*UyWZ(~c3S?R1X8i1#{bvlNh2H{iwp$4Tta}oH=^7C30Fk}>V-!dOCQm%`MIRt3 z&ROtYWwEEC>=O62)+FNXy=`M@$r&8y{F^FC=|4bDcKvs5>7LHEY1HVcL+tnyiAk1R zt*KA^%NH2`Qg)1>x{`S^xr9Z1ix*V87Y`pbKBM?DP2mEmAu%O=0wGcRcgoJCbZz1G zv$eg3DB$r~i z-1=4L;ygv+Q+Lz#P}XDj{QgR3bzW-fYC>7fU1I_qfLXbCe>TX5efG0os&=MRmk61# z?-Tcd(}gMszIN1B=;jUXwiP`L__@yWdxxGbYcA{_W9Rr_RiK3%^IC02r1YQy9)l39=+TdQvLZ?b^ek5@Ch z2FU9{MnH&7_{htb-TPWE@tC&_x+2CwH`-{s`Ga9|(jfM*kExsbIqiFhyJZhqbJfS- zszHtTuie*4lCcSL^dLI5QROB>+s_iM`W~d_veK+Tq%|FvX4X{^?~)8c-E* zFl3iGDdMB&SC;XfCN}55#@xY_FZoa^2wF`9__x+{ZQj=SkjWWc5;uD52Yvz^2ld7Qw@v%~oeI63(8Mu=;N?X}m0zDVdJ=Yz2gL1? z|2{%3JM;`Fw!Zo1OmR!V3bABtzI$aj=>oc1>j6%k)3#O;mvYxRP`34oQOHZiE~p>c zhZpriA{5!tO(A)2k^yaY_6fKCmAE_Wb%|#f2&5foN&Q924R$;r(EDNxqhm^&bw)Eb z!<<#!rA@1*kT4R3h&c9WZxuqizZBV2o!P!7ITfFNF>h?MEjF|mJ)30+D_oduG$nV} z=Qk=9Jn^I(br64`z;pqDg7LOBr5#_83W1u)dCv$`LD- z4$}H=N`6}+J!3&8JH5ukp=!OE%^`*@46-$5Sv3%8x7wgUZ&Nt<~GpI&v6UV^#8?Q80%khS*i5i5oyLabqUCQj{@M6}EA zOoTZQ{yo9^fLe56z5HyTqfKXl`H9STbZBf0P>14>Of-=0UF~s|J|Gg*abmvZeu6BT z*^0Xetu~<9DHkoX>Q%noc9@&ES-#gbjfV8Zrwx}$CN<_Czp9s>jlKw}zV)^Aja&}x zo;%ry0i09Gi}tCW^ZlYS zB;?Eh(q!GdFKBO!*P{11yGhj}<%rc#zl)bX$2NWq(8M|uFN}QE^iFR37WQcGeQBbd z-+>}E=wR{7k{e&l)7_5UnSvu~Y6(QGn5a4ZL?;Yjeu+WyBCXmr=cz!(hkcu3obYpK zup$MIPAS5(y!MKtsux)1Kiv#9e^d2HKRjqPmY%Nc(0#p(L;u?0>sjs7%0rlqh?|)- ztVN`r>vfB(%U?DmtD;k34cU~I{UmHQ>I!juwpWr>nkdyMs4=*?PCQjHHp8#n=x#3w z6i0W~D!?kAy9-u=yOeAVtoK7W8l^K_E;q5ILnE*QpHL5g6o5OES zkc*WMjnV^*Z;b`tYkqZC6*XnY55TsHsxI>Vh^s1vDX}hC&7gG|+cp-6V*oZ+n!+VJ zIX63g9~)Qtl$#A8Scv_^uSAuoXn~tYq_zT&edD*Wn&MMaw@*?g<%sj_2+s@+S`S{~ z65IF8vBdKfc*Hes$*UUhInEUrC#ce8NTKYm%~T_o@pOD!&}%rmKM;^w2U2&|*{!&z zz4|^CNOY5Czx2$dLuB33PRVui`Hdowe{KzbA-x_#Z0Bh9-_)s*#phjLrzT}uV2?7k z2ET1F9$rrzJKfCVOs?;^HGRR5Cm6rPi`l}j>UFggCDHb7D|FfR0ahyC8Nz+NNDH zohZt#hb#wBD7Nn}FD|v+HhG_ShAN&NPi@6zz+mz9?z>=f?A}4KZ$+%@gP%$b+B#1c zFL_**v9RP|@<{eyoff7>K6&%e zvP`^1fh`Y+BGmFXUtZ`>AAe<3=>At$d+z-{4}N^k$jZoNVx{F0VKh&0hn z>o~jZi>GXYu7B7l)oUjh7R}LtE^gFxAc92?w)Bp}q6@D>GWw$BXY8agBDWl(BlWSf z{aKn%rd6C693a8^2tmdL!#qVh)sYg4(jF=TdU zoK6O?AHu7=0lYx&;m<@>GoW=?o7mLETK)3A`SxHNM_bj@(kxX_L?>+1KGmxH8j}w#hC#zI`0W*# zwCekGltqTB)H=2s4e#*WsT?tQ!NF?wq_rvcNiZ(uug1t+W_KKnGk*77;i&8Ua^2cu z13-rFd~1EsJO=3N|Mtd2U{-o|UvA7UsriXF)Svmp{65CXK`uma^ju|3AyTXe!3+MF zfG-u(csJ&OjM=ZfKaNvUowwTt;211HfQdO0!Mn8ly0;goTx+__wQ^Bpq!7PVUK}N6%lVtKoyE0?^FFDlQ z5g{jjm9@xtTj@7h&FKwk0zYNws17N=e|^xPz0ZhHvLh5EZi*Qd#ND?Smt24rZ`seO zx6wWv^o!<$XwTx<`*clfQA$)0FE_#8G{YPt7bUg7O>2LP%Mw*0e@*#njo38cn}h_# zsni$2_|vvp{{_*`{t-noxm>IfAvLuU#%kDQX5IR6e zD!9igHHVl)t}dB=^xM5@TI=ZAQlq-#5?FQo|IfD5SSmSk{sbfCIJH=@qRGVgYnz8& z$@(7h>HaB_ZZz>vI#MPz$>Q9*t`-R|5i_nGshda_Xw|TM7E*1_Je?9Wj?F@r&EPg8 z+bizgJMGRwTHQ3iw^e?`w;MJcjO!yR`&?ZsgrAM-24IUue?Tw!YBJBfqeVU-u@wH< zJmQKgXY-`y#S_vjWGR^w)oi3eh=2^*WBA$a67n+e~yeT zvQ7G^@?lXr{$@e;!*}~X zHJAmjl+%JuN))r+!pucVkJYVbCDlqOT0c#93akEMdZVF&;@kC+;ABPYCQOOX94qg8zsyZI6z(Fn7xGI0 ze6NNEfd@;6X&@fDAH+kOlL?fm&wc7|w#&&*yKoVPY7st79Ik!5pi`^Zeo$zG3Vi2k zi^yV9mxk*BXRA4T2#BL6Z~0rEJ-6jCBV~(SnZ#ErP=<##dFM5gFd4GdKgeFB&TF6s zb3>BPW}9h{r)aKbcPuQ@oxnbM>Uo^pPw~KI*z!)M`_sxXex)Df?_n5Bq%r(!EW4DK zAOfGZFZ&?NAL#gao3QFb9SFMs&T04jm$Ik*@AoFx03HB!VL^N2boM^Ns7=7XjbbSg z^ODwWbLl1GtquUGBCndkyfd%kPA~DSF83L)O3EcZhqGa#T9d9|%O&Zd-Vk>l8o9Y^~!>9l}>*)6o{ zx>rTHJEG$)Wz~MwL-Q253{MJSj%lEDu)k8$45|LGU_}r%R?`#CE>h5y3>x%y>0{mO zhffWgL?6aTf!6rH(yLdf{*hyi1uxm&%S9vM1O z_8hxeBv0&0eiIyXe)({>#)YS~w9R>pJPc&ZT0ysJy_}&X>LM^R^gCL8b$JV*1<7Ma zsrtQuM5bikjuY2=PL66fJQa#xSBJ4Q-2cwudNUT~9yQ4^bF5;VyD!Q(a|qQsgOtLz zQ&`PgK4>Eb+@g*%KHr6gzggE4|4RP2!)+iIza*|nEK3-qX zCpB_}KdRPF(n~_4YsYu@2b;iuGK~lJS3V_pDkz306M%zbM7*9`(ul0pBDL&K@&kKC zUFJsJDZ3OOn{IebrxrXkt#+zzi9A|99Zx>yf!L|dmkzAEC^5r&mR zwR1R<;|51nvIq`eyhAM6K;Ph}i9Lvv#AwQ!^kwLwkp9%|>givJa(cH;L* zJCUX*@l8{v1>MyYYV~kk0B|lcuMp80dUQ6eL8D&Tb`m%#dps2x{^o>r_o9>1TTL;` zJx!yeRL2Z4!8!{OyklaieMYOA9e)*wz^28aI{EVT5mIOubaESVOPX?=798L@)PCJv z5yqQ?dMHm`W=dk=DKk;k%cDnP$~b^DrAQkXUHk5&Vtt~!s8sW!{>2au@}1&t@v{j} zk7qa&p1xz;qQZ8ZPuhf{H1eytU!8M;X|%Ful)~lUZ8&e%&UsId19NO74m^Y_Otl2JCYgsIc}i$(v!V= zP5Z?FD}+(f>ZG&uqQMiiM-HHPhW2p4UR@NTx(A-$CM*-wtY z&lGf7TNfMSanQys)AFvFkcUx8PbBiO&y3&egmUQKR5H<$Ggss4Dc`+M##V{Y3BtQw zeLmOUcmxDwi@qA)WHUTcgx}qDQU#o&Ex4xcAv{0$e z$1IiErq%QGw<1Y@QK2yFVErr<1R!;0=LNF!>)O^w%t4Oq6$kr*EbV!^4Np?0cuYSSh)$l-Br8F*0}$uBtLT(xsx9vg5^vA{B_^W9jLt1= zM>4{9FNAk6u?t0#ZjZT|lj@#=5^37guE!t&0vZk+U0=kK8XhJ0aybsDYcUQG=q2{^ zTE|*1AJ zrX_N?#v)u){+W!ezo0C6V!aKs3S9fwH00HL^7)@xyKc-OWz%~IPp_g-O6x&9`B1k7 zya6ZKxR0{OkxUv@n}?Z3AniD-&gP()GrC3qGAxgJX*-MImb&;4z1rf-q3VLx8FuZX zaL7i~S??DQ2@Q4Dkp*pfSz4sr0+AF*RcW@xao>g38_!9=77a}mWwe%GfH3RU{oXK zHy=(H46_4MV%eeS#9hkU|F-(TN8{&)9h$*jN};2xSHmb`pDUm@=P8~ zXnE`77=8nrxNIa?4&k_Opmtd7{ph+sb{6+mYwPa^q&kl5)OF0K7fTsgicy?M5hm_Mk@=3F|<%tiLa^IRtRh zS3UU3UswDj+z4z*kllY5jAA^e&}r+&|CeL%b^%yVh%ROrxoS1wiGG_UjMR4l&R4y) zT}>+$x=g{nVQ=DG`ot&6(p*`rYx=*|RlMZdu%!zcyUpZ*I!k2emA}PeTW<|?h(7La zKA$;-)0=NO9ogaLLt}NlLw)T9f#y7rE__O7+e2Wg z`BUygfQ0!dLoqt`MJA`V?t8E*;pIMSNMQRbPe4vyG+_v*L^cAUP7Y+3*DShk|fHeAXH7~ z*0FIoHI;-=>Ya`BT@V=ya(FxI>p|yNh`U}&=P=#M9rbIh^vbo z+v5HA<&W8sMG6(>3rZ`7+BFyqJ@2*C!{^ex1Z}hd+j*&xz+L9+T7>y*inSn0sY#tV zeXTijMO$y}n}Y0dfFqMlbhR;QTtW85#o%cV{5KG0AyAhj-H)xe$+z3b+mMOZ@S}O1q^Hp;iFhI|Bx1G%vi5n^7 z6{^6-F5@wP|H8?y3jRj!_;UN2USV<+&l&){+v{GR3+6oA%}UlmC>hlFpJ{jU%l~{# z(K)TNI>1w@1L{<~-8gyC^WNPRL;S@ADU0G{_IQu202xFZw$=j+*7OX zNyu8nQ%+0Y8iE6HCm9HWJK^H|HpNocxuaM#6$|c=mHL%=5tR!8rC{00rYYz)yPKIU^yUgcMr0P#!JQu=#)w?so@XB7O7!9;A%nC_658AFk%b z@ldB0vx;r0Shq&+Rakh`4Kjnc3+iY%VE`OsQcTjQl8@a3fbMlpCbapuY^87T6Cj4w zelkxjwFVC#r6s%m21MdLP0f4ge4~zfX_5Up|INnn#SF5Vnb8-&5Ks@-!iSe2{WXvX` zNxklZs=vp8Rr^3hZG|l(w<7(y;L>#}^E{SUQXk8Qx^e;j%XI`iz#-;){s0Xhm>}+x z7Vi0v5qy{>g(rb&#g;mUcpAqbYVts^}MUZU73L?EzXzu)%9W5VM?RV zV#6IQ2r2i_=kcbbx$kyb9Td}cXR2z#E2=mJre=nG?Xbdw1OI6w9uFPSdN z$EB02e<=<8nDEYpmc%~+sY3GVY`q*pTsU}6&EnM2|3Xtqi~Q2sLxValt$wX$Rb zccl?yP`*b==h}y`I|*U4aLbJrGm+NOsonNDWAkz6GG9hvD?hMr@Fp|g`kQKCBbM*c zf7$FEERCz9~MwN-PCi8QJx< zXcyn%HLk?Pr?x1_Z2Wa>r5j!u8x2}y@}}XfbLDTA{?WK1jFj|8FDK;!Ql>G8hiqKp}bNOIQoJXH7T0)>7#51@HAp-johWL&a_Yy>87}+0=_Bln8_eRM> zv^fS>T7YR8eeJaC7Lmh?^ZV2rdqhUOuq@5cJ{^r8eec5$S~SC;lAthl7wt5mx~W+o zA+~{!)2x3GJ&JC>;~#1_7WD$6H>Zu1W38u*Ncy#y$aG15@$b!)0q81bna*j*_Y%2Z z%5LQT`?IeEOJlT;kpPWU^j6&c>O&kG0w7I{gySRQ6d=p>y8}xAu^X4+<70Wr0B~w&E?IA&Nd+6Hut=}!~tfgy@ za^hi;FhD9AeX^1Ps=LMyexhyeIUvzA<7j=3yGCqSBbpHqns7S*VgeGx33M(yY&$}0 zCdVmSFlm=OSL#1o%bz)rWi(fhByY$ZVcYxlrCnni0!~#lg+pdBUZrPN{q!rNpiWk;<(>913g zj61)f&--AyFKw^rD0u3lr5~q=>g<{i&8bmZN$@FLekvy}Ppl`q>=rTJ7`;#dSzKMO zVlBP!8;^_m9;D^9S~1Dem@SO?F@SPU;?~x}$w~rr=w(aa7Z~21r&5z)ZJg(>BD&#{ z;dobh+GJFRx4z#t2Z~7_6YD#nz7sR0D5ch<;{z=QWR98g%Dh+_DQ&BH z1Ojs6M$#>|a36%5R5U9F(4e&87$D=+FyAQ`Z+x(pbT9Q~MfI&653bjuPmS%}FLM75 zW(fxhA$^ryS^a#>nwy)42z{eZ9MZ^!H1|FB_9_g}@Tt58%F0^!!DevL_|P%~jxj6^ z(ed~8`w|Jd6Vzd8T4pX%<$O?Kb@HgD&0sCv_eEjDq9>L@U&g*vzs*WofeD2jbw<>A?GZ_SO&5oFYxZO&*;?Ycbm$lW_ zMFm;O7Pl(o1jM-(c^LB{P9%1x;BO4p#4IEI;Ja8B`I67{rq3x8u|K%=nG&;K*c!iW zFt0)Ji>*H#GcNaR;S@pGyKXLN^?JsKs~?Zkl&X{Hgr`xEsx=SKm1~ow>F8-BKhlR= zAhuUqqXEf_>1h2*+eJq!iP~>+HWmre2mV|#3R#-{)|W!w;qRo#C7Qm{=h*YCrmU_* zfd-IQ6|kg>%E`OfFJ=S0hjJ;(w{0^9r|{+LrVIWM(putIgf7RdpDaAyraS;_hewBh zHyn~t`#3zI-i0fX8lAcm>uDPqS#l0Ee*|F6je)@ki$cM8T2jD3WlfZj2LOs}YE~QB zT^!JcNxwb5iL+;iaBF>HB&H@ab0ULD+~r~c-8w8)~cN_%Z4zL860suTL_gy5#PuN$zFDcSC=j0#W2 z2d|f-&N6qChc_MS%uki>sGVg-=%S1cpBIgSBC!GwPrK@V*+Q9|=1P@)7Q?5O;Fzg@ z(lcq6zlcZc;I00y=Lq5w{VM4&JH-J!fK`f*kzOx$*Qg{nN6dfMLr5oAs5)-O(hrgH zT9!0(L>?lI$#J`#4iU=Uc;w>ryj^-6La%AQ+%D$x*qaB^bZ{r4W+Q0H;8kx%*2E@n z^xW*+|Fb>D#w7yNDtve_NLNo^8&VB%vQv=^%T?b1x*an6C(nkauHah)ckWM9)l$L()K|SaE*L&9c_5Cz!&05dQbML$MzV@}RJ@G9fj3wE! zfDFb!?(HQJ=t#CQzDE^L$SCNRh*ovbT^P(~<+bo9nbBh?sOr*1JfTT0X+t|Sw4ze( z4P}l2I<9uljz_^&>jmLBp_{D>hj-&#I`Hz1QYK#3tS9R+nm_+hbCH)pxQ$5GP;Djc zJjE$3Bo8Z(t2zoM$eMbJTCTT*0$n&H+pUF zUrGrs9g-?B0CHDmN@$w)zGmBElXgXpw4oc5@3L*DMe7i=a`Hb8B2Sw|4pKrb2mc;4_Jn0R(3UdCMJmHvOrB|Ug?eM2ijasW!7v(IrbH)dOOn?-1@huzZ|x5qv_5%}ODCjd z?;N|XMF|$0A5;3E1wpleaJ71Bq|Vgts2ibY{?%Bo8Cj|qyyrr!W!4tm{q?l5BX?hBMe4-Aj?wK})|N{LdTLwONt`sgW>tGJ9DZ?Z*~58Lg?1p^B3i*E(zOejdl{`UBHLF00{m&SXJ{tYezBmp?8|Q;x@*=mbH>+&clEOv40( z{0~m2D6KCg<`2V7$q{bH$fzu8y_4$vs2hbsR1tn#jgKmFH;W1!(=C?RB=f%Hnf-Pr z{Ay0Z`j+t?1>-F+@wbRAPj_bb+F}NSZUw`*we1EU=5RF+?Z0Ml^Xie#BKN@+YNvQs z1C&F$b&Gw0a*MooPsnNp`_VpF#&%7*zsxtvgm`s)&*s6rn({TvwWx>vs5{!m*A#B3 zFX$*NrU`8|^d{_m(u?^Xq8&LnQrNKgXcA0hG4{o4k|Rl0y03Id$@XRdoAzTdkk~vP z+d)({hqgJdbf%G8j=e|PZc>mpXEQ_R*lhU@m?#y{`?0#df@>6R0R_I}9Ir*9Ho8;9 zFx5I{y+>eB05ph8?tBb)l+8-U0G*<{$A=>%;9f_DItbb#HDZ6$%sqRyJZDm;N1?cF z-_jex>{IO>=T^~0Oh}7N(Vx;$CwKPtdFPB-G$nChhy3v!FG6rrsi|gVzk(*B-6jKw z@`P$%^qVa{eLs_z(UaUgQtfa}bNO z7{B`3@dMYpixyo1Zup#x=^yH)Hud!WwN#rinq1VOx;ag1@eH-|V*PVDjm@XJ#vWu} z#wRj%uF%dX8=oS?(C+_YHdxLB`5FAAGXnVVLy(O!<&khhzE+vnTw<&(PopNxHMx|$ z5daLWW0Gf+T@$Peh|oWlP@BQM-}Vlx4GG|+RXng_ozGU+zFIbjK3C)*lD=$#(CWRP z&E(vIf?t|%cY1%dLI?(2M^5Y1X2GeDtZfaY_&CEuBc#!OoGw3)|K%Mo?(^h`!Kl4s zpC<+0TbKc$d!N;S7TLO>fmmx{I8K}?ZeVw+5^*v zB>g(`Z-32L)0sjHX0X2Ky>Em_(?p>!)z59`3HQEHP5QODulnrtW~jP&RWI;tsXHyi zkNNps1_!2yYnEEBg)otChae!`9+~YnAb(8;TU+%h1u7dX+*MT1O1#R0u}M*hAx&th`BCAxo%sU5V|lCPmnSc2I!Pki%nl5pL z!djh%xW~LC{RB$bwCqWgd^l$~3Z*VzpP*ePXoas@*kJ8nukjZ4u0l%cjG>6BPU zQNr&^Y>-FOmj7Wv-)%wJce%??RencN_`c^Ph5%j&)<%!gV-2dzU)2@uV8rqsJR|Am zpLoKa#lcZR!2MlGIY1vhs7`{q$cV$yiy|Md7FiQGzF)6RVJx+s=9CPp>Co-2*4;bK zU0pZ;P(ZkQ9xxVK zZuha`@*Wv{XopzZ#0V7HwU_iVXBfuH2HPH`5`B9F@?@P3i&{vR?mM9{P`tsN_`zZ< zD9NWIj7OrFCbPyjlyfMwG$rs}{dW73J=^ZR9m3T;(2?yKwaj24O5UEyrDITRafi<) z@PtnI=bOqrKDy%FUa-=yhHwEu4K!6M_=J7smYRQ&F^8P~sT;+!mNVC(plrtjGOGSf z-E9t-S_usgTvNJpQEtAs&6|9z;z!C$p>|Y7Z|3dHIvs8cn%AI?*L}5iXGcp?j=jYG za>JpSTpnTi(@Cqk&wNvZlGTd> zV)yMtoY^CD>FDUs!pi0K9PSKFw4uys$m~-rRnQQmiKBhV`0lp~lazR=FJ7R5eJlK4 zQu{r18pN?PdAswwuf=O=^zHUj4Rnv{TxiNouoX_@?9Lrobo8&-D;2F~5%VHHDy#N76M39J@%a|}a2Mv&&dOnq zVPFqD%(wy2oc;A>Ny_P0|FV_QC^#?E6yeg*En!aVuoJHfzyn`2_kCh6Crk1uCw%o} ziLH%_YbGz1*==uv2Y22JDJgEM@l+eN>oHkOH`EfHO9pWq_l_Ap!<1suA$htdN1IsB zGp!L%0U)|t9aBM}W#}sf{o+|nV3(XgW;m=j-6syaObuUvR#i@x*Gmp-MVU~xaK3bL!bu9>W@ zb3Vw?TNain4R)B=A_y)^g>BkY2_0{G8Y({o5h0j{ynvH%&g~)ABWBYNB%IjPT>{(fh=@ZVW9n|iTW#(lS@@&1w z3i66CGk;~&w$wNg%X|qmElV^ZK@x6#SK8GUiMBf;9EdZ8E-Vj}Lo|Tz)?z-?m3e2o z$=&d2i2Q5ePV)xd#ZBJD5{l(bhWf6C5~&^=OcEECDQV{+$BclVA3v3ZnDR7^Dv zyJ|rNs;^SNJIa?d$Py@6k|T2XNSgp9A+IZyE;)O#njX@g7i7jKgSJB`^6#_FDAV4K zoms&?BskI(n^|9d;lxhOg4x#4-tFNsS6dI4W5%>8X!~6J50;vtj~3SSv!Xeeo^0qqU0+wc3w^r(tf#o3y)IYWK5g}pNMW*AkvE3uar zpKG}>P3Jm3*4!N_Uk6Dt&txZweffy%a;Lk@Tu#YBm~sM9ln~BKHFSXHFy^-Udy;s3 zBDZ7lR_wxTd}&ae@ZYdZBS zWM(;6zf=mgBq{k?FPo2sYe!eN>LPUFLRhviJfJJ6wwZlk!Gl`jQE#Kq4YygLt0ixQ z12Q@Ene=uZMdWX%-#wkOAf6FzeIdl&Of1Lix3{8e_Y-laE$=*sm;FWF-o)JTX3+Ti zZ;*V+3F`c0y5B2FGhgVCI4G3R3_+9M!+294qV?F&7Su`G~3bA{N|7BwH^xz8#CO@FA1BB$`=CwZHe(% zZ@>D&i3?A${d57Z4zZ(@&`^owQae05k+OH|BQkBU-R+XakIuF&tM$=OklEw7z5eH9 ztJIp3Tvr#P)9__J&w@)U``xPoAH`k2-}w@-sUTQPHz>DqP#8=a<$@^Z4YcSl_BI;s_x=%YlrU%ncof3TQ`WdUiev@0QH6x^afld^}^O2(*A(%2aTQCN+|p zaPST}XeiD8SlxYvWys&@dUlsP05-zWoNdEQqE1t=pwOoU{WYuVTC2$+13D@b2VbjX ziA+1t+QFOwH|%j#!L-)6gx<@vYo>&bpB)BfW;9fS{-%%o^Ta1P6T@+D8@#I#qpVzg*AtZjuwy$s*z8u;k-gc~w;A+TIb?xWsf3&zGLqENTm zD<0W)QX!kd1K?Hy9sYb5r8`+yZhBr-S;sSmfVa`J9(K4bQGTB!)~Nkcw$dYX+B)=a za$|kyRA;g1yD);(uSBBgzdggN^evb1C(lPV9PQdX=8jKQrAs6)rGxs#M0su(94{(N zxgy2}HtY=2^;)s^X?MPoAqp*soS@5CHs0YnOG|0AR|hO#n)KR`cd%x%!Wg0P%S742 z)-Z;WjTKv*FislNxyQSRs#d+2>^tr5DIk)AkxT5ODl1yv9(EYAcLMA{A?l)n@l2~N zWy-GW%$a6i5_t}j=$b_W>Cdczk64IdGb;loSel{BmAaARh`WtJnCq=uo2m?hV*ZppDrj)gByNNzE>^rfYzumFs6qFye-y?M#U3{NN8xi&G zpwoUte4ptkSHW?PnzdkG_kS7XpFkVx<8+tzvhN1;vLA2S}h2P`k&NI zA&$1csI?_0Uxeby(D2{~y+pF!8xm;sa$8{+cXplL)hJLu%h1s5N~V)rc>{|5^cF(0r4S0bLfS@J$)|(Ra-czNCw3rd*o24aDZ%6DMN;?&)OH1=PxQ&=lo_KH=;)EH$9vQ3+HWs$ z;QYj1d}30tBJp6SUE#mFjztGnD_Pp?UxnplKe1{|sLGFVYW}|9E0N+y2kJUKPBjs2vv$y_=6wLkW?N;~G@?#pY zH#J>hzMAcS;UeJZ1>8C-bv#Fm$klXTf+*ic9Ltyr%Ou&rZ|jXc?zI?g+ppIw3yCY? zQX)^WL*Rs=_M6%X^FAO$&i|2h*8Gjw=?$sU&s%R)YJT43abiDcY$Pv2Mh7+N&E;cA zy@s6TM34@rE4Vq?=rJBsemfza%jO5%Li$aI>R-*GXEwb^q z67~MYm$IDglG^sSL!CVPOMgZ5O`qt$v;MV-u@U8DqD!Yc1JV1R{Fl89(JGT6?FSZ1 zn~Jw}k+2A_HZ*ZA@TcM|OCqZ6BydMYj3{4z_P)Fz>upY)xer{RlK9-v`Xr~P%K}$8 zs-DWRXDnhsZF6Jy-6c;S)pHt%MYC<}&*6|B zqZD*dy0x*Dapkwc5dFT{)d;KX`nLL_0Lw@~U9fCp!*}ltM+?_dqFm&@4dRI27T4+g z1v|YiyQ3qpUOkvI1KJvY#b>hsIpk}_$#p`JDVa*IcJ;N269D3fz3^#UX(6uMX#k>H zsgKZug3}pKy7hd8O0J|@!AM9c)?`mui9%NJ9E3REfGH!OO70`^v0ISp*gkV8sdD)r zfQTH``9Ka-dYXDe1swkiHu$g0LAjHunNK80-_>uunqw=a^KqqnW}fmGqr`LPwIE6n@y`gKH_i7V}8*{$9=O1x-LLpBwqkapW- zKu&bWt2^tH@&fPjQBG+Whu4Hl!Sn%j{~O(?~;J>7K~zRyB3l8z=T?57K>X zGFtIxX+592t{7Hn*7fn7Q`q*Uh%~9*YwXc3Ws1cGB4h|z9HwQCSuXwO_WjD8Ml|-* ziapRI6KLb~yge#dSy`p-qDpb$e>Jz)1;uTI zMZ4+F{X@RXg{0YYmv7Ffn`L2P`;@UU>6Bjw=$h)FE9$|V{+y%1t7}CS?$==q8y?K` z7P}O4Z+`kGX#noS;iII3izGq*FE(T4cHD-$1)lW$lrxaxJ6Y%xRM%az18qlU!+^@U z$fo$5zWx7NUFcchHGvCs&=}HY(0ySdu_v8;XTAaF4&clpMxS!;Q8u90q-07{s1E7J zOa|K_b!8|6`{FFq3MImY^9jF^a^?sfnp<3x&~yJM(UN3z_vIy-JL0y9M%f8sQy?qh zf(i+s+B8Y!>XB6*FH{0zBRf9aC2Y8{3j44hTN19F8Qx)~T`Iai9k^Y+caV1Pg&eHe zn{!DCu}F`5fjtJ{M>k?PKcC$MV^@9wBcc*uavRoZLAQ$Nw$l2%niR~Ctd?$#?W@PZ zVTTuJ?>43YMKjPm)%P98-VO6N0t>re76ntDH>jPAIv1YOdblM&`;o8o$iU>yh0l$T z(AtZM3^(rYrcW~zZXAFAslhHbG{QS=8ylH)Dnsn2E~e^y{L2Qry#1Jwz~I?B`6`}< z@)tk_twL{}#TH6vN{y8t&mk2$%NfeX0_KTs`_)Z^h6$iU)@oYcPza;#&XP?T?dw;7 zaVeBoUV-;Vma#=8Kvs&ex1r*$eIQ~set;Tzr?pJx(Di}9u+yEb(#Y?|GN9-6mqr^@ zs?$Go4Nh$(7rb*grlOo&ZfxhlP>srYSjHmcSM;!Q>SJ>_=v`-&R&k-8c`jrAnRZpB zt>>Z;sYyawT9G5^PijP>^ZnZK)ixKw_9Pl6{>ogowxJ)ZKNAaMiwk(2=PB`t9o@t8 z#x3DOv0CV6igp_;j^$}u?~j&qSe4b6SMCZvJTl}q`5GrihR#iqe*4=Ie#e2hJN6Fy z@zCrtL|wGeuyJpL=h5m zW#-D+pUG;)Ru7595b_oNKGtH_l?Xk_?ehM|)&^b;Hjc)-brkTRC}+A^yHb7KnDmHx z`ejfJVjruI0Dsj=__OdphTXJxuvDT=iQ)J+TUUh~Zd-c|1?X0!o2OYpgKU}HB%ov z3d32vy9!GLdP&rz=XL`sIGmM-fG!Oq3ah`jH19-D?= z9my43?zqA4V&+WdXiZ0z8Ffi3NB657)sBiK*DWf*NrJ-GyUV;0ham>byYZ*FP5HE4q11D zj$b;qQ>OPQ_P+)6ga;$Q+XR#C#SEQ3~2=57h_@`>5( z<1$PYzLZ>gz9)HDVkD5(Ofb;(ecRSzuMfE;^m#}riCq%URhRC9tz<8SK?~H%K=wzeQ-b9_bw=r+m5mucJLIa ziRf(l*g4K)=`JCm`zY(i2UIuFQoQ+4C~Ub7cz0kyARs*w24zInVS7KoaaJ$t&pmbI znJAHl(n<+op*nhyYWrPw{~&ocxxUSQXz@wyUi{}^nEzE}G~o9zHQ$$L6d-m#|L!{T zTMLFCwW%#FpPEDMKY?n|zbKiJlRLRZyJAdy_4U#P3kY(RnC!9c0;i~YaM>E?Ek`f9 znY)!uwp&EMSScrddujWsqmh%kKK*I?2Zb}98zOZ>&b%coxVRT=8h{h<`)nV9;IQ;T zjJzv=tk1{jcktugQi_LXpc z#S(&fmhrASpk?L~FFs&f&DCapM-I+BvpPDAc1bO#thn53vz$sdL%pERHez(Qny_cNC1q(M47grJnLl_jQ&moid)3~R4!4K1uyB2q7LrZ^x zx`)pO4}N?*)7A4eS+gxZXIub1rC*A|N_2pnOS(t@mn( zj7vK@_9rqv;f z#n0hzPxyieRXr_@T!B>oNW_7d`RwBAgJk+1^q~0_aQB4}jI)raEP!@;mV})`ZVJT` zWwVEAtF1n=Q0#Q(4WV2ks+~U7@T3}=m&}gM9~H&{k%_Grn^&4u`NjqPJ2nl!=>%=& zUrQRNRH5^`G@HQDv@k=%!qqFcQEXs0#dn1?NFt<7wvqg{2ho9$`z&VD>+iB-0Hq8P zVA*AR(anbZTAD3|Hq5t=e3*3+E2;!|dakGct_C()oU~a)>?`Jsi zM5kBF;3E#Loy4=vG+}aVm@Whu0{EnO#|IU34_f$(+jC}Z1g7d<0N=%i(-WUcf9m)4 zQ;>%c%Hc;5^}O3&*wi~8NTT*!Lb;LEr;6f0bF7-fRCz9$L(-v45?yC-=ywTU1(u+P z!J8Fs0+nJ^49uGU@u?d_3kx#lTjT`&yu0Z*CEs@QmgXrpBunuo6YuZhKxnm<`Q{Gr z{khttlWrzeX`kBYyg?0G^0vb(fosyZzFNa#$%flD;QAQ&gQl|J_C0U^hI1C$ zq3v>@#{j{zjNVjfCt8+FVT4GX zpyZCu1bePV)KcUK%gAq58>ZN#&qRiLlO|X?S9V&fX`1F7e=LRZ{+0viSHFBU z1Yn3vDpQe`lL`MUu#&QZ9Wga+pUc&Zz0T{hpZoankFEF+SMT@lgsErQ4C_Mv@iF;+ z7bNT{$JC8!D=yP$qNQNF{{A;E`zHnU!8~SitgA^QQT`F_a@PbmB6AZIVHy$lq7JH2jIXu9&_Z2np zPeuGB5p$wA{(WTafFnx<9Y0?eq_2;BF#*dCJ}Sc z{`a%=d-b=(4^7YI7RC$rV8|;-2>Spw4Ilt!%1{0Q2>OPH%PFTjuS7iA3ljFw}SabG;l-#WP($qv6{9hd^A{MePPKZD&$`&+g>q7 zDmh`L7P$1g^Z8ge=vE>MI3;(s!nKpLakzG>FCJgZJI4(^VfKFY$O}NaAz!xZ8num8 z6(<93%VC;Sa$}26KV6!Aa}}W0gTKwg|B5?938aQTU3%H)+-4HTK3*PYnb|%`XZnUn z_>mD0U2oK=j1$EruxhZCEF&cpbdf88wPFkVVU4xG@zf35xkC8FvXW?73%&p-x8I`v zrh-grg-FHbQK%ROxuuvC96>Wr-)osLiXX^C#48k~A`p@f7FveRV3tkAhL}mY34mz5 zV$nZe5`S@~8p{tb=O+izvcne#~eX`-KkvL0w$T-^7q<0@koUV`QpRp~4zPgxSBBq^}*kgT-pl@E)c zr0PRrtWw4fnVd@6a2F+?@o%FCd%Omj^1LzD8c3)VUnom#yKbVW%?}apmlC}rc#pHf zGtJ0Q0Je3{vVf(@b@i~U_3*y+gBY$JOa|a+h>^mfQerv2Wu!%FqE)Tgi=B=zZ<=xhf=)Z5_F?%Ejf0`@frMx>5Q_^A4f zji_cd&FEHO`rgRsh>jEX*om0p4COE~{slQ`8D+_u^UbsZg?l;|faCC+%h}*Uo-I?I z^vlL}!t}&Q(2vAWBhgf*$*a{%*=gb08D{wK(*q##c&{0z$P`$~b~Df<$toN>-N6PU z2?GmkqshJARC$5jRuH03SGdt+)ZGUvo)1d2BE?y@P?X)=4GlVuJ~kiO$_5i;;uPGH z3HnB?2fXXv7x-X$uN!{obYASE+V0}cB|Lm$LY#vjTkcn0;7dWlU*sxB8c!p( z*|vpmGepA?nGtp%@MC`J+|DvVbvvr9poYO>%-mOKn+Zt5;APW~XFaU?eF%3SdzHzY zxexc;Tj7#o(XX$`;ke9mOSZ)sr$n!p3b2fA+qGwGym`vr3fBslQ6{IX*|bi;LE^zt{_{UiP%U1cnjW!24gAU|ZQbHCli~j7oBjvlJpWX|Hb{Ex^KcGwA z5VnWsRh!GV>=qRbR^Em)_2CqK=k}~BY}>Ccz#pd_Zf~J z5VVQxfF#1}cKbHK_+FCY-t}<|Yrvt`l>=qsOh{x3JADz5LU3Cu2VN>SjoH)TNT$LwU=&JBMK*LW`At}3D7_Mn2Q$To`R2q1f7(y8_Qwv{Ko z3!lS_kRNVdfI18stL2T%?Prf+%(_GjH_N@<_)m`7sFigqNrz9C5wlk_gFd#vuL&s2 zKh)bGHX98RI`q2$t*E-Tlupe6Dt(B0FF zCY7q>Ziu7*b9TvNQ)+Ejk|ZH^nUe|PB&QK{B{l`{KsE{-zZxC4ZO10*dX8j@^PZg( zItp`P-$@b8pZ#V;=E%^}P0uym)cPI>Ei{BMYp~)nKyuctr?O+Z>N=^BeT4qqO4;eM znH)*=qufof z7~kW3)*)<%uLYz9aAkR-70hHR?1!^a^85ffanJQ7Eru3x9uC~M5I6E=pt3JNt}5F1 zGpOC2ZX=fA1`vp|7Ikuw`|{kyQ&g+X=D0< z2y8!lPci`d^l>dW#x_e|zUhvVKaA!$gRH+%Tl|vsQVye~9CIhEiz`trl8wDyJ~Zac zY!8>#AHGlZd22P|0MJE2g<=;#ZDdm9VIgHot;w2sfHJM`x&gI+BU>*25_wSBZ!6MIRZW;j)9+4>B8YnCEj>P&(imJ-a) zt)LT;I$jt`p-WeN)Pl-dy0yEW6jU}q`~CA0Vau}Mf`3r0o?FYu zPS}S7piMgqVZonW|2R=h{0={wuu&sv)r0`e{Xg*1-yI3aE~SqtX8R;?*I}SqFP)NZ zwe&nUf&NQ`@s|XdEbCQ3T*)MAVrdqU=oi(zZ4Id~SLM@;Q}79+?f!`#f^MaT{42Kl z3pm;zrg^;S$BZXM9C*jrKzgM!Ld0VC`vJV9T4dwn_a4M4daopoC0kGLJD`i(cTLac zK?S{!3hftLIX0T4Si^6%f3?D&{szfj3I2Bo@Y{X8#RS2G!=&a^w<_Z;Z;X*8}>VTe^%A3)^AFOS_6ckY)sG_%b zWet-<7?*#xiImYaT*kYN)3R*6V-dPxIu{<#I<2~q92T{GSzqPdEiw6!{S^7VWmM3X zXKl1kogQ~JL>uT=fo6YrIIQmf0MUQ8+5I4!x}tFs#$ChoZVt+1bv7(^elbw!slVW2 zNpKn7$!HxlYkL$)lFLK-Jm8 z$pbG;Op-1!B9V}y>Y%D9L0RW|o_xgLpziOC4gr=>csW(^e?3?ZO_Y6DT1g%zgegPz-i z%165|iNfn`k_HUX@!tkQI5?8xL+_Kyn$6U?&Y^{}MsVR3$Hh!{ruW?!d%W>4uE+=q z&+|@xc(25Erm;H?$k|}pY3z?&j}zY`r!D+@@Y1+Ca~`c?kvBzE*`oMKN`-1$%BQyHQd0ISUZS+V0ag|iru9`Q$x?B$PE_8T=f4ZSE8)5Gsbw$oM@>tT-?)G1@dm;BjR$RcE zBZrH)gxjGkJ_X2p7OtoL?m(mFEcG)HF_ z9#8pvRhqTRh&pXJiPe{6eyGTl2jS$!WL5Vv_fvg+dJPX4#(_-%4)|M4si5V6rLSc) zpqV&?mNpT-6Z(=cZqvYkdy zHvglbVumc$nU(z4d}Y7Q6<*H>Ck50{5ha1F%m9XtChG(4> z6z%K{-3GexLyOTVn=ul0Z&uREmsq(^tDG47WcV7^b*vc<>3VeYIhlAbW-L`PunyV4 zC~bI?*-tJKt654u7Rim$(2~(8q@9ZVj1CF% zL41uNPBS~0s&6y8)#UF~K{uM)Lwx&o6m<16y}it3ft9ECd{(vU26v2F+9v}Jp@`XT zx~TNBpJoKU+k0(2`#^BaU5J8OzR-+>lB#nPtGCJ=;_k$Hw4Ac#fSl~~GK>GjMl!?NL)bXoD|G#gtV#OU zK@BbS0eCC$On2zXNDnPDM%k5qTBRu?;cUIq7;%umvQ26Q1%EDFM9$wIt~{ru^z{B} zgOn(Hcll2Dn0pC_aoXphas_x)wJ_7K_a8ocx|y~*I_(}`R<_%q^L#pReq|4xxw5Is zLAS#)v%VfwruKT{nJr8E_-9@>+D-1bYDcou=frgm~C$Z;H>O*8WEG;5>9fhr8`a_K`l z_hR@kllNef7D_yeb9!863^*k_oh?A~&^sQ*5%!Pml(>}t%ItwNG#NM>yY=#watAi! z0PKdBhTmpVJfc9{hVg8t;zc?QCQo*3!mCenmQprV#-D_nDM=Icd_Qw9g3Xuskgu zR0UV>cv(tkbl!^m7HX3wo$M5h*KcI>{vD$SlW3wRzgQ5T^od^}+6F4bdq-GZqdZRT zwrQW9b|A@+Ufoc4ac>tD)aY?)VC+|gxJQn^(#advQVAFm-Jm( zEmjY~=_1C!-UFs4`a8hB@W-d3G}Pp_N|d7X3kAxY|? zq9nJTTf054e~5cnJKtSgT}4t^S9h^;2Nh8@{N-ngYIk4RFtymh{^E$C`?ZZXyc_MQ zX)_JgE)kFEPafTN+U>e{1^0ez8~rgvMQS6;#6|sA#9lgO@0@7#vhaXez*z|ckU75Q zY-W@f3Q=L~@YO2TS)Eg0Y*;UmEHN%-v%>^OlM9n8kZA5&z;`7K*IGqRk;lx2Z)uLL z;`Hh6;DWKDF%rb!0=(m+fFGLGpP?D;dwyXhI9hpDd@*Yg`6{(^k5cL5=8?;HA^WJ} zi|DGSBc;=FL`pRqE^w2Vg2B@aEt%#5wA_?j}|J}d^H7iu& zKraB&VI%?>GG|FK{29&hJ57HJXc5a?b|Xg6_)TB3t|qh_Nym*WpWKP+M(A~Gl;SaNROzq&s?*$5e;sw#~E8b2{ z+ujm7qKz0z>3FUC75nN77o5+{B2R;bAG9h51+e8&+kgB-@WvNG`WXM_BawMySmh7- ze@3f~t`}~M0*?f&uud;{)3tHmai|jwXrUdF_#ba<%HELIgS}X~gvA?{J?WFY^U5c` zY~9cujuElu9a{4_oZuAYLsKH#)9*L_V=0N>LoX&b2IGAJceAVy;RqpLuuyYR$W$9r z)e6nl#pSL?z8wxytDSl|YfN+h|Kmq)`r}_A|IaJv2HD0@iTxfqa&(AH(^B(ojWs_Sx_czwa*Rh#HT zdAmRB7+CGw%iq?ge>ua!)6n314w)mi4(&As%vCjO=w@`ZE?8h^%j(v{Pgj0V&%?{D z@UQaa@!E`?b;hGU2`t%~d~yMAvGbPVI+Ph32ff1)9xtLtUj5GppZ<(Pyl#RS4=tV> zH>x@3(^V;BpY_RZscYbFyP#jw)B2gnXb`xZ1p#1E1AsUResTi{LQB6oEqt=BKi{p% z(`pJ#Y?e)F{pdgY#WZ)uw?ZA4+ELXJvz7o#3P1_96!;F5{=T4P~rWlD%TJHG!_ziG;ob%tP@=$`aSGx z|JMloqriK1^#=-({3=<}yGpxJ(#()Q!~T6mhL~aA zn7gBDqsAmOM|(SIPa(*RaelX~h3|5i+s91l&%tV*A+J`i;W|}vO)I(JPgQ;26lP&@ zjXNOZhOpySXs>Ap^QEpWCJo5q>Nm}4N}tK6Z-q1&kV$CL-th4}F=Gh*rYKXRew<0&k9QjFBF8=@y_t&|)Vxiil*$;BNQoHRn{w0lZh_m<2kaqYk zw|Ub*d&;OPO3;-`sLy-qY~HaT zPcm#kUa~ue925b*Ft{yyAUW{k{$0(iYuwMMpqptgQx`$6E-+O$kL$99@@2D7OO4LP$#sTa;JegAmT&&cB&J}m zG(9+}$U8}wJ)y`ysCnAvLdUd%xJDmCDEX4OR@iNUs&Y9|J0*&(RU532-kQdeQ%2ru z@0^KMAZ}AYspv$`8gpE_R@QZ_gVW*3ibV3U4t$3xwt>dN-#X zFk-g4^4X{z>RWwOKNeRJDU+AVeV^K1jR*7vRjq#I172Z4twrT3P4{ofC-~_dLOKLD zN05+Q2c_b22XHm(U8mi?;5Eej*vHbec@;y@!wM2_Idsx>L;yWT<)z*5!1@i<^xm9@ z>GGKs2>95JL9;QTWgULWhk)SHF~7YDFSPw$5*5g9p`9fY=)kuJD0aK}Q^<+_cK8q# z!Dbpf#=CDBrP3PjAb5A7K+=YqJj6}RF}%~CRJPlzw(_WC+>2dGp%sv_RmIqS7~tD_ z4B$OaN8+c&PcqNbHvy5(r;{;^a> zwu#5>joP7k@?k6Qc9O%Sp)y4hUuyyzygh3{^rv^?W?D2}{>P*@VpRG(zuC&pp{h?w zP8M~WpBSDb7N%8e$;*P55!}Qq+GBcYWSka`m@h5&is@ycDEDb%r}#?Xr1Cy-w3apF_5R~=cK%X9|AoSm7E zS(&u6WOs)rJcJ2J;P_o^Fx=b9pvMNjUidtsT8LbH8nKJN{Q6=}@01z|$#(nCdqdsS zi(gAKDrcnXtkAQXulzip^ZT<-2ZU*1>U`)xHIH` zIkvGl&yS-&w)nVg`Hs1jG0+EKoIB`U$;?&6r?(s(!-MzI!=-g zab7qG(ejaHcI*Mmr9Uql3iH9A(%*I4nG5$Kme(O&_udS{5LX7m4{?5*RX-n#bTV}JtE9nvL@ z2n>jTvhkPNJyh1U4sHcGe~z02uOFwy9f0=_wSD9y`TR)$InOj z?!ER}*SgkP*Amt4_m_9ebZd~n#_>n8`N-i8bdv%?1pHsw%&NhWuBa9e?S}wz+PH$T z`>>ZeSO;C);0KoYP}bpJp{{B<)>T~cewS24&`Q=0ntgTdDU5y=8Fh>>zz$XWuy&Ox? ziwQmZFyu`#Xpy7rpmts(sLJ zjnnqG$>sGs0p1oNgElf?!XGeWR+=`)AWj#56*yuCM|Kql06tdDA#Rit(+Aj13v%zbC+@puDi1G+ ztcv+R>RLfvZ!U;C3#mww`=VvFr!O%)MpN-+;q0K2Zlp7zHK7!U+pqw!8^8t6|JJ*Y_!I9N{q9v}5 zW@G7rc)NwT1Fan=Agvp0!-)!z!T^C6;9s}}tO8`^n2x?@&C48YAV1JB`+t6d7Cgox zWBzQZW+13;$*N@KJJyPx)Ej*?&GbMVC36>D(1tYjNUN}}|247-dT(<7>lqvV&^ea1 zf=U!<>WakwV@3eq1S){F;Q*yow?J;ps@L$PvMtIVvhej zgG;MN1r@j2Lq6;DJDn=No6Sr!8@W&pWJoZ35X`H;j1j_agK>Lcwg07ksKArYu$P*g zW#g#bKi8q3?HmR<4n>JwPhLA^Nm07}Y6b}QEsQv?Us-WmjujlhE#;c+S_<8-prAtR z%IwVW{%`gI9)rL?-<~z6R0;80hT}_M+2$Ll_=~&MyQ(u`E|t5_eDtfm{W@G{z<7;v z=8FpbI{NC|&gRGN+gIjiy52Dss*pC0-y(7()arM!`(NLHh187=EMx*rv3)vlhv#Xm zjBhe}dG-Lay%#4pnO?ePsS4AC2xSsU*;ls)t%e~NDjq!5ih!I8~5j1`-dEpV? zgHzrlTyJ|xv;u$968NnDeAoZ>N7kY{I@rioMuH3A7j1Z|o`7D#M_>cirN_qGrH%)E zf;}3Sn7Sn69y6beeG^-;9LHCBPSL^FOxWt!Rv9% zlYkri?~RYyv~l=+U1E5%Kp1AaZ5X%-Z%mG}03MGesD2#2Kd;7tWCi2lNiiu4Ca)&p zS|*1|q1C=oD5pXQJgZ;IRudT3WMY)teDs@^(;>l3|H)zSy< z8zj=`bI=bmA(Bwwq;E-+%GXd>rdEx{JOY;Ys1+&0}1dO{ue>0xt0hs4t#%bQUv zE%CPrRydZk{x81`&>#ohUsiPfNk%B?Q!#n>Py^|K5io8bC^s8W{5FSY*x&fwc?UiL zCRYBlllzzFEdp~o{pX!v0Omh6ONDSFe|#}LlOOtN9)EnwWuQ~)3ZB8je|P$%K);^V z8JFJ8LUQvs58P3(@G>bB$$9vKpsg!!WNfRw75}l=g30lX7&OnLa~Sded!wY#sQa_Z z)V{=FLO~*34mnMa8y>>6;|~SE=$W6R(T`zbk3HnI5RZku!_re*GAJ>S^7$Re|G@)y zcs)Xc3lz4PlmkH$gCawGa7P=5EAP+Rj`6GgdMhVeYP0 zbA#~g#|Fe06dFMm>O=I;RxfyTFOZ)T)%X9tKT@b(n~2#$03DrLsq0*ymvTZecv^l} zV<6RDPqF6Mgd6O)mk#URYb<)aOO?nQ8vs*UzW<%gY6F|4wLysmG_;9y8Ptcp(SW4c zXkh$Vkt+Q9*KQTDis#cre0Cm$Cu^r*wgu6;6@QBb>yledIC@5rEPP9f44Q@^kp2bE z{3pK(O1hFKBr89Y{FMwch(_gM@6VPdT&%zm5E&!dKfva8A-JQ$T0`Uz@T zDb&{@TZSj=65D^Vi{&jfULfK zXmPgyfEp!*D)Z!6#l9aJ$YPbko-WYr#+v7})L< zSgf{FT$A?TYM|UQC3O1WT9cQ^Ca{u&HY*-@9jSkeZA#^F%LIS`7_$7|LDUa5N zpTx>VNw=yeUmq#(%Eq8K&OgAzn}o&Z7L^PW{%+%;ZqX)mPkPT7Sf z2?Cs=gkb$wjhBPt1$%K@@^6XUth|`Xp8O&*h`Lh6Wlm19a zhD0sXmcbP0p)#Vxg&U-dTxh(H+&OAQ*zl|gJ%cscY%gJa?k-RXqOzELngqd9mujc2 z@`+Gl1$!Ime%;P)P;owI{=RyVBm{#UffqNnQ`lXPt>}DyXH6({QhHkVmyXE7`pLZz zqMj$8jiqhA_U%xXu6;tQZ4h)MKrV+*j+1-}(Z^QYK)#=_5eS!+S)RjIljt;TS@?PN zov~<{V0T$3tzoUo7nZ5K?8oBvmnfj3T(7$ed897o@{Jnsvd279VS}vrqC31`B5Tpf z5T~CUaBp-6M3(|Pz3K&sVkB9+BQL!Ma-16`IPKLd@~bM+Iw6jYuiY{sLQs7k==Ht!+bD3 z=@<)kj4iUGB)qZn_Kf$b9j<+`$!5E&|FLhtAGCh~4?{TQyshtHWw^Hkx&~u?_l(wj z>c$7Jvsl>I$C7HrHur%xOP?YCovP9Q^}R&`)o;+HOpuX9rUpK}EU>t(u*K5xig+ff zc^WNakS`HtOwjdhT{LbwXtWFoIi6PQI7q)*S5b(W>Pc(Z<0!I3|K4%)LpCdEta~K* zx|P!X&m|nJCSYjOL1l|XYpkLID}$7@Z;dw)$A*#!v1v$7KxR|l*FC=BY}S(WQn60Q zen}DILx^C7IhZvFGFcMime%-WI@H{kSYlD5uNo7UYK13{y1$b1dym__yHF_MrG248 zNJobgwM+p;fWmh&=iR1d=kKS8D;r}=#S!#h$A~c}ptvJkKA?E?3auPEEIQ1B@?fht z;jv#JkzpntMJ9Y2+CMh{;v5Kf`w3CG{>R*lt-NMV>c7sSIA7qPsN7*H#N6-)!P7nO zOQY@KRkla%fk>*OG1?)|nm$`kTTCbiOX=ikF$!7aX1JA(89lF`mfPx?yf9+>p^g4)F82>HdslRC@tpO)zWgFM#am_DPTo-WS61z z*)&fAK5UhpqLtM*<~4i_uk_rmvj0S<*GmruYHy%Aj`7SDx7O{ly%{BefVP#`0Dnp@DDmBq#k0 z3vk9tS7EJ!`4N$eU+nWg=tcNh-U{#po)0Za~olcxs+`J@YIfJO@u2ElAynBi)JbfB|DR zA%}{1ol{DnO%d#DIIS^pu|l)YQY6cKdwrOBt#WwlyL}I6w4tgO0jd!GF?ak;+Xch9 zz6j~Y?~V!BlxxS`W7F$NC32p(q=P_zXB{x79&n zwFDmjLxx7Z@+7R-G@Dbq>)4v@g@ut4Ft!2u!!je%vqbRjnVKA1q+8zx`VHYs5QSo% z`v6d8b?G{Zc#@Z@5#VQ?2`Scx8-De(<$#S;p$75&PBk`u4n%XF9CEZf?qf)6aKn3= zV6o?~8(t=crY-&PY<#Ht?O%pn0qK3D{R&CTp98kpU_C9oqlAGD>?l6EZ^H+M>v0^$ z-+cN+1zhWIk;T$?4;r^D1Ya?B!_zODpFTb<9lDuz=alt?3aQ(O20QRd?!qhp<53j0 zauK~i6O!UcWiEL}>U?!V_g(lg?vmWsm*K8-BqYzs8n72}R)2q%D!jkgtw13rks_!# z!sHarmgmUy8D2jIqaI50d|bSQ{T4KF0BCS{d+Ap$@iZKQXZSn#G&;d3vVK`A7+Vnl z5DkM%t;DLytq&*sCl^k%NVoM%8XnhSaYwIy-Zqb#oXN$>2Thnh=fAJIJ#2vZCsX1m z8))YUez{CZlwi5G>keiAeAcLi1Ys2C64wp0UOGL@`K!Wn=eYB;A=WV;-67RLz|9!m z`SV56(w&bF*X;!2e?^kIy5B6?zYq8zqjNIJheX52gJ*P8O?PfyL3z6Nm7c~$i-Fc) z>cG&MjikCe{^dY{{GgaVypZkBTgpHX722XN1sIxTGfa!m$cqwS$qQO<36ww!FZw8S+3QDl zi)w*pH2@TpP!XV|^#Uvaqp;m)LfL7GM-&9}3l&DriOfGF59-y^A1Mu(`X#kHHmJ|t zZ>RY+(j(iwS}VkmTPE>=!i^EEe#cwauQboWC=Sf;)iXl4yhZ43en1U)5`U0+;Iu07 zlYVo;2n#;kh=mZiQj&*PYX@z!ZtZmL$){KoJZeJ8=;;I%KW4vmQRb9%?`2K74Iu{Y z)fy6+();`PT;fOb(9A>drd=PvC?q<9m4qVw4jf~1P`+PH$G!{O2^8EutTj!V!m!y# zIf%J|Lny;9I{GB59NLlS`5 zVYx8K?N?@_<|(lEwd&_ebn4gbsZ|X1?8!eR0=>f2Yb&P;-Iz{lfSdaV$MnvYFP+l|{{e57 z1r-Z53zn;HEn0tU3ZF`_YY_D4a94`Vyrwum5Iz-KAvN{-9q-U5Hv$HX`7q1Euix!Y z8CFWJp&5)hpBMe*rtKT9>qSTSlxPt^Hrw)jqpk=yCM3N-q^8`;yvTDQFryse|E#YF z$Tf4LpGo>xPxUxNTL+Gq6)IX?&-z0B$Us!P@JnZL=X<}BbMl3fXh}dU|IzrI@SsBp zS`q0siFr6OAp<(nOWY+W3Z?e$o+`bLg-TKfO)A7;Y>o~zq911T$j zEtF74s7(X0c~beYj|P-KWBImL(%#=E!?5!EWa1!rT7R6O7igb9cZP%@I|FU#32#Qx zU^cPJGoVteSv|dg3n=!ZZR5Y^l0JIQ_iOdumm)2=)@pOBR+paX3;?T1)V+hVAYIla^wuthJlOw(`2eoo%e75zkkOc&^oqH#LUQAhV@geKg z-)Tr-F@NO5(FF-XuWE0w2*!ftpcEgH^K1)KE@%RC{Fmn`cD|3x0=<25UpxPcb=T=~TRT1c*(J(WS6HRJOe=ic^lvrf(4|@VU z?0R3t#h~IG=_M9J*;{(^H#It)!61#ig8LJYU+LS!vUaAwA3Bup4b4Zy2f22a)H&wt ztQml!Ki&t(5oO!MRISwfk8rt(fos>i=43&bV-8rfwkr(Zy>QNPe=&v+k9KYP@iDQI z;N$|Q0u|UTj`X($B`eBDtF-MkFh# z_o~6&8Iwu!55JDq&oa5@|19J&NFaknK&oT6*+v`5h?3xLeinEzCOjb;RO}aUauE&k zrI&wgHSfp<>(lC@*d{B%F^N0SrgzH*RgUnaH>rH=PX(FR5-;jqgRs-(8m;@2Ie$g1 zmDuE;*vQ;c&H&R87h$_QShIX5M4u#%6$jl7t?iPPUTus|N53N^yiX6P)9iDJ|CS~|_on@&k3I3aDI<0!wQUzjJyMAO{rlS3`;mFiVHD?R zc5wqz9c)gh=8xS&_1>Q>s0wgcUm39DUQa317xXr8J`P3m)NfD)b&rne<+)a%SNrJo zO>ESpz_Yo083b684+BfrW!N>*}zjkQ`GWo5Mv?;Y?b3KB{}4iDQ>9oNmC zsC=CiuMKU*6aAJZu46bHO8$T&<7AQp)5CaI)7a74v-DSxjt1KXeEy_C?qIM?@@-LS zs03h@rA)o&^ex3~u%3sXytj-yeuaKxkcn>-s*g^bIvD=IsbiN7SU3!$eLF05S6RNR zSpg&OW?t=1FeB4ybX;)G9sUa7w_IS_TN^kxT%YHcP)PW@C`;#W-wc<*0%4@t=mHGi z7w^b&Ud5~guKp&v&6ez7l~eMxBm&K2+U-W^-%RxS9$x9k`|7otKh*++cyaDYN4l#% zro^0yGMw6=W6%qGz=X<^)hbp#55skWQ|MMM8Jrm1hf2v$AXEEB`&z<>#Jn-JLSi{g z7HIq@3uH#g4E=s}MLKZbH6f~_;g$wTu!2w-qdRs57MdzIZxSsC!-#&wz1DS{H#?43 zUzpgW*LiuMVk;@Xdc%8O))63;FGU*g6Srw1l=%nd9}WFEdA1czZP#0LOJady8RvOc z7|D*ptZ=OH_Q<#xHhX(~^es?$jLUt&DE6&~evrm0&(-H+jH!tmf~ntb>i&e~y!Uo| z(Q35=SO*n|6FdI_!}fn16DzVMIi=a-RIsnkwA_nj_`J4IzW-==?hc04GPJxd7wTFX z!fLtn>9Kgk=8GMtkNffA`4pJL3WD?=P;;U#^k_Uo3^H@+h?VRfwzo+Gc&iIH+&^Y` z$<2gkV<|LHx*P0^9hMvhZlPkfc2j!8GOVrvicr8@xry|Q`#rnW%Mcd)QX`u6W|5(E zi}wfxpL8?9)}lax3f9fyFozrkbPR|DU|F=kE3p-OkAOFdq`#8L0TuV;*`Qav!G3L3 zt-@*GrKK|FJ=#C`WnjtnH{0wEwEc+w(4yc}Ob&n1qM?vtpyUw=f?2_HyqhB@{!aJ| z;iD`m>rR$-T7EQA>#nyk-3%-rBe(Ra2CYg5~!3m$Sedj~{ywi-zp6Gu^(Xy_#gv3g9~{Uqnj0x)F$P`$RX7}M5N^ED(gR19lRwFM`DZq& z*XpH4UI@n4X3hvz6ssFtpAUDyAooC9;2%Cf@aX5{GZT~=j>R5m&dW%P72$lMoj0qF z?Q~d@!4N!?KcLhIMuGU1p#zyO>O7cGmMWb;V!SHN(4Tdi zN`yL^!sB&~*~`??4zStgshKfynBXbLNiIPHe=PdBuArBCSuW`jdRaVTLS2&R0#3$j z?rH8(5H@`lh72nmt0314OM-QmLK(e5Wy6E`!8|W$oD|?^?v$gM>(WgZr@Yu!{vrO* zz^r_$8#Lt;lc+!Tmio>1{s+%h4Y~dDTCj$3Q(@57J=K|k1yOgx-9fVyutfPhs!wxz z4ehh*At2<&T6rUP9Ldg(7X^?BF|_8=`Ij;L0AKL~U$BfthUTy5j#Xu}eNK1*pAfeL z#SG8hR48<=k6Euh*{%pgYz?SMI6GfN>Ry@yN?P`6fB)(}QDb8U=Ni;eG+XmL>dAPo ze{k*cE&C^-NgSX_X)cB@Q>uPk-?r|(wgGCdp;F$>_iRjRt!9G#I8lo|T_e9E*vAu~ zJksYTN!$|=(e>ou;GijrrW&bRf$>84yk5!HPXCt#3XkCwuD=^7#2X_Ik$6cYX;3#v z$W%t?yX2VemeVvcu_W%u)3=?(MC>Ct2xBR*w-EiiKm?Coia72uCF>Js@QGaG`zmNB ziPXJ3(=`br(Le7Z!dHj)e@HD=gj?a9S0r{>PM+D$lQQAsYzaLcEwinixK?Ti_x*_p z;@{DOl@>{0wZf5*hJ*gt8ce60P*8Lpa{rx{xsIA%4L+&(OPTv9Ect4XX!#O)qj*Of zb7u0dR!)C%kw+^a%??atB9tgvSd zZ)*Ety60&A8ldeb=?@h>!t#lA#EM7kUq)z_-f+A?TbA5tWuCN$A^5w4U5k3v{;rJS zunjm4jD|vfU;8}mQ_m`bgVTxcwcDz6BSrzqlQ|=p`1xXg%UPnY#g&&K?CB&bRm#2q z|C%ektE8*ZfoivOr02V5B(5ljI^&nf!>wI;j4IW$C8#YalnGPa81%}NDqQivy)tkr zKy_rM=*@XjqWS9nn{H@yi!_SRS}MPa`dJ@Y_RcsB1kJ9WbS$Wr5ZB1Iw=tH zVOYt__7XwfNDXKV@z{5?1FPgv_VxA9)2ZFVQsmjecX*&2%!1wFwMD`t&%-0VkG%fG z=SQ-6p>-rOp=iPdFYwh>bquk+fZQ_W80h*_8^oMIf5`lwBj^T6+xO|N$qhAT2EW;w z-LbgCiLsbwU3;Skvj>+BzO#uY-`OitO_jV`dEq*4!O=>vA0Mj2WeK`M3QfbYUl<98 zPG*Rw%5-)l2w5+yem-iDKD06EPM5dq;=ox}pbjLf@x(5kxP~7HN?Jo*euQ->;cBp^ zoiB4tlz?0H@V2uqwZ%^2>zCwtBA~)k?=LkU3l2l(1|QF2n0K$yny9z?wNEvD6yr5n zs+V9izd}2h#V2Tpux-+;U(GA%$CkjtlHik0+K_BMBb#MCW?OD%B6a*(cR$&eT3h=Xm~B<0T?7662I( z2yOAFYf5(&H-)rqNiB^Imd<`q^ouwl-NM|OYKWmJ#h4&G1}Xp?VNB2yagV&@Vn>R? zhM<*ZhxePtLAFa22gyfEuaJjgqX=uMA4lIVbDSPuCybGf&b!zzsWhwb%u{Xrm9R!C z7j`A}vTO8ENR7_)*Tb!Cj+CQi-b>=8j!SD#bxV^oJ+o8x$Bp%>Hm69hzZ;czD_9GK zc6jZgria}aXlY;;#!}X`X`|k=Pcc7(!rs%jQ6g35#`oEsnw1Bh`BgP1_{4X07&4!$ z?0AseKBLI>@={*c7=;RmrQRI0dA_4kw2lFjF6gexGG^;w&wt7d^TW`~yYn22g9e^Q zgpUwcLhfwLkBxkBt5gA4>%cU`-SYyU4p=n2+5S^FUWz*(g|<Q(yVduek1dFKD4~4h2lyy?3XJuGa-#7V# zJS~0n^$5P*roN)KxuY7?d`MZjU^^zFwQ;P>!>*0eKlt9XRc6w7?7#=y_P175h_3dl zpRz_@d4oIys!N5VI4qd<`m~T{C&9<4lMh|TiCvvOyqn*UtF$!}c8m9Co9sN-U(VrJ$ zlN5b)Ir`)-v@;Fl_jM%WG7)Fe(}a10X7zDr2hCR;ny1JGH_p-doU^x;c{tD?pt%y- zjP79|w@t}C7v(GMlv&m+%V#+{O6>~Kp{FFJcAcF(yCPEi!f_or?0X4PWySK5b5ug( z4gcP1qxX%`zQ~hKk7;kN@Eea<21UK_*VYmzc`hjN3u*g-ZqJ_g>L)iv`?eV*AJ(nM z;3Li@USh34# zV9)s0MvraiyQ3#6$CJj$zVS)Ul#E@_CZe7BoRLhXB@PnZccp(TJ~teaZ_6O7l?;_H zWeB-L{F=uzdbc5EIU#RBJ$wRU#gy1C& z%BM_b8W#raqL-V3VyNRK?1<>=b|xLHyOd`KXX=_~spPU+kTr`ZPtsX7jZ0JJ^AjFR z*#D5F?HZku<6|yVw>8s|=+Y7zptYY9toGUAS}tl7R5m{*J0tn`rgn$H{o`2w|w){d9Jw?F_oWPLv&bWvM}jNux0 zQZpsVDG+|d(LQDq%>7-$RKt}HvHIQ|?RhJ#sa~r!ugxsJ>u+Ivgr<~6;ErtNaPtTo zN~XU{&wjzKQ)+ov68xLx({LHy4rLE~ey!A#$g@@LJHUpu>JeP1MV7r|`JFAZHw?z! z^POG;r(!6XqhG!O#PmTrXq3>0`2G|6Mqw|KWPg>W#^DEo)<)PCoyp-FW(sNU?e0YO z%UvKj@8@ob0qe`Z)H#=)J+7M@IYg$=GxWVofWo#F{aFG;&8W{=^z#5Zw|II>xH#rzdA zk$xv`_bIZsn>VbfP@?ms5~d@m_DjMxE62xs(uSG=eduQ z{cz>mET64_%L#xVYrl}eBB29MYWV=6EvrBW-V>p;#x1zkxQo!Q9P2RxIRfJ?9Xxd| zq548)9q#pMs440bccf4TVPb6&xuWKZBy*kP9IR!HxrosFG|V5@c+0CnGfIbz>O0SiX1Fk4S!kpW}IQ5yldAfTKRC<;n;rz#{tiZt-8X#WBGTTIol_A$Q`P5s|xnY2a;sOR5@sfM^wKk7X0m(9L6^!*ad;v64CQ})^r~HE$*|fvId#D zki5?2Or~HVV}`a)MC+f_e+6{b*YkE1xa|5fBQE|V{4lAp^8$mL*SHQWDy?Xh5A+W-7x*^q( z+XY=f7PtHH?sS%xLhKWh)Hne#mfkN9W5|aP0$AfNc>U}*o z;>w+-;=hbPK=6HCpwNie3Qg}k@5oq?Kb;cE1U{R`C7v?ULMBj3bmBL3f0vP9c{YgS z0MBn%6OQ}rWWZljn-QCb6OL&<#2DnI%6nX%FF!whClM}TEMSkoc4uG-1sTloftuZ} z3fRp~vssOkoZCfG3+%xl&>_X!Te5cY*Hm}K^GN(EXn>>WjTv&Bx)tXOrWLJ_KP0C`?8!oEQDrEayGx z&?MPuLkvgCi$1lO62Z_mAH7!{(T_GKdBhR680jd>>4s|LUSaz9xnZ2Q``tz+7${?A z^Hab^;(y?W)VQ3{JIz_l^Ka^6h|c7G?ozwtskB+ss)ITj^`F`})FM^W#78hHuzWVRqjFt|UigC?vA7*v^YCHe(TCtU(R7yS3`veRH+Eg8%0o zGJ3J6R@}&P>gcKWkbEqSM7F)SkdqnX)T&E=c+V-vGQ`ZKra?q3|IL7T@UdL<8!G1Rb^3cx znfWisUggK}S&wSd*Cgvalq`m#Qs83699vGPr>O~R(84vEPDvp#D9QqgI_R*spN*xz zRmNPD?j_}@wy^V(oPQac3PS3yz06=fh|6p9hwb+JQ#;ydL@aj!wld%--W4~ zbx%Gk!mld~zni>r7|5@yGZr94P)F-c_u|8b=*V7i5=B~`{LWaTEG$d3HK@3lJX-L{ zcD|)mQCYvqQ4Lg0mmwcA8rGmMI-?J!Jgk^d;`*=O6iHi&E`{X|@U*vHsX?SFruXi- zy)E_}WiQK0Ag4v@zCxAhksT2z5hxi}(z$-<91nQcYgqt)YfvXm9de8w2}lo><1kPV z<3wFf?&R#O9@{=D*yO%w(xtHzU2Mry#j-odiZfmFYW+uxQH}7=rYZkuyWS-QWzNyy zJ9&eFB(8MKthM*Dy@2HhvvxdPOsVPg)({>yyJOcca){ zba*88EUZxni^-G%8a(M@>{@WG0YbYJ&+kQn6C`vyIuUl>04$CM^}P>QK7NCs7X5kA z%Ugs3hsQ=04hxPho-auk-`a8%mF}{1P>FpX?6PYJ*;z^JdI07K-J%O1x655cQjAIz zy65CO`~1=IdELQPL0XV}8x4*-Tqrv9ix|P387*n8>WzlRB18Crx;A6rwYzD1nMIL7 z1TJ`vbnjS_7uHdVUhJAhzR8#TkqP(gu!JhEhK{$X!X)PyITjs9)MLD3KAmvQj(-0c znE3AM)kqfJ$3=_`gN8%-OEvzPlx~yjm*)zt<)ar51m)=cuimro$(>|pL8apeDST?# z;z?h`+npsV^QfUBoN&lZucN74Cvvl&;($}`LAgiO54&&WR)u?z zeg{fWbN%7=HD;xRSL=$(X2n;yW65WRZ-mdJ5bv1|jn4?29>}|701BJU#qqAf-^ap* z;_MisJhzr_19W13Dhzeulz7=@%Q=x$BBo@g?mc+>`mJsSqqJu)MCR1sf+3)V7mu-- z9g0jKzS4Z&KeM3J)|eobTa7&f$!BN{F4h>@9GIb!aK23wv%<_#lpeUF{HfG)!?9Zy*zW}gl130V{E(lsw*?$9{%!MgvR6tNU`tQ ztDd3@Feg~rnRhhuE=+_#J_au|mr^!5Sw95(!(YT`(3jAtt;mb(D2GLXOkiojE&qn6 zu5VSK7;E?{uQyU|u-uaO`uwQfRSSwc{@+eLH9rLp3_qT=4)^T56-AXxPK5f$(?E#r zVC|lod9A-C&G4RerBL6&DhItoCfj&O-M_>{KFGnvs4#7e3rf$B3B}Wpd{OYh(K$x@ zmp9wPxkjpb)J`w1E^32X+tCmG?)RATtFnC*C6xKL!*RxNyu<=?@5vre+q0ltkv`SB z0E;y?KjsmmBZG#&FInL((Ne)n4(dv^nI#)wHkg61pu|VH)TiPs(9Rlgu{Y}EvB}1P zZ?*g&TVt=&n~IyTDNH92z2&)hGPp0V4K6_GfXw>7& zdMmP*j>(@XcPXSmiM5O!B(sZb_lkV7OCkk>i*-`m4SyN@N~#vEH+g*;k+YJTA*2 zDz>X52HUz8j@II;y52wq9cR7KBw(z1!R9Y)=p}mNAdfRZXa)undeU#CK67Zj|)w` zW$=#LT4soteyWJ?j6OvsWZr(wZkHF8qIiBd0f{q;MTlqD@86%;*ctYIGBGxcki2~f zOZAQ~j`z`3;byWfoYotC1qK0c;s(;$1Ap3fpp}pHS$!73Tq9lg)8DQF_=guY=TBlc zptz_dVUxbCO;oOhrY7Y~fSu0>Yy>q&UWq^JIN9%ihffBz@+m~~ux=E>b`9H~@o?(_ zb^j~2A6`EbQYCm2ed*q7$X54oNKZ{W{%bQ75t*ARcG(7L5@g9{%hXjJksDT$Ii0J72!^(s(1fIj_%$opH+s zWQgj4FKZy1-t2-X!^?B4>COn;+yyPwAj^DV_ z8*7S0@%N2#Z?P?kQZTK3Lp~lq{*Zph4S+Zp=%|OuMgNXnrv8xLjb9F$1+{t6yFFUj zKB%~)UJRz~zp|Bb*W5D%-BEt>v6Nwk_IzZMOZ;`-MOk<#R%Z15sJ;N9yAGj(?NiZ0 z_*%A;G_>-h&@9|9%MxV50W_JtmYi*P$~%;sdf1b$>Mlh;_j0*V;xuZBQbQ%7N;#Ie z$*Y19HrC|tn65SqIxNX4n&mQee2I)Ueim@E7Xh6&eQ29xiqG%+7a6*sxPOqL;K=6V zdJsG6a#~&8=P&Alo)$bMv|46OYkDI?Sn@_yccC=_w72WcWz#T zeSHWf{(f z&RVsuus0PeV|6ITOujpVDx$sHpAZKY&B)#Q<;%f*{|z`P6a{=3QuqBr*eeLF!R(%; zQ-nBdjz!IiOmhE4VAg7*E{AxHV+fZ}JB?LV-iPIyM`-k(Ffv;=2KiT>CX8V*z>vt*8cZOf;tRLu7A&#Z;g6RP!NJSXzaF zdpBIKeKF;6QJ!8inWAg667O8gtr8E-ByhjnI5?!SMPo;YpjsLsNsVnvEh~(Fd1S2C zLw0t>@{eIHy@NzweeIN4)w3_v6aCA$1m(9&nKA_3+i_mHcN|>Qa%`SX`$8eFzXwR8 zMnf=kL5<(^E2;(DK`Yl;3xpLPnuJ5zn^Whs-`BVi<}D?n!^`QpDI^1zvLC@e0HHxE zg=~Y#Wv;IZbrFwXbDya46D<-=)fIw1+coM2czn6fsJp>B!gG;b8%6=Q`(9$(lBgf# z)V>B9vz3PQ3}S* zZp}wC+}t)HAHBG-TXory1F|f+_4F5DRBOsKZ=s?{p-`1fcvF_VnZr!}PWQZM`QPYA zoEH@!`Rn_;x5%9e;C{4twf72tG8Ws9l3cs~$mgZ{;4%(yUoRfQk4f+*Cl)9ve70`^ zPu%(PNZ@(TXw%tGU%Dg`&_@w+K2+6B3>?1K;1fFNP6Gw3%tn|?U(<6pDYFY_u83194}+POp(J)it`IOfGRR z+7!kvph8GRqV8KlS$H32QMgNE?Ye>ig>OEQe|f0Wf$mUz+@EkS1m~v^E9F7B@5AsR zHb0-sG_*#-=AJ5*lDZM!J#}z09bgo}Pr;)??9>j*S^#`ZFH zQqoGkNAujXn7*0fLieU%)8sZy9=hfR(G7Re&W8RoEuS6OLtBS>1%EL%AF5(;h2O_$%LIh_690|U))6&qo*$cM zvCJV8>0VyWVE+YtB{4oaeag8oN!_pY&GxBQqH)0;+dMTqBKF#JAgz0X=N=kyze4E+24AS% z<{*6g*1C}Gt%RsWLN*c9>$PQqX)h~z!$2_0jb?%iMdfqCW9xlx%i1Mu?~)$~E}uR8 zcAIxYWt{O}&*)j0ZF9ky8^MRddk?Kil}RZ=D(Gi)%5ID>gBmK`d7j4tddMNgxI)y| zBh57G(aQhoxLNs8%gVQFQdj()*R16AT{uzOP3o|LN4`HYKHXYvKI)l|qmtb8Qce}e zDV9sAU@)^gU^{fLIjQwhYTQ^z+gn|fba;(T?7Nu&8}_9L+?}HH7b9qHS@tk`{zN;>!sh7RG;|2vOILb=1bKM zWtZ2NA!hKpL-^zyir}TY&mY7A3a=e^e7cK}levsVr%qQIT+KoHA!4HgeX4eosTtDQT~;gPOZ8R_-Z z+E5*ZYu^fjRLNJsZxAc)_7HxsfXVt>`U04_ip*oAVFmu=agZ%y5koT(>@Ijp$ zUZ}p@h%}pM)|S^lKA@@QPWe`+0gtp(%JzByixv9~0$A)8j5axgNFV=Ym@u&8ax8WD z2+k&b%cMy_oFfqu%--=Lj4mmP1WFT9JlBPN@*GfdF<@`UkBv}8@btVa{WO6mP40WzMA8Aurt&G z*0c~O$dUq6g4@a+TsncZZ+{R}X>hR{WLBoDqcY|kVMg+x8eutMpR%L*jOkX?VeGT_ z{GG$(=W-fjwfGVS=z|q%(8^3r^<3%ovx)q~0}Ice<~7M7b-P7Md}}97Z7b(|@@q30 z@sgei?$aCk!s#Zxe6-3FCg$U_Qdl3F`M(;PR10$FOA70G>!fVGg{4h-N}RH_rT*oN zTi$*4-`G;hiHlhu!@3JnU0%QDfW}>cZ9I1%E~;p*uS3l@-EN zaDs(YFmcYDN}+v-@i$U+h7&_i)~oN#N6(4jsICkw%bAW8%DZMdz|Vlps;%x@G0}Z%?&>W+e)qgVIyVF4@Kd*F zf;N?VS@IgSU4CeHSeqhwQ62q{6zUtpUM{xSWzm$}FUtXP--m68TpFZY)MR?;3xkCh z?^2i$Wzny*k-qPYaU-dpj|Dkxy^ABxWVy#%cJ*ozedEg$2gMEgWsTD2=qy|bmwWs! zAA`NWJ^2&<|HqiR9Z9qt{ikc(V@)05c<=pxxZvGqzqWmHhu%+Dn4^DG4(xR*T^)q`@&;V9oCI>=$Kthl^qA8~4Csx<4 zsB*2}3eYFH4b-r?IOZ8sGN`1uzPgqGgi)P2pm5#lI!jARm>U_T?ah0_`O-mrsDh<< z-v{DLdY0oV!Y(&IBjUzHlu2}VRhc4zGRVbt$-r4$Erz=X(|%U$rq_R%W-%PSU%5}WowxC8P_# z$N8N~lw(9Q)Q-aPAEID`fL;>;ENdD=$JLsBg537&_jOKRTbHS+w5YRkYZ9^?D#hAj zsG=p+DZ%#A)(>Gquek(3qNVEag$BxhZ>w#M0CK zCSty&3tz2nfn+xAIYVc!umU73Opk1?QCqNoSUd1yWnrlsZzl6Ywo8Rks5k&=j`QUoarfz*So&qca+99jiF&k5OoV8$ci-tm8tbgym93fVvHjRg z8*O*(UE7(%HHyAeb^Ae;Sy({CHhc0RXr7y6>?#h(<8OTT1WQ)xT9W$D>VRN-y~LB| za5KYJvTMt1rxeDC-^)F3psE7czxVv8fBlrb`*LDfOVJH#SXKkX`qbw8iv=Ip3m>AM z;(!gMFqNM#@S9-TQjJie1J?#^5IFs*8&aCTVE4WYnX_>xB<#vV+D&?kzLWBwNDtRV zCe)Fv+z&Atp*u8Sv4F9MyiTnZ%!EOw(a-i02+&>qXrhSDr4hhOiio>C%bJ}Me_ zc~7CRVj$UL53d(PNSp7%8@ylx-LHB)GTeAL7?2ZX@gJ1l=M^LceU50mCI4kht?x;p z;iJL~3Z?RNrTb|COZN9FDcWgF5T2Ht3LlMcnzL}!k%gY_AUb+A2^8*X<)dGc)eC?% zVLmzw_e?m|uZk6^-2K4BoI%`TOm2uDDf=joZ?G%5asVi#h$Pk|ENA;@R@5c~F|Hg9)AJMdJsGTt!92se!sk=Nk z!D?F~aCYp)P|gg}TF6n(3il@td}jqob7nhf`-5>{o`fR`WLPXerCDERVW8f9_(4xE z##fNYyl51GWeuN`&JN0OVf(LrP?7g{``JyOtr@`$ee@|sd?W|2A`>Pc94HT~+{Q0s z?NdJ;@tc%ANbo3Ye9VL8EKE-O`~~dR&G%n#P;V&8rZ#{4|0=ugcqsh;-%i6`$%-?w zLUvXOC7bNz)7hgtBW3#};}Aj-u0&;L9GP($=d8#qBXU+&I-7I+-gnjK)3?v)_wk3v zJ&wnFz22|=dOu(OMfqV&03vIqP#rB^&d7BcAtRB}kiRMRfBUCXx1tKT`h8=SUT1;{f@c*kV94I80Y7*;nk$go} zYdwjDD_whnwdWn8{d6n@+>PeB^@RrVjQqI8jiW>I{Ov9&EI{1$5%vX(Vq>5yn%?DN z;|gmk3(m0mLj9OAyrYEP$ByD-%6!Z+R!j8VH@W*iDPNP5=G1UW`x|jwG^N=CzY;u1 zSUo%+$-K=Z6LT!8NLLZQ+|T0W-X%Mvv|M&OQppgNA3zs5t1EC-9=Q43){G7S=ncGL zmLwUDC|q9Vx9B6nAJ@hl^)*^;jmvvZRm}GQTAVZgR=`C>w1GXulZ(-Q>I0h1LTgYe zFJQOwQ&|s@=m-g`_W6xn3)-)C`=4(V^4O*{nQ9MSTe@iENH%_uNjb4JxHB${G~7_1 z|d(&qp91E|!+$mb6 zDY{%UEq8xTYtAkoknYAa@{K_-IGh2xWgL@qu_Nl6FJ*yALhyYx=fvG^emXfEIH@$G zxJK{dsMNi(GcnKk+=R;FPc)Q@W083Qpf{vf5#;1M#Q;bQKkj8fPu?uHr7 zpGpB5Sw<(a`0ULm{y>%JR&#<_>KJEPz?Y?2*PZe`_wT<@@r|63>m*rtBxH^N@O)?6 zFC_F)uc_E|hJepjBWYDOV~c6JqZsNnjB;58?_c!*T?=?Atncy|#pjONjC#Gke^#G< z!ACIxl>4gOP>a@HMMyvt#lE{f%=X*MTVMG>2kH65ZBX#;&PvSaw1)`ncz_Iy{U+p8 zD;+VH`4rp6frL}Y(4EJtrx(N4me#74Lfh*j9`Wz#P%dN)IWHPC^(qX%SsH(>K!mi* zY+CzvPz~uxnsrMSa>AxYy?gjT-bsrHvku+*_xccVmxk&ZgaFKLw10oAzPX3kGikd;cllZ#t zX%f)MeY%1iAsBroJLvK0sM4D8Z!?J9wNe_2k2%Los2Qw#KO5Gp8sG2mr!k>Oy4ktc z3tunTcAO~2AtGFNWCP-wmdU)Sv~{j&ISK^VrvZRp_3DD zmcmABFL3m$xG{4}7;r`QvVw@bmb{aUi@KS&lyNVIv^MsnJDr8|Y$0m(;>Vqoo7b9q zu0~cfn z*hX%UMTN+$%^(Xhnhm2I35Hz-K?M2g%cDqH!k&~_Drgx>#9ZgK1yR7bm))?$ljLlG zxVOA3Dqt_wbxU}&ddb2F3u?YXk@I79xXcqc2zZU}*@7**xQx(OUKa*2XEFj4$jCMT z+%Z}iZn9b|2QRXNX)I%{$@pggrI$s6u|2%_=}7{??^X5u5w#RWkUC_QZ!jZ-CW>W` z?A*r2ht#|q3Nc%{T0hW2?FJG$<~fCa%MbXj+37Hys_>9Lp7N%E6`*BglVFl?pyr?) zRLbJi{|xriqzk#bknrwk2;jK0>-62EC(OMzhwdtCi`L5bh6x)=@&O>WYK^PN4fiaz zBZ0SgKl`qVusP|Ma1P(T1COrbHMqaBNy#Vbb&#+&_xhGdDs+d?@mr6Y2~?%o2YJL7 zHm-+!P6mVmmUKNGXATn);mK8pFB zwi}Q0_00OzRKCY;)}+E1<2A`O!Lqdls&55@<^*AV6`;+DdyM(EJO$=y~WI`o3mJF>~1XdR(2wcWKsdk;G!&S6L>~$LN zv8s=DRtOcIZaHq(N#aA00O=_7pFG*N(Uj}3^xRWITG370^-`lI_EmKKi->uu{MTDZ zzaG_l@SOj!>!}7a8AqqOmbiELPl&W*i%kl`S6&`m$(c_F{Rv4oYqn5f;gHKffzx0} zv{c8(gsY7VQOd9Ac~fCVF8rtit?EpANJkQqSOd5;ZU82sA?mAV2@nqfyqL-OdooBG z3AfikEe3Q%kG^K;P1|^dK<}kc{nqzws`yHjxrPK+$BYdLHYD%2Xz2}$LaX3P?&W~h3?dEZQZS3 zo~Mg|Mg;~WH2D9=c0)XHb0>}S-UmI}-bt8E(N@&~olWb6op*&QzC6Y|OP$U&w*eFc zc?FiThOseLRuWXL%KbqeID3(lqFgUF_i=`(>U`{o@pY<_?5@f~&iuon(?ZU@^Z(@l zkPP;=-B@ersKWS2poEX6*`%g1n(*b*5OtxGZxznF=v(frrUdpn8JmxxR`CB)WdW=G za@*ZUJ$5Hdzj!A+{zb%8c!}Shg*kcG<_&AT3VC<6WB`cu-0~Cc2g8Nx7sD2|})`e%;P$ zgTtWc{##7Q^=~+kjoIF8uj}l1h$+fWdFH_&FL>fqU8Idd-+o zAwBZa%gtuihYe+CA%aQD@Va|JgJiD{oA7*}?_y|6N;-bet%5%a|FF$;qO@O2BaA1}>2M>3 z;hE5g;%l;o9tK5G0Gu=FjzR}_PR++Z>=cq}fdYlYV+3q*Tr;}Rak{%aVCFzXu_ORX zZV38e`9tuS>)51ORF&#Og!?9@;H^OdXXPX1*gg>nqfE8DOC*TCZ(j1>$nc2AjP~8(U zLPfTJMFjwAC?$2xwt3d;#`-Qz&wda(TkrNtvysTj-C@3kk>j-nO})NacQN>j6wR(A z?VyoAqHZ>&6@hx9Wr!&KEyUctTQWiqX-oBwwpQ#bS^3>?PsT~z9SQe5=R{k|*M*r( zPi@`=^t8^;UIG_o-%nJ(fu7HZ$aD6W5UA5~*HSOtpXn%>)18Or5B=ptp)1b|pvzBE zH&2a(h|Pu!qrqdOS64%FEcxroGG$wj{d96&L}}2y#vo2DIa;4jFO6x-yXkTa9sn8q zGwmaQ6BIjHZ)$ZlXp465z1}IiEyqn|y-7u;8g`6XBf$NJ)> zxzMWtb`(;rKDO5wozm;_o;$Z-KlVi7|CRj!2CA7D|3}Lmz<=)Yr{v+gLbbh*EL`TC zp|fSMX91w-;roA}OU;P7>6D9SwynQ}HKFjZlzwdKajW%rN}7JZx@iDJrQo72#2^zB|*vtgV690wW zff#H?o~zIAD75+9UH)=JtsMIH#1*20+)Q$ZX{j@|a)xh?YQqd9GrIx6|LR`5UPUH#l^om=ejM3(Cm?kA4oCgn=N5WZYd4 z-QDxb1K^eW01fjasph6hqzV+CO<|GKTRL7tJt$rAtEn!6?wpQF!8sFfuE2&}$Xgg! z1ucBaSYY%0?}bBH5{(7w^L;5VL<^KsTTHbj@?<(Pq<#0r$yGZ)8S?2D z{=5ccF}zg`N>>j5bWRp%qdbpQ;?O+Mf5uWTt>b>hm7T>nt;v0*ezMf{k27R7`F~oc zKe11+zB0w9v$EZ6@mFJ|g~`(Qhw;LDL1r1?e^Rado*OEbP1$Ido>-qqc!=%q8)I*zrL%l{0f`}*G$f%-!qdkW zd)r%l**n%FQ{6)S0M^vEH!}z;zPK=Z_bFsLVk3QiJ5gGY5X{=;S`P6d6&faeaq|3f zrQn5f&Kx}qu`a_ObBBZ#dd980BqnGKyrj-LUOz;$l<1^`RB~ox=L%jHo_43?BNZil z0#>OT`IJfeRv8B!+Dg+OYBKRJ>`|P|uHUY4$#uKIW4W=JvY@j0!U7)sAnyaKLMWOc z(20#*!-Oe+E~m$Rw(Q0mD+3zP4C@h1>h6j4@xF)~c+6q%qbmF8R1i8%cnuqmG4%bh zho1#R_ANWu}&; zQ=7l1z-A%@+&HoU$K|69xpiN7L3T&z-)gwZ=*b6=Yp1!>D>e zWQu7<8l+PL-9|Hf(Tqmwc8+{?g3WE`gq2`QYtWGel47W128``2XVC87~DD$8{~lzfe%9|~$4WLurZZi)MjV$bbf+3YfZp4BA+Ri03a zB=Y;9svOky&d!^yml#~0PDr>lwSUcMSHiXvse~@ve(LZQ1=i- zcWHd=&B6PqONj`g#6rRQz;yme&vw=5fXnIkjVj06R$p=v2De!Xu#ZSx@90AB0njE1 z8C9*Kvm5s+5COTBt{&&#!zGyUlqTXgvo}N9N3{>AAS`GCtv|@NqtX*;a7;r!l#QNm zhU%@Z3R6d9o%7zg!`oUW_}obM!d&8e<%$9*I(&G}Z#tZY#nWqo^^#+6H?2W|UE70N zqRmS)C!kY;RaJ)^#0sd>JQ~70^LNhfQ&Ba|?YKcn5TQjVeN14b!i_MIHdjt56IBeQ z)QT<+rYYlU6aiBC%mqIUQ>_%8q560B9ithYk#|%s*{lbbZ5eV$94>eq{hqo=d6Gy^ z5L!+7^k&T-t^OgtETl) z%)T{KL#$GWzpQ|zS{a0UHu#)@DmQHFJ*?p5%FvN?S<#d%+BO4Zxj?Y*Ww2A+_tUIk z7|{LX&#}IK!jo?BALElhA%Il+VNUXQexwlQ|NbA26rB3$yInh+CoXC1QK{lQxeNSg MsOhNYtNa%9e{?`AmH+?% literal 0 HcmV?d00001 diff --git a/docs/docs/pics/deep_compression_algor.png b/docs/docs/pics/deep_compression_algor.png new file mode 100644 index 0000000000000000000000000000000000000000..10ab273649aef2ecddc300c34ec2e4faa50d07ee GIT binary patch literal 111446 zcmZ_0V{|6LyY?H~wr$(CZJQI@wrxzTC&|Q~U}8-2#J26^oxRt)&;FmY*7?+{tMBf* zt3T9TwXWY)F)B*Zh;VptARr)!vN95CARu5?ARwU0Fi`(mAdu_O|6M@c)TG5g8fFMi zK|n-6WF!%$LvgRc|6d=4g9S5S zp{GC`ocOOU{x=koeGB@3@Ba0Vfdm7b;%sz znFyR{;=MOJyYM6BecM-!Y(gMJNrj{wx}S-^R74ADqV5Q3y)piu;r&Y*{J)Y;^c*iG zLpQS4GTkWOb`^g3EY;ZSWyE21A$#hVnf+#DLw7YFGhB5zNAWwHGzC)}mVOnR#4tvCC27yIjKHJW(~q~0#$O~FV7q^ZU`T*6`i zSVt3!6AdTx@GNNMI_q-u!H%=?LWDSp6R0L>VE!cAbogh)%w>PT>ci)a79)34nscZq zf3!=p`<6tf_NN&xYDbGRJCXtNl7)t(@bAIHwE2E(uLnXt%)D7&QbqLPOAWrqkpgOlF8#V6nE z{BjX=lEMd3eQtJe7gpH%fj;qelsvLA_~$l^#W9My!ou6bgWYQ89h_)>sTxDM$$v>i z;q4P_k5X-9V>`hX{LLJg4}Oz06P%L5q*hjFHrsO8K|~pn!UiLajW{@N`4hE}R7$Oi znJjgIGznp(c8DBzUGSRU_HQUNs^Q|JAr-%A&RzD2auvV0N_T0ut;;SqSt%ZRom0CGaICb~XlknGOShALh>FXhS zA!^cG{3+NTz=ozu2XPx6n5rXj5Gc?;X?UJjN8?5C{Ur*MUF;MVi(>2@%s(h%8Al=z zgbNbE2DU{M7=5lmbo%T(6qg}>E#>R&ZNbU4#X1_?w$4&ojYxr8!Xs41UD;q1Ma)-U zIa4E6sfK_$VVOeBZ94hr?rsD?X4hW@mrZK&)Nxkm&XuU9sbWtdbqH$DjelhC3+beQ z@*DFS30J2^WvOF4dngKCwrW=!OMo3!oM>K>w^1WO2F^SxT2JDJuhy32!IDl;vr&Lb}4X z4Z8);9gv`57(~R+yb_oCoX453p+LC|DX7gff952nu#_`ZujZ6eB8v2ta(Xd;lnkE$ zZ*tz{0LDm{2Dw?63kZf-(sS>ZO7md6aaFCjqKqOqf%^rZe#|udu7sXoiSv&KB<-n| zOj=Cg|3#4eVI&6IpC|3g&_fWJ;FM66$KQNyN1(tc;UXLW<5XgoUb6zZUx^?s&To32 zPCEZiu*{?%*S}F_dZt16*P)Cw^G|VF+mt56_E4p!z01p}JjHo-nHazcN3fl^Mvr3e zDG?LsKMMg~nIHzvk9RqbeDmI;7+t>rZ60I?0wCym!A=|yCJ5mmCeZr@(S03G^5+Ey zYi3@2R)`VBFHqQw?qReEst?$DfRN7d9KPKdUHfb-YH0}GJE}AijHQ$e3&-pMc6Fs2 z@t2=@fLaR_{k`ji*9_r}%XQoup99k9*!H9FN!;O#cIx{#68c!?*rFgdsS^vva3AHC z80p`8oJv=rQ)`nGmb_Hez;FGs6m+BGXkQetvtEk9I$D=+`$p)Y!*_xZ!;M> zz2|UXX`t+5z9;;i0$qRGBV6u8F~1SswU`HqtqVyvmv9;Ja_4fK1W(TVyG9f3r@}Me zrc4%_AnEYJ_il$>C|-q!<68G@Hj1GZdNy~7Q^KL23&nq(aogD8L{LB5v^A=c-CT*9 zUBxw;NTCHh*_p9^u|E$ zLz$1(^m;SZL{-yg6$c42r3YAH@2s*Ln=O`;zE~!46ZBsu1QODbd0)FD6t;x5W7td@ z(uK+>I4IDc2ncz9Fj#>VUDsi6N>*zOS8)f7R;nC|I?~Pey zDvU7v%CuRy(og@z4tZMd28sFML@-!r?uDu>rmc2DevvDP2>;YISSLitX`_wQn8-Am zzKx_Vz`8qbufyR9p|!15HlMUP-{Fmr=*XWipa4Y%&+tl=seluck+rDOJ&A&FY5?AB zon>+m{&+LNBIzS-A6{S-k9*EcDw-2pdtPZiUoE1O?m7=w-AS`=%nSBy12Lk8lao6n zQG~_V1;x@#1E#Bo(?GZuk%l}xfU(=*;ZI1VY=mrFhIg{uO-^8@ytiTt@46tZB51KU zzdYd{ehi|PuV+5q6j)XEjsh>8G}uMIt9K|t6g)D=$ri$TNg-<~)zr!s*2jmkl1TKs z4ZE3_mVJn5|6zC0n?R93CLMM=E|1mg35?*OO>DjBtt7_-rd<&?3DK)e<~}2YKaa|6 zd_x^Y^EBF&+_wzw%H(N|75K=3sel_(1|MmroUyDP9aYB2syaH0o`jVx!nOf?0xD9{#DT(c*^GmCjw3a*rt2B8t=H$3OFIlR(Etcp_6U|CqI!t=b)&_B!NEn#=Ls zj4%=>S8^VWs{{O&c{;U}3WPfC?HmN3&1g2S!HfO%(dC{{3?xv4C6wmsL)#a~qHYI<|dw1BJ_lp&R zolS{uz%gjV_YTZQ8tRhbj?oKG|_V{YiX8pmM=U`Bp zxZ=D`I?CB|@ZMBi58+V?k$_}Z7Uk{-D*@v7{FU6iM9dThCPRL%>C~U!FH$C`j0(3c z(-a+@(qvnM(Ch6+p|-WJI&bI5EYw03PC5uF1!E@bSG6_^aXP{mu(gg}mea{x!# zFPhAF&qXG3@mF7?8Sx|NkDbs}=-`Jf3O4&CX9L{#cEZ6K2|K)QC!6g-9AwYJ!Bs>4 z-69Pva=cVL%@yP>3duap^N=J(`Ujhw+XQ1=0>W*$LIZ_qH$N)oA2!|aJE3+))rp+|1IEf!B)YN zr8bLwxG!I@rSKt}aSduah753KFya*)O`S?3)#d~sgrc-1z{nPmO-G|AQ5Jz&r{lL9 z%ou}Epf?`(!NwifWCt2(X#+zgaJh3=Ng6tju;zN%PvAeNFvG3!=9BN{S2<%? z9ptj2r`^gHIE<(mA@aYGWkq?2{{^2ZSWQTpvRsPBB=E$Pao_vCCA9)$B81RD|>&CYhtpB zk41u;TJ=Khs>do*l|7?m!`HCa053h|Z#MPgXE@U*ddnf@d0NMC@V<)f=H9H-Ov$S{X^XH*5z9wy{c(hzZ>`a5~j;9Nf z6fygt1|2}Z4XcTn^mm&e^lnPSoPpYSKgw4+`Ar}M8^%)9Nq$CP;4z(y&L6j!4+&84sjKHW@x@fcx^>xAx#&W0mZ)wCNbGZXF9N3W_&p21EuwWl;;VSsTZ}Xzo zVfs0d+lX6fFcuHH@=5CnZ_>jcM%;M`J6VD~WT|=5msjq<&uHg-xhFQ$#)jysdO3m!1m zz(N<{2Ye_dVDKe!mq~Vm%U7z_FkWXhP2DtNv9%$58TM?L8`mZS5p&Vl;93KW5naZ} ztcTG`M!}I`k;JxsuC?9{OBb6;tG8=aSpX`QvYmszXa^c8Ckx1~nBW3g69;#R8&l?q zkOxyAbEcsz6j&joh?TWGGLF$jUQ`A3;Y))({ELB3qEC0NxkuyO$hgnH0=0sIGP;&` ztuPzD4s6$ASzL^=1gPq;EKpLb`j#_)s41%cDaEkTvXQNeSD|8ojADRSFSA1Y5F8yHa1xlUN zFp`8BLD>YO>Bl?}v-q>QE6L?Z3g5lH#sv5$0NCpeamK4tBJo#Ar}Z6$>tlh4iz24` ziv^wkm(ovslUB%73h~bpdDopAPlkL}OaNBn&_j{kut8srg?W3>RX<={n;xAH%sXEoiC|O{S2k+YoW3D9O zzY_0{2S$k(6qbv%3+(hp-&9hH6nJ6cn!yQ!d?zL=Z0$D_#g`h&*2rQi0iNt;tV`wx zh-GP*6`77jZg%YBgEu!2mAjK{UCz>;@OwisL0Sy7xbp(D&lZ1Ux#{F4AH`{v*2G(k zNJOHPWZK*YuxIzLMK&)!4@ZSpSbXE_0MGSX@wVZ9tV*wuw)y+60{=AFSwM`KaHK_(8YMeZ6#K`6O0_M$ z#t5ZCQ4hg$sCqKiCjvk9SMc&@;9w4kUPwPrv@XC)Wb+5yvJ+75cnRleswhHGxx+b8 zAYmxzLmmZXhKHVWU;bTes8qr4Nmr-5%0@J8j3S|_jYgpC7U~?!Mtu^&ODldj_I@jw zejHQ-eo0s#xRYKpOX}h)wNY-i5^scU&7l! zDC9k*TIPl~X_{YofInAMZq1AIphDT3Y)%pjna#sQwFY+WFY4|~OAOFm5TrUs*4x*C zE70K?Jdk$}nf_SIMxPQATAO)Ei6c;7V)eIJ6*-SL^O z?|0AoIA*0|UIiaV_f(}f19{{QE@yTT&om6RbJo#z^8HDP!D(UPq8n2Ke!TOOZkYwK zHuS80vdP-c>!h4%xQ1URhvS=0}o5N{S5Vi&-XH&l(TtGk<$W0<; z6%uUto3C2JsGP!j`rBKpW)6LCV*1`=#mE3?Sgaw-ebzCKpfzh(+ZL4h!jE%(EfL9P z(k1ta$>JZ(Q+fMQtZrJx(&w8ji?X5;`4|aOOyc-aCKov!5Gxe*&31Cm>0B8lFXgs8 z+pY*&-+kDeD+u9k%er~94oHOs@qOp#;?=JW zl-fMV_0qHDWR!w5AzJ{fp`Z&424Wy+cim$O8ZInh)=c(xpA%@efR#(N1|b&zlZga0 z0&7L@|-iTDcJVv7k*x%tS z?&x^HbYU#Wgn;l)&*O=>Q@d(qOmu5`3$G0r64F$n77n2BiDBEiwLiI&Q^ zBq2v9FcnI2-5$pNv03U;uZ53!G0hR>%ARUwlZQcLW(gYR4~-yBN4Ts!wG>vtS>K_) zhmuwv+8jX{Z!uIFLn=s$=X#2kjm-&tYv@}pGBqjP@jgOqM~}T!rxALAT2tc8bWq~m z6@z>Hjp2mci;`mtF-F&>@^=CXitjWUHbEqIB$eq2$5IgjVFGo~Vl8Z@8K@&BoFzh& zJ_0ViQe8i)OSJwUD!|EFGW;IBc#+_BRL zu^4G{ZE#n{$(|CG?X{wUYB-jd?7XT>1!NiK{LgvU>O~Z3cUTTRHL#F|_!klHF5w}! zHAEI>L)^bLU8z(|`i+tT6dU-@r)*;BK&-#^S<t@hqgfwGAj~HKO+n- zP?+(5^_>4!Q2zg;MXa$RX2-v;XZ{Si_-xzupy`&hwxJe)APgsnuQvy5yKea7K^o#i z12HE8U{dPPjQN5u`H^=H4!bRGjfp8IfkPz1x7;(}@4pB`e<{)Otj?8;bEm}jBwgww z4JtP>{J~@=A@-k#xj`0zBj3vlj?uh zggqxymCa}0qUn!4jLt4{@6|+6GdUK8^=(R-J`c+vMo?Ld^~i>d)@(2SQDb zK8kJQzN4n+B1f_&H9KG05%{9KMw)ilwN8X|cs=3?kYl*hQ$RldQuyuiV~QZtIj&-0 ze;nGnE^u(J(@N%R)a?Oz#BzLRScJ7z$}%?$)j8a|r%`+W;agY2K^d{OF>s z12-i|k+wv;*?&#pq7S`Hwm9~2XfsME8q=WN-yym$3K39gx>+I8g8KqWN)EN9ym7Le z=`t-$=;XJulLVNB_s&A5NuqvbfmkZk;Sa>fEbz`)xtohRtkgibLqRs^J%VE6&r(Jq zUB1T&GzQ=!m)*x4C4Du*&>RkS1g(R!;)b5aDZ`v{IDZR)1AZBS<_P-pTS)BRjrwH& zy2Kl2eAR}3Rk}B>e;P7SJ>Ycsr<713u9r#bl1mtFglhgIoAX}dy7<~hzSv?#Z(C3@ zeXXQFh|lkl?%@lq=r<0Ne18S@o1=4mHDjMgX@ssMQobxQ3Iy%SHz_fwi83iu=qn!h z(Hoka>6S>yE9iF;h@a_;W#GkR5h3qf;fFx9=+sR)ex}N4RYnd?Zo=m%IkMQzFk5_A zqcM13T7c@B@UMn6d{R@BWHvC3QCFmjhH``DfFx6=R)fSQ$;(hVgX!MdQ5t?e50#B6 zqY$J;bNslr&}t(OWFU?sJVzqA@ql#37!UdZu_Zx*=+vEQw(r4v5i4JE!y9A!*=71? zSwDILgLM9!c?xe08l9^2A-ZmS`w{@xK6UW`J}AOZOuHUD>tyR`wv0S{nm5X;LFu8xn15@o=O+TA8ZS>KTcx zwcr5y*^lT3IgauxN7l)kCLl9p0b|elHm1?1pfYgJx%qa296`Rjg0@KjINb+v~ss_RfFg zN#O`~gW^3`gW{JU>u8oT$^>jAB#P@%t;(w96)z`&tD37=1FbvFY!_;136?j*yiCwI zyAW5;W-(=G(&mLsM(P!DeC{9%y`#kY71$99gur{x*2KshEE@!(N`>P^KC9|=f&^h= zZlQ0^m-cj6OmXn|tUANBM)sF0eRN!xv14puv$+T1vb7M+!~@t;a}LKfF_ZZHw9F8ds*}&j-pMN{59_SE+Z*^eDI(!DzjHG3my8&+> zE_BZYtLYdQLXAD7svgQy;wvgUx3koGBIi8adbNvICz)BF8f_yH0C9c1GPhchm7#1bQX}uDCHxRqOcVk8K*+1uRR&?ZXEX=pFv+#Ol-^;Wb)lBSHevhhv8)ES7Y+>oll=&cs+>?MT?YhfNInuno)22TpMa zsc?1(nTgmxxlL}@z+I4KBXdvcKQ=)W@U{^#r(XsWL-{H~6ZgU+(gfMrUL3Th5_^0s zrj5m*G~pPlhQ_ywQ;(?RfvdWG~~n zW!mt%bDhO18spbhNa|Y9%%?F)p?-2immf&8uMqj@`ST2o&*(y>4KC%5c6FL&Dh`Ei zeW4e^Ls|KFgM&tJ>#qRtY8Naa{c;Prl?t$i3%3LYxfCu(=(3!L0tI)6Ib@g4dSG?& zx_djo>22488N0DRr3gO#JM%mmRKicQJ6ePQ%Ye`}fB<7?;_KHj6}cWj3jtGAGK_Lr$sEehaI_dNj@CEvVU^_o6f z&Op@6QZ<%FaC2SB+P%8fCP|0c=3$4+bfr26R_JAqNAOA&%4YhwAQR8uH7%fVUM-8q zN-d7}zHHcedv*JAUYRK;vrkGNiL1IgfHy;`7VziLoXRI0ZmxzHbbH>%^ruf_^^ty8 zh&xe|w9U7AaQbZC)s4T$ixiTQwz*wzRR%^vz*=2=P7kCVLp{1a= z1t!kiV(jN(vG3)>;-jSyKW4<$q|V%aWhY=BqrEyMw7EKE-l*ei$FVM4%YDx34wtw5 zi?a6YY+NqAW*{)TB@43BvgJVAPq(PXb4tW&{G#U3%%0iTUJgN+FIThij(q(p?Hux6 z?Y8KJGN>b@&9&JQb1l(4@0rj9-^$(UiR$-i68%;4N%4MNPyt(=-8!qo{$Se0O{Vec zv*w?cQSo0aL7?r{=eqLH>{e=Br1-rb%APLAbjytmoPa99Si`3{i|1!Wv$p#99P#}U zhv67%hLcm)<;9c($75n(U0dF2Wk6QWSewTMbk<((8#j@_s3$Z2@L5)?87GvZ`QU*K zFv!nXYxLFcNgBjFXOMcexU0QRqPNfCmAHGg(nlEZ#|q8rVj<-6##-U9JL<8@D3cSf zBbt!kEAldbJ1T?TDDqLqqsa1$TJPT6SZ#VwFsZrP5U)A*Fh8eE99Ii4P18Ors28W* zLd?(c3%OQ%J|@G1ZqlHJ(|TCDz79b^P)kHrVusX3V)zVPV-9o&+Vs`i^Xc)^F-h5u@7}&Up$EIDg zz1qo8yHw7t@3XAU?_3epEU50+QglRMnX9XJHJdnf{@X5#ewx70zQuXn!_x7*qe)NL z(SlTAdT-;oxtS2ZNv?sZF5it;tC#XNk3M9L_tw0`N7{E#6B*f=r1(L27)smAVta~j zB3kkocH(PgwXuMXUE2>{tJIUZ{Z$Eqob@ardDMsiXSQZl2OzbN)Y8O#1fghFL|A9; zW3T1$f{K-u!9VCIXm7L-^0@U8ZV7Q|c4=S$oP7tQ z7~nja3xEsYZYGJyrR>+4&CO^Skh**{Q=NSRd~rJHHp3CF<}+sLn%F5FuPnUS5uY2) z7Y6hnK*>*)@;w~=&>Cw7%kvM{aeok9?(6|S@0kg!D3@sFBxJxng3?Y*BO)T1{xY)iYZ_QCVMn$BznAW0!$3P9$Bj|UPi2@ zW-9RciIV?9;fh|oIG3%aPg4Vn0DSDyW7ojQyg&vns@&frkbwJ1K34wsBZujLGl)Qa z9eHVat)R7xe8`>q-r*eam`I2H1PifiEPTCM1be~rOsI*6pU^te{N&3PqcIG9xf3%A zb+fVB7c+^Gy-@3)&qj6WisbEAqAd5T?9rX+*W*tS^gw`ckX6Mc9$@>L!h%xC;^)|k znbzT23*ksC<}a+~E>B9GpAloLlP;zaDT74Bt0SI;L2~l0L|bAcyG(gs3FOn9o2ztq zfUKm(GFR(Av0S-B8)#@b@3Bl%oeG3pW`9vV-@h0=ZDfPA{x;L$cU|(fd6DREjHck8 ztpk+UzfF!C@~wjuY=`S&J6d9*iU8x-*sJAOl-WM3SPnA{@C+;$kn3x+DWbC(h`Q}b zpB@JUem4(fdGmi#R<+ItwTQc>oD6}JR;^x^SeATE!bO!V9#%B^3(GN@XU!>Z@NI{Y zfm0K?qo2&{va+uEjKo{nIYYsJ@mkYn{F#u$k;Qg8iXl^&A4hbRVfC=0Hwf5+hhus)S&M5{ zHVai~4-GxR(mlLVo@cO|_fbvWf23X@UdVYEFqA*lP4xA2*mW%qqjYmARs zm$Qk38m`;D_$I=X)hZFVY zmLB9C!XEQes!e+aaWRtlQl-eWeAwCTb4JCDTN_zLGRdu+hR3+Qm&GOz(h^zEBo5tE z!_GMx!q%^8j*hCUwO%Zvx7k8#z?dz@y0mES9qdQB3srzY#D4pcqHHyinH(&kW-tP9 zlItMW6m?@fVTK|D`^o6Ypw_JrMpb|CG-Ac25Zxk>s#dl#D*U{xk3p83-qmObUp7Pn zt+KX(A_pH_Z5y~)a&J&@NmoofJI&g0Da6%9cWznOt4E58Qs$UgGE`+=!ELE&voDF? z1^0d#EiQ@muRc@hrni-Ut3cMd&80DxbBShz5a;2ni&H!PC@FrYQ_FIm|Li8uW(7bK z5mZHuCgAO_w_nax%K99P9kp40czq}Ic|!awIxk(Xq{R7K!vi``U}=l*x{e6@Aa-SP zp2DBJlpEdWXs4=PYqH%yy=K%CZ5Z_(>bzofpq|z)RQo&-Y8)7vHWxsE(c$R;Q}`BM zEHjpAW7LU}7-wH1KZ`|EeOJPzHeBi?{lQU8Z#N$tC(Mm1Er2*5@JSc%U|ww-?Q02o z?JRA&GCy|0GWt)|#Zw#PrZT+YciZEeb0@?4T!Tj9_-w1u#Q6Hfw@WqMPezrWq7W%Y z5u#IxJht}P^cBqhb*GXBU&*{a%B%jE#rRZYm`f5(b1L#PF&mHMT$61vHJKtij)gnY zl67|$$vnz^;Pi{yb(K020+8zN#*Q;~gglmwNJO(8gcwLI{Rc@`fAy8@AB&iibAd+# zNvO6=Iy%A;s&XYSMUifancQft?)#$Eqg6^CnIW46Qe}|q@`KndMdn1LHa%CE6@pvq zJ9IFPS=68l%1*mJ7_03xUunowBK4-q)N?h%D#*?T6?M40P+n+|wpw`vmTV2y|qTehu(;*t4Bn_ z3$9T(1sQ;c(ZU8HhND*t6mm0AwH8B0xomF>W9qJ26vwr5OhyR}58AGpbZYuih2x!+ zQ6YYu6<|;=_*#%sQvN$GmI`VY$y&?u03A?OjbW==f^y~5U&+^<{5?d?NEefH2Gg_0 z=mN_eLE@k-cE7w`lHD-i9Njq1SK4SRtI^2x!i1@WkTNHc_GC|#<*_CdH)gBqKMhVw zWt#X>RasQcUZSB^9g0$j@1*yjhK6{((8O87kU?^xDo9b>FQg_BAf}D&UM}mh2$!un zI*#eLyKusyWY`yiKbC2}O}swu^9N237>K<7Ym3#Z;=A|y>q}II%S9xCt;76t(rZrb}Ks$ecFEgf@`RU*SAoyD=Yz!66c@8_3o$;rz}fq;M@3!-h$ z)I?j=8_ui5elfBXIEAttYHSbyY9G1tGf;+;PCP)CKHXNYSOZey2gmP0?8fALEi|%{ zMwty#K*zt#NTrkR+dkTggZHzST8L7`Yz}1AT zd$&v!PHRb%mfYx}W!Wi-8%v*|3yM6{k}o%T<>0J_>CIDO1(KH5Fvmg}*e!!WOzIP* zN%e7kZ$QrPgw`o>o%tY=k#x~%cM}^2_%SlL{HG@=96S;JL(9DhbYmJsxj}=|n9ICG z@wFeMNcAtcmbPS|-JnH;L1ftky>>4OnIcq`nwCn%u*{@Pm}ufZAnbGR(}Q+l4&p#2 z(e8e8>>n0(?-3CWJf!Tqa#RQ%T^mG_CWLD~c7q+xYJyy%sR++@9gqpk7WQsyhEpX{ z{U|ZEhv0QILp7XNt-i}FQjbq&($=dn4Z8Zyf0Jq^6E z>sesy*O2{H8XTpP5HU{C TlPf7$?MB<54<{>vlJiJ=nI7US^IT9=< z|Aht(zEg=wk55QHgWTP_-|`*)rvgZ`#PR)1ou+@+Ds1$*Rz9jGK2E=TI=w$1Qokg5 zD3=FVI0|)1F8|Fou_s|4{J3#?Q7a`IrIE4kqak!Jniy4*j>Lo(hN1;ub38DrT~aNg0Bi~hLP+<70w$PnVUY+A$NI`SSJ_JciBUdjf3rnSpZ z>>-S&K_3F3aQJBrQN-mCFs~e?kVcsZVlqI#RNUmJzQH z(B1G@eYb!oiI?7-jTU7$Uy!8IwM(zVwa=+nk}(QmR_Cy3*eqaqC~P!YT(KE=pCQlQ z=QW!=>V~J$BB=;3DBcaGNRp2i9(oESsLfZWx{pxCnb=w|GjM!0QRqFD=5!C%}1{n7|VEc$}iq|_cmHq)WX*S{BQevfr z9sjcxeITq6l`w#oD#LA1Xl9{sFK^=XR@5^0C|w{*YRGB7viNbo4SysA_2-Y? zCWliS4JNYC_e;O%IY6WMGUorE1nBiZnmrnK@5ll$qn#~G#Nsbz)P(=Dj)AAqmT#7ab`#@=L90tk1GYLk{ z*BH4_2jLi85E4k3I|>LZ`w_&r4vg^NnlMUExnY{zL%VBisNBD%M>90K&XnTd1G?=_ z*lcVH*+qxmqit)+n0AQ13)_eHYVG9NYipN~rRe3;o^-%H&U~=1cXZmCRpTnH4Y5B3$q} z2_E#P1(8W;L}O)5v`;b`tvVWnh)+qLGSy)=o)#~pxA=*HjTDR6!R5#)rk<_tc1KpE zNvqK`#ct#I$0@RC_Qm3JAg(-ik{G3%A%slC9gaolg=CNKa=vypi#>g$`x-d7xjInG zK2eJG;4plzLbKa+Z*DBR`AgfK4-noeaEgNI94@;US$uX*7sEsZ0TIjP6s3ikWV%wP z>ynuurbx&JBVdl&aQ--SV|RIDj(S+D>rxWqn0pOi_hsN47=Q!wwWsySUMf* z$@bf*E++KSgA5_f>5_%@jpV-){kLw_&#_=4Pe>8$W_YPT^N<|-A$>b`(OfQ)9bK78 z3jF25+F-pjw1Rl?C^GJ4^jmFsP1VvysagX3 z7EM4AWY5DXThG6jka0=@0fee4xl?FqsXAt20&#J1laJW52e@nRiK`VhLKE?ldUAO3 zvofZnv)4^=S2Ol*X*jsa?i>IYU^Q{^D3lE>L6HXIQOmDB-D8)1t z)6lD3wUUU4!yECr@rz$SAm&PT%+Lg#FL0KM;KtO}R`p^w#ZhPF6k12gD~Z|-&rrW< zH?PosRK|AzlB>-_x64!~PZ8*0>?Y29JEWmCS;4FSO0rHO-Ej78ZyYl=0xe{!JyU48 zLiExE6cA9IIN05~z`D%>XkW1lvpJ>?yKV1;fjQJnx$h~`DwJ4yD&(Ne5E=K6Gi~|| z?y<4!=0-aA9Zl<1vFrMevwMaq7#NUl5z+<`Hma4|dK+TZRvg%6ial&}{z%!RGNPT8 zp*AXztBh!+WcAI`5k~fWys-7$9W}>FeV`JaksA}{fT0lJ9T;tIoc@zRFC8>f_x4P_ zO7x4y5r5LS$&kGZkjo@S@X4!fQ6Jf0;$@>ys|&Ku(L?y|dWMQsfarRH$aOr%<#1Dp z&k!M()Y*|7pxwtouinqU?sJLDFJ@J@rVJc3I_e59xOsrddl4see6=_&Rx4|8IIYIk zvxl=SaagbNth-PDWKb*C!u_MfggbuL%?J3wgsHo)wFw+B+SHlFg|@w@)!F#GbkM1_ z%WW~Js=I6BZ_4o_V)Ido`rC77{Y{Q0_hPVnwVfkOAWelhbd8$OWbufjfjwh%TP#Y# zet}o{C4dM^xMH#>!e%w;`j6WO0sHanak3mFY}X(ZYmA>0B$5c0nyf|n@mHSD$|R;G zv3-uY37ifU{{?OE)(pdo37r1(0=>6!(vmji<|!^5I27Wf61ZiQPR5EM#9UTvya1cH zY)3$_1vRPGWx9P-(sX>W&w28(9)t|qMj*=Uk2j1gqB6EK9xtSW`61ZugqO%>{6eK}Iz&I{ zNck4u8){|Y7^pygJ|uznGvXhxL38lW)p{X_n2x8|U`#>P#KHaoL;EEd@l{ran@TR8 zEcdHA)~a*pRk|OE`m2^KnaCUSoNGJuK3MkU%eYqR&%nOnq>Df=QslFJq{F?TF{X_X z?ms-_CQ0%iEbP+=Im7L=V4eG?SuKS# z2x9kn2Eb1~3&_aI{_~xf^+{oGr#(6+VKMB(moynYV4cv%w_;jtK=l0Et=fEKqn*FL zk1GFABU??AqxHW?ID#0Ys0b-xjZb)0{K-5mv9P$9lAbM63i6s5xPP~w)o^Y;iSFNY zCgWjhnzC+nGM0H~d)nV+(r&-P^wH@itWY1DRwQFvVwQf(ifRGM2UTJCZ#wK6;p2-& zh>`G3C&)1TL#kf=(YpMF|3h{{Iz7k+hHU};Bo~0GtN4>tONOBdMrkMDAJ^Ze#6FeJ ziBI(nHtPFwW9qQpj`%wcn;N|4&1W;}1M%{`L=rB|Q$hk_Z+~C4QUP}wIJ3WYAP)bA zzX01RDJ@OT!GV!Mr}>6&_wqmnNklV;Nko)r_qQg6^Zmm=+~4=LC{4Y1dTb2Z!WhZz zc;+vRb&>j3_uUuUJNHKCGl-Vt1f@1U_m01JPLU(bk|Xx-?*vkSM1was4kxm zMxqLkY`6~nm56=rg~l(grtwq61mZx0p6rbc<4sz%B5sF5Q|C3vQS7c!S+0RLbS84( zMF;OKlM$Y`o()c(rIc$!+^G=lpKWWrFx_e?2p-;vO>KveL>brC_OI>V_viYjek*+# zyyH|#`BLQ~Ar@9vT8_{hBt{}8gO1>_Bw~9T^j7!?Q5JM6?cM9A%k`J@#(oy>8xuu0 zOswe~Cf;!&zb6KzBBA;cMU*KxbjPto!bO9(^_@7!?_rRpq#h zw8IGN9_ayOXcTi|uv|_|GzR#%!}{QOE6r;X)iYcKN%Z)$oyHpa2y_9Nt_ZODEeqai zR7#3Q>`J9hZY8pzj|2oW6ti@oN0voi9!9hah*5FQqXJD!u}(z6{s+MV<~gnsvla%? zz#v&2_ z+qP{RXP%SuzGwfbZK4{^>k(BfvhU zPRo(+h!T}WjE4F2Ig0if{*ui#m5>Gb-?lfQAA`#d!JnS|7aOn>LD&OYPX|Uy7y3Wl zaL50pr~6}$kN%|*+zcZ8OeUcR#G?=VkI4Oh>EqpD2Zw*i1&A*O|5v zo_PPIxj$U3|FL6sAYGUrpn(?yg`2?tClZSeh-&bey)%^=aub7|A$b2E*`Wg-aP%GN z!)IT_R~!V<%@w_4rke}G?AM^yDQ{7v%d)(lC0rC}(Tpg>X(&$7)UnPM>T{L>bU2#X z3DAO#zM_0|$8`NY^vbc2-PQp9yVSfYer#!uW1j<|r(Ive;GNHZ6=Pb20GLqDX78g^ zs^xtK7ja02-tJ1Bxnc2h!XK1ZvK~muho57Bf|;8HUo>tIf19t_lb?lqbXka>WXgQb zrJbxNgdHO>CJE=m)ay-L^Ek~*m^!szWQ|gpGAQ|2`0Q+4?@A9q zup(|~se$PC8PZm;B4k`KFffnK3Do5css9WJkS^Mm?hAE^y8bA^0CYCv+xS(f;M>K= zs@+2<_|K!ti6xAUi*#)Tnc9BreX-NY`QCxI?8dpp6h^8>e__8R^Rqb{fypKQCEgtC z@q5O@@Qc8khQ(i4GE!LZK0sNO0Q(Kbz-_4;7_c_+$Su8aAsi@?Wua;HQ&0>&p6viE zj)2%`6vQBx4t2%n3vQ3EW5VD4F(l=NtpFv=DM$8E@MN(P0`?|E(=xwm!LrNfEMp%C(9+Q zXPRTNxt_z6hdMWqK&X&vym&op>pfJI&CX7Grno(d2yJz`EL1x76g9V$wk+vqbfA$I zV9NCQj-@@N5k1ay1VMrAOHsbrlt5sDUl)ugRPfgV{Fo8o#nF3XR5^gUykx#dLtVqm;rr$D+zngM=Dqkb8~9No62;*; z!h5(vVq;^WY9&NgmR>OjC07I{0>e1lE#^81^u?h$rSJRnNk~fdPi}}W&YJ?_s%nU& z`SXs^KzQ0Bl?yp0D4#;S7SCe+U!Kkd>!A0ug8%_@KS@bqi@OP0^%#hluzPA1=u)w$ zautnPxPLhe2(I(4hWdu6#Z@f?D6IarE!o6*LlP({eq~*XdK+LB4i=p&*ai99)-T^t z0Q=b8xA>G!sDxoVT_hlk#E{4k+Qo|~H)SPeposO^0oL73*0+63nn;8-fJGA#8pz0F z6rAPzJgeaPIsiR}LKmo=9fSUSC6tL7T(Zuz>t>CAaB3zF%^sLSj!wi)76cMF?P+yJ zqG-nX;i5gLt5mg*DnTdl%wy~Hh3d+y7@L@-!iy-$NOtgyzq2qvbz>{#PsEEAgPu9~ zXqYQLA4hQdft&I62-cBRER=);JZ)@Z=azzxl%y!m8!s(Dkr2wbd+dap&}`Y80Znp4 ziU|AgVO1&uf@wB#pzvG0-G>Wpvjw{O1Z8eR0^!IFF#$fKXV=$=f+^4LV!@lbWe<=K z)eM|3zUaakm08#Os%K}j2hWFq7V71ZzGf>y5dy3Bg5R}Jext#v=Oct)8p-y8n*J3G zL9TCOQmUs1Ip-OO2*@wY!wvWUW_--u@T@53=~Iis0kjRfw3#5TXy}-sO|Pl4OsELUxA5avc*=GT|h>sX(wnM91S zis%!o?##}UyCz!(dV7iLYgI>^s*nI*0p>!GM`bxbzVAAkDMEEd9)&d?O8|>vRVNWk z>@NaS2rXtkgXMv=Fk4Axr}Z>a^|bNl2yyL9Amn&4_#O$Go9sN7JN}u1TVaD-nZv6g zids{cdn19a&zA>-zZT(_SqBh}mYMB**854y;YW4*^it(~ z_+9IQ>#KsQ@$NxN*ZG5;%6*2DL>5@kQ5M#5TwK&~x}_!VfvG(=jXU0z7L|00E5{rT zU8Wz&1rOhO1N8g-gs+7}kE`p=9d=B^Lx8p|OE8mF-Mo@gfq+Y)#e*lAMmNHE z`x~9kvrdB7&UFa3S7rHklt8UPitR^@LhH$!*eP@6h3Rwd5K=%5;2Gp>r&WseZaCD}^SH)4nZz`*+~Y-E zy%|!qKKXiNP>+?K5s#??L)+2vOf7i8Uut5IcPX2Gu=tzNz`wNg?7)Z#ny|S$HEguk z7-hB2%I4&2%*Z6^fkxH5qY3Bn`EwZ1LAjP=*--2dIFWy5DtxzlFi&_cX&%x&`3%}f z7`>eFDCZN)B2#C(My&&;I+^yWQv(p$y8KUe7?lqS@|N`dUGxYp2ig1V!!_d%C~TOZ zoRwga@5O|jFk$HM*KlU4q)muxsicTo8T#n-DS!;vJ4X}}Wz|C3!iiYb86_C8b^470Q5?7Q%3FS?EKB3Gcnw zzJR#<0!ye%n(LL$5<2pud~nl@ygPw6eK(*F4(XA^MEQ4TXHu>mE!zYL#EivjTaH%> z8*B?;U?0Nc#~?UHJSp3+wQONVD!(or?38oV-%Q~_8vw!3g9H;XutQkG@gO1ZA?}+E zPGw@>UJlW$Ttf5)b#fBUuER+|8*vixB;|y-07H+2aXo_5#(?|3D!4HSTH&%7DdHfh zW1p{ld$&T~L;+Rdz#!1hfW#n5blLk^MdGIbz=T(IlQ-W<+)UK%jDvFdnCF^-JY4i3 z$0H-0TuyjenO1t0kZ8nuiO~0dFqh%l+Q=Axmom!%OmzR}l9Ak0Dv> zalf6fUe=QA)|>hr(brizDJ&h7L;OrOHG-|MZ zPM<v##S~p?YGAzq4e9UexmcN6G0E%4Pq@xoufu=!qFQ29c z;5C2%GUynRlCn}*Z)p*(N6Y6M-=M4SM}7w98JSs+C+tUxS=(>=G@y}pL_=c6x1sa4 z{p4ZjuqAANc!WI=Qwjh&jxVxSM>;e7jeEP<-ajc19wIXxTnyDH@~eab@T2WHfZS^A zLgZaRNzC9F5egojPxgFOKH;zb-X@;W_+PzR`YJB1x#WkPyTY|eigmT9HqI#_2Ri|P zMtipfah3Cav*#mt7g_ux9EyNUhnOOOHF;(OfxOYXyL{XYKLaDWRcI}9O1MH`>weR~ zkz<@M?}nJNAwRif10NCGk8gIk{wnD}eB*yrsHaKb0$jkhs0cF+9T zaozA06|dKX2q2gxq6w`V*xynv{rGh0k?AbD1`6URZNeb23~2pFn8NLJOiNW+0Zxr9 z5U20p8=PfydyZ(z%gsL0C7^y7))N4A;$=mS$EIm9f6(J9Za1C-}gO z2W9))OOsZW7BSI(FTi`_{#vM&Ut!O&Sop&6Xea(cGxV}iLJ^I^4h-CrcV#n=)+CCV zI+2POS>LTYT;V!~#2H&4Jn;XEfQ2JCH`tVTFrGsv=Je&PJQLXOW6S62+SZ%Zh~H>{ z<z)WG@Xo7 z_M1l_=RGg+2~)L`8o{A`Nxk0FP3~5J7~u+L)IxBz*{2T^WmPpHIV~wJRKz6E3&%)8 z`nTbJ^-jpu2t57cc~G8r=loVVLu?VXJX{xNr!ZXdUdSzKBT0Rig%wyOE)~DFG2jN!;ycxY&m8U&_=SUaVa$M^c{Nry$#V?#R!{Lxofb4B6G*Ox_8Y4jkz9*<93Tx?0|p09`O z*YqH6EyJY4EA9Y}X&I$TZ})80QmH6|D})-D3@^`dB&oI2h)yX9Psa{XtR2~-zD9q+ z-)nL9n6wWlGNi+7NYH3nVcG8b`2JAL$i!U?@UOgJz8y(RGye+q_?ctLzYmM%cuu~% zzBQ6kSZ$HdD*IR3>+GW7kl_;wd<0)qperAgQKZ(|+*X$qEgjYg+@`x3Dv>Jv4Ty?3 z-g$o=uL&N?jhNrCo6V*bgy~*f+t5*G_F+)G=g_oJMF{XFwjAs>xOREgn~tUifpLT4 z5sMB93BzL1KoI33&dg8Le1=Iy1lpFLO1$~4GX=MH@7{BO4yto15dTTy`YzUULM`A1 z_YNBSA?93#FJB(HzzUu2z#Sh=#(@X;RkQr3wu5pPZ38k3kkOn^aC`*XbRBmEIZmiL z$YpYR<@4}xrRf2Efdj@CK00Vt6~F=0P2(cz#QU)sgYlKpwe>KM!V5gC^`t{3v32tbKZ4BF#;}bkAGU4zvdKe#Y-!|UT;@p&0<{~tGnwUgT$h^%oz~YDX%?w%O`;i!c9qAw__I^Fzs!Huc za31tlGbmC?XnunaKFhtKGgtez$ZC5-tJT9)W?ZqmsgeB23SwY#*S5kS=cS~FJz|2S zR-(Ax3$`x{DR!TWaGz{^PN=GfgL`9U`v{j*sA|yU!actuE*fOi;|_zYH}?=`ZLo?W z!wj+eSR1*)+g%EKSBy#891xuCQA&3U?YcWXX`a4q`JOJT_)k{YW^CqIZWv}s2BdY; zQgk~j5x6Xjp8))9pWzD5bBR!{H^JJqn1M{i^ude!3z6PfIS zk88WJP}TrO+%XY8tZJ?~*25Pjam_Q`k11h<6+4DR@8cneOQ54th-(FFeSySmSK3A6 z1HP5WqR@DUn49xLV0#mNSy}2gcNW?yIfZ~DU_y8_WeN!BcQS0oYuP!ZB{`uMU!!`# zjHo?Amkpzx78wfPq}Zr~!z}zQo+Y5tsm0LL-GF9Rh=}2g9JaHY`SPl3X3J+o5e$1L zG*rB3d%cRzGc?e1?y&^6lss4YUbByl-84A{LbT%R1RIDQR&1My2D)d${|oXrp=2fC z2`WCn6OYP^%4%x25u!JBM6Yfznf_bi>50Yxa(ksDexHaeoB8x*~QY8oIu&$iRuICOfiY1~C~h+9y!n zcPMoU!J7s4p@<>@zWqx+;r{Q#$0`<2iLs*lm0BRzjA>ooxGE;`y{{Oh3}QNfp`jaG z5)VywT#yU;z-N9w9dco*nAP5145XOdfcV?fEM;Xs*fVb|&*A6(5o?@B=Nikqx4lk_ z8cQ9-HXU2Uc+avL6D!V#&bQo%Y6;Le1(3A)6hJIBI44)TH^;t2uB@IQR7b1bb8z_6 z4=jJVW@uY;R98LI_A&;irykd31Wf z%%Mzh_-}Q%@#b7=LzcR7Q|Jyh=-)agl%*afYpJz<659cF0S>#;uP~sX&LB~pc!XbM zvVR**M^4AWovkfE1|s~UiiF+PSH3}r8QJ5m=vHcDUs5W(FgZA61Ebqew@7vMox9=`+T>CX0xHE6BkWj<4vo32$cGS{buRa0BnnE8dP{YL;6GZ;xs*d~~QY-W6*;8)a5dIUzc2 zY=xvf={+r=Yi|cRR*t95(sG0&-3-_{uB$L03L@Jw5L$k^gugfbnr6@w_?6 zYS;eZ3)3cVA&si7qRgHLCM{tY%#jP_;yy*nFXD#(em92fVuTwO8Q76Kb{QgByXBy) zzp-Bv{GelpRMTY!G$gJVsfQh2xCFob3SWJpDux=j^h^B;iD0OtD*jz+%%uVB4itAr zcZ;1aEVG>Gh*Wp&OZNinhy~W+$gm^i|Fos8=fp^Fh+`7s))fN{lBENz-5oc^>@8g_ z4GN~%o;;eL7!3o|DiKLJ8%x$t$IY?p*tL+nULjuA(e%H|(wJE%)N`S`Fht3r3R%(KWt*(7PTogJQ3Wtm_pBS@+TNeD-TpeCK3^$Gm(O$KN0dpZ;?rU*;gjd zcntSO@S47DPi3tTgIzUgo_K=Uay*DZu%`;vPwBl3!ujjc7jsJtczW=J3iuu8GiR4G zf1KnnR^fMWm-bnXDvGHY=Pgap2GdsY2jg;EV^qCxL#4I12J({O)PDArNR$V#Fv9Ks z*$%w-{6Ijl3|LGIy}gzdTaIdVJlzSZ0jv#`KA_CitlGL*0Nwu$FT5KhTuvbMQHIj+hX>%*)nkzRzLtx`?okcB7M+jf8vz(Ip;V3f&Z911qT!r=vLNh&X5enaTD5!Bw?|n3ESWGky;dcW#%*RR|2mk<1J6 zs)G%X3qHATW#9QhTGlrPN|W;Xq_^epP|lX6_{&PwPAoUXMcIYqy>70IaHjW<_|M<+ z?C+?6ELP=)7Itu;cK~yy2j2cT&we7uIAiyX@X{?rYPAt?a(OPaU_uw2vDHX+3>uXg z&`7~!Dn1eUAvkqd#0zw=KJuOvUd+s!(062a-F48x)sR&ml6u{f{0Hdh`BU^MK*#A| zYKq@qW)1k9Us91+m!L5{2$Vt-A72KDd{%gQI9oRdL~QxqNbVo6;KD{%WXH+&_#odq z{37*7Gq1$9GfEgJtdNHyhxCCDyJM5q^s|Vy^{vh_mk`NN>Fq)>~{(N>@CbZHM-~^4}^npKhqoT4R(t~(dEH{;BR(=p}4A?RSqkTH`1Yk z_Szs~9vSA$?5CI615&Pdo8GR2VmH4r?gdq+3-@KqW9#M*t|wYHcY+4geMru6=(SAu zhrq;+X4^?-xBM|uD_O%=PVOL}NH1@1(Mg=xUT+a> zAk`cq=4W~9b!@#sPKgU0g-ESFhLSI{l&EO4PglZ(g6_>Di~K+|G(D&W5iFXR0P|Cs`^5du&zDN}MlR0F_N17bEv3a+)qff)VgKd$IAU@aBRJh#F<=h+ z@OprY6JX6fRZ6h9fCBj)80hYiL!4&FWr@mV$d}$>o47PX$DZj3fVTp^3)uORL+u^! ztRBkzZTP*1$dRt3W9iG$-vhEjXR4GcMxe3}UzkY*36Bjz*EVjyqJ9LGb{C|Ek||$b z?J%>oA;FXGaUw4q45RFQe#nUqIB_o>Op@|r_ zY%#(f4H8dkx0^d6UNgD9iFOAt&MioMs|G&Q6EhVv;0Pm zSkKEI6Rzd6DS=rErfQTo%L}PN9Hs-`^g$gaKKl#f_Cit_qD+eZOW8k`CMCAY)jq$J zIWeTcM>En_kzDITnTvO)4CL}f@>1nY*f&73om!n!zvd>Ze6ZX`&j`@Bp?s|5O1!(L z+t_zXpkc1d7NI(*9u)9N=)v385f;Hlz_i_jZS!SnO$`q$x@}pZjB$+DaMQT8E>EO8boON= zq1=ykvox`xhnx!p_~j`h1w<{Hps2lCwrX0oi=ZsyMXVFTc<9#VNtK-kWj103a?eCV(b3-@AldG$gjpSj%?X)V zBPWg}{G8!rckBgrIqdFZq+H=;q_N4LLxX4f6(LHbK;hz)rAC$Eq3K-EEaOXMK{=#L z<`ALxRv~kAEm(1?Qg02&@ySuN{Ym=`$g_CdEI=2_+U(;RKL_+rbwaXYfzITAL_`Kh zwwu9{>X19I*`ifss!N63o@7@Z-PFIbRiGFwEbvVyX>9XHeIu{l86==EZv2}af@T zb+}Y$<$Q(&#G)uJFlD?g-@V_&>4`e+c5e`gNlA7d6*?3pUf9380F{GF@_CS>G|aXH zgioZ-OlD;HB8nf4+L4(sQ6HIThcRLfN&cKE{wZhy#vJS!pwTejHImG=S3|VbwVQy3 z8vMX>tH~S!J1$n?O23Vbdyn~(l%`N)0`rs{V@z7aS}idgJt0S z$mFR*kJR{=x{8WQDHzkJis9-MbxN@}=af2F!n**>-ikyiSFv}&q+#fi@YtVB1~Hbi z0M=z<7=w7^agp(_WjOH(AwBwO;m>*WC`+*@}Ggj_i_ik*&Ti z{zOX-)DGFwg1)n-X^+>Hxw-nBA9!`X_}q4>0INVq)Z)i$kE#Pq5w8pAL>)t`r2o%h zgtwP5Y13biyz9behj$LP#YlIUt1EG>JKDUwdv*wdb{=s5>}-76t``IDkDLCo@yWg* z=vvQLd+z5Ahq45#%{C|3=S`Q}g9N!U*`}Yqz5RZ8z5!%8?p)5*qp8etQ&X9R$Yx`R zNBDu6Tw=TZms{9^uAg^__<62kcjtJrP)Z*P+G70T+j<@j3QlmlU@DbCA1F^qP|vFJ zvtk-Sfh_>>ykyYZdk|_0pz(XBc{406vc!T$Hg%MSdqL0}D^vp&d_J79EWMDjuF=OW zqlHspRcR0tZuO(1sR-<-ZVh?(JhmK8-J+H5*E4bBQgUf{D#yj|fr?;F9BeTjTp-!< zZsi8vykF%dCBCuDETFS5sgr+{RaFIxG!{{W$z}mc86d39lV#Q}nwGrRG^#C17OD4@ zJ2e|B!OXucG_ox;Y+wfXp(|s28%)rWJviQ3Y?3=wNZGMlN8LczSECt+^sCE1jYQVK#79MNJ{A_*bJf@;{Bk zxy8GGjnr0^FbA=|zO02=pXDDzo-+PVgEplqneAT_t%v&Y=uDNtLdf!etk{;8_(#K` z)v>X|48rvPMuy6dLtj)-(#oj+tKoq8pT>A$`Jv2DnfO1kQz7_c^Cl{GW_{-WSHTn_ zWBfl2ZCV`~n}2J92`Z52qEvZ)pglWwbP(q_H-$)+ZE zY~oA4RP16~HiJLpw@UGlGz851MpC)2Pc;bK{qv4~k5axL1^gB=(gD)RNU{KGQ z0RbbT>xH785e6{|kj>p&3Z^d?pV6*Wr#s-aw598dvST~p8l;?H-bBxaagPrhWp!V{ zy`R_=@JC?*b}@|)^xVYo+hR{-V-KzmKvf=f$v)xprf-GvPFgm=wO>#Fgarj1G=r5J zg5Xf_@QLqhk3fZ+IjTL8yFuT40AdyKtlg++ywSqCe4Ra&YF8s_h{NU0cYQiLq2xwUUYoabT1^^ zp|r<3fPDBxa(7Fby>U-GS3YhuMO&}*L$HyiZgor26CNgT)cZxd@I3uVb^_zbX5-{I zB~)t>N#!p6E02Par&sgIbQbz+vMW^Z`h(S4egnN2n`}oT>g7R@jRFtm7aE-)7K(8F z_3!N3p5AOPl3F?bdeu_{;T+qBw@*kUk)2@kEg0+4nsnwia)BX1gx zckVf$={;QVsIA@M)*eHzxlx{Fa6`PKiKsPe5o{{9f-K9?!C?P}f)70l6vRDN*>Nr4 zq&2W#`uoOGBahb>x#IJF>|%dhV1yRaW&08WIxpay&S>YI>JZ}G4<1d?(^LdN1%NaEtuyWNv0A8g<|?6$G$JPH+f)Hq zl?{u-3OJVuG%BAUHp`Zzo)|jwbBG zLfL3;O#q^>WZ9Ic4AK~4t4Yt5Pxqc>s%#&=7+|fX-lL=05^?r7r=>hr3%n<97bZot zDNzkrjmlrXX_Nx9V<9ybhtoAL4}sM-G_(Z-~zD_tgQcP1}*M=EbSwYaVW)Ny;xX&l6{GrOuNDF6~o- zevaWj*W>1@EnJCvVW_!3p5ZC}SLzHCVl89<{bQcrO)_1ejF`@myXWohv3hpzERB$Z zCi3i=T9T2$Q%v^W5WG*19C-Q8p|yWX?yWNFU%mkBv%6=*tKdd;=N6E{MHY(6Qh~}T zT>UE+_0~I&jps)s99wuX|5^lHx);3b3yL7Wfocx>1_pD@yI}$XELn1spKnm=UcN04 zbh%1tF+>*JtgMTJn0dI^#Mahc)edO7=`T1No0VYPMKBPLlektNo~%q3^zrf!dU^(s zL64=K)$0_);?>;5PloEt89tGi{HmXA4n;-h-6R}Aggwj6KmkvH=w0B{1gEXl6Tyj#3aEFv2GP~Go*N?cCc00bHR!w7`J#f3rUAjlF* zs1kczR1#0|zOQURfbG7r*#H*5s3`JHhZCzablshyY*{3hmn6fb8+9yUVi6|Z=k%Fq zq-a^ZNphK(UfGN<@ZmC=zWj-y14!X5@V_vrd?}F` zzb30|NT>sBr}El9KZf+Xr@lgp*;S0_scJE#vVM8pTB1K`nbZtB8gH259Z|A1C!xz z&e3~WdKCC-@`zKD7xMeyl%+@j_r~^7CEytjBTN_~O3p6r--H&(?Y;RE;&zs!)Yb1V zWN|YCzP+E)$C<8EGbc)(VyRSzgU!vcxxMb7m&AP zTs_Jm}~{GzF@YS(N9UMyBs6P7_Fu(wIy6X03^zHvIMPkFLicfu zEBJguag2Muy`Vkghd<&&+dzRN1lLHiCm}23Md^??yf)lYcZN8$O!PU*oB+kY?rk5= z&f&ab*?_nh^y?Vo6aW?y3Ksw1_Em|X-kkmooiY^Eui@NDbOXie;ovxf!CJ$d{rdC6 zfP~`fZoqU~l7Gm|g5h%(eA-_aCP!2q$*twuBkPXQXXB01V>3@_q7#jx{lP(HLRMCK z?+_oFRQk2{!EA5&h-!bwW$ZrstC^x~aAS($N0p4?8=O2=6O6GTi%JLoSk2F{?6viM!Z@EF@SC59c)bNSjU)|h6i{@yNvQE3B z!FM;pDHs;*wn@G3-c%IdPNz{RC~Jfc;&+5Mdy5NQ#44QNp3m={Y0#-ieJ}QJ&WC6! zuiZ*~s;cM#pC7{F(=F+|rm7%~JzWnJ1sI1)ip!HgT_gfLfBvl@l+sLtgUj-RfQR;@ zV_7X+<+wR{C)Xag(WJ?>T&`X*!jAq+cU>q=(IWC-4DJT`COo##^DEfZKwQ|PW~|Ph zofv$_#xU3rK&*-|7ZKw5@rcL!y()6fsBbbO!s+zACVSh05?sN@>)eXF$&}09bmMPw zEp+$MDT9F?{zKG5KpZijnfYb7UmX?VWd$0@!F#j%SggfU|AB&`olFdYbG*O_3QnkA z=4bP%1(c<&;kQ~!#B^6?kPG};EXtV}QyPL}4C2{U*uaWWBut&bXOXe1^^<2s33Q*x zF}wiZQ`)44_)J!+;|=|FYK0_tZ_KZtfymDvgjFp3ZNxYMAmxs-K4WW8yEwHj;XWS8 z>zbmhAQMbzoF8G>8ZXnU5(>Px6`bs5)FmuT6u9%xx|@jk+&H@i0jWP5M__Sb0rj0B zzMe&sm;wU|<&9i_Gr}eF?01D+5PH^Pg>HOYM({57px7e`5_l zF$#v_Nvx7PP^bagKBF8t0Ln{RX~c+cSp=gAMQ-7uF@-y$=j6plCFsCUzvzL4kmx;> zkJ2lf=x>3R?;v9M#!iuK+32Aqs#y-0yWk2svn;-E5+U|Jt6EkZ{QMst!s{Xta8AT! z?Vs!q-is>5F;f;`(<7m6W5?oUROEx9N=|k(qI1J&SF;Hgdwxe;_JD|Fb@)Wd zNjMna3E5Hj7HKfStn=xD19z&ufejA~YqEtSFXo84sJPS^(Rd@6OQs1Ryp<(+!h>+} z!hLa#=zqfR?p2Mcb$#EK+u=?gU$~FIy@ZPHE{BRnG3>0R-`{P7(nn z2|=WH4~);?3B4kQ@5jT9>H0d(l}pJKZ~L;rKvxMR?@kXnll|D_iZ^1yI(~dXoAzOW z20hjlaI!!F)3HU*dI6`h=K|9>Ez`UNFt#vn)Zr3zTqvg;E2!Aqy&eYR_YWc8;0Q|1 zRs(-1?Ov$;p8MFD(e(VNKE83xw1|_h zC_oDV{MlAzc%)}zWc`MBkA519g6fZeA7B8sBPpkvqD5uCOAl&b`z-$;7s_w^hUM94 z3g-{J$l<+CyTyaV_S=Qa-^}5o zZ(S-ET2jzJ9zptI+8ldsP9aXO0nPAgW7X9-1=**~0O{<9hP66qhk-WQODo|f;l^P} zf*_K%emlBl&)z4tHyg);I0b55han>)pfeyHOUQe=kn9$p>V)Pm$k>oF5Yz#_9B=zN z=0bx@AEpKwCPV8J4GHWo(Rat_3+&08HT`%vdyKWCIs|VJ(jG=)KFO`81&4n!CgiR` zC|e9gqJ_rmupRP<#mEJ@(@Fe_aJk5zx1th?aj20kKqQChK5%C<=1k(%6%yb>`c?Hg z0k{5OG)6_HIi4~z;sZ*KEsh7v46f1ZIfsVYp1LZ_FtAg=IA$)1dsfF(vp3}?KAsDk zI0?^#pB)Hoa}K2&4M62$OvEJyUwx|&0ZAN|;3i#E+A@@LR!f!-{7dyNOI(? z3rwWZ0Pvc#I^jvf*UHJtS3ye$L5wLEjLP!Xrlf>*xZLjEQh)bmL zh)wUx-U)qWa+htzChX;H_@OM?0alRq`cG64y$r;NQjg2}`z(jGNSqBfHOo5~ZKV`M z8cig%A+SK!jWvEhh}r6#F}XfgVc(SL;spP82fk`^La|vZIW6>Kk}z>I#%Uy*Ptl{@v_B zkT&m<`F+ot>dKN{XgqeMa z*cY&WbxbkW|2$!46_{$VeMO3)7vykzqxeh0@0-M5*&8y?!5tBe@*#h}dBQEZ!4B3p z%LuEl;cT?qVs~xu!sQQ>^aKR{mVxG55SxH#l-99V0PXvD72_=?DHW){{b>zbD;3>l zdglqsf32ShJfYqY`RYHWJsa|N!emkKd|JBgK^P(iSJ)epq9VPdtY%@e4Tjf2GKE7E zxD!b;irae2^i02oCGbmJ%)VzV)g-se7;z1+`n_gO1+PjCnUQN)xJU5!`kLAb;PBOw zCyaD{Y7=myyE9jT!0m4ely>?EJ4vAoIIAEO6Jt-a7!DHhYqJ)BLHkQ@CNU(JK*efW zhym8tup_JpK{2hdMynq&7%(L6z-Q0|G_vV40e)&(L|o%7y(Dff1~U;tQ-jQca{7$1 zq7O_bJ(?L8TmK2WII&BWHxVHNWH79oh2~H&7bkKUYSecMPw?hATF=apWTEl(8j_0Wa*9pwvm~Yv4<|V&3(si;Eo{w1Wr)k#=rQUE0 z)VBSZSJ4_n2^)+`7W4^sOwE1W(+Q*G5}A#tj8fllOCnt%%nUR4${SB{TV_E|Tnd_M zBXjQ>yIN7y#d`^cIfxR%6T}g}1tUDjCzy@5dBQ~bifE!gO&PGR`?Kq}Yn<472HkxY zQEIxR|AG} zIL){(9A|5dNqN`)3>Kox@E^-_*@utm5tqUldg|bbi8SCp?lX5)8Buu~59>kTg+eEI za%3)=N;Y8bnJTP70-8a$^pph}S&*aqlLm#%4Fo&+S=C^?d8D#qE5}D2VlVIcKD558 z?vDa@R6oT0>`SWWb9Y<|Q6U9`!QlW|j>H-eQ6De8tP=ghfNoj47RtxP0_udw)Ek_s z%ZEfiPPi6RnFq)R&aV6C=QIgCxc2k?xV!=T=5WvVoTE^DQVHF=f=WDo%dmsZHToR% zqmxTwCZx5}%-+ZT#FCGO7((nW7t265r;(A_BNR(z>XJS6-0NWj@CP>OR z#qB$Jw1-ECqC9CjlSn=cq`Ag5(v>w>18Mi9c`yMMMTHS=Yhrt-aIXVR7&S9Y6OVkk z&eF>nl-1^O@I+0NsKf`wlSV79-4R>$AVt4lE(oH_lCcdQBCxdYFW4&@O66wI;Vrau3}|>F6F>+OrZn(Or2>u#XVAo)Ph~Rxa)Zu8qVY(P6mZA(e?v zQ8+Pa@Ug<@sG(Oj?x52lOtJsN0w@?S8O#IQp0XJwuiz35c32z5i+m|y2brIY3Oriy zCbkkCqa3RtPKffdwHb&rt(6iryZyuca44B59MdB;|2|(R3)ZDokEtcT65h^!yIJj_ z2Z7~FCFUGpg^GQU2t&Qm0fI?tQQI&O2wfCCB>%=;>+txvibwsCRjEp~DT1NDXF0-w zI6%Lx81gX@z&>DX*Du(Q*q_ugv%Hsr&({|M?)cJAnqqiY*qnE6%Jq&-wM3mA(9D{m z|B+}xZ>UTMBJY9L3+S`fh5KUQ4cbH}Bf#3dTe{i5BQTZQlxLDsnn$2ZPKw0?_A;)& zQ$DDDD_+^(BB9;L9v>L$taCqiqDhq2)%5}^;=GTt|4L$kZN}CVrDU7o|JFTj)_pk4 z?D~m^lo_KMhrqDfqrS69)!LWvdD$f!~d9#2|JmEuBcSVXkLWe~#F7_~aA zDbh|B)e(om6_iOJY#cG)wAW|{LW@+baBUYEF?i%^E8IaHS1eki{CQX2nk^c^bc4^F zV5&QuI4XNdf3}+QC5sIy0q-IT!KZBpn-wCznsVR{pUv<~wQA=9qPJ#vI1e6hVbMYm4aSRipmd`()&JHH)!LIXP1YX<5%D-!{yxAnV$r9`qV^2s>ksz89;%d1Dz z8v;!U2sS-ohc@{0NHXzKBN!{BKtGV>TFq!4Y$_GSKeacDITbxhZ(Qglkn46QS;+M( zcbxv)jBR1zRez7~s;|Xbi+;Udih%-_NdIFW+-0|QT+HB)G`C;{;>r4WHnAh!LmE9; z5tr`&!_+&+R}ysH!V}xJt%+^hwr$%uu_l<<$;7s8+qUg5&wcNGzW2{Qr>lO|)m^K1 z^zYsQbmAW_JhiUR!> z1D)GxZp0IP2m}KDmxzK+H@L--m3ol$SJj)Xc0AEWW2C*&$cjh<@vSXhj?C1!F<8mAnfV-srqyeJTyL^1IF+Jb|9*Q7&fOt=ySy0ea$q6X z@gVB!?*|w%bD{k?RN;BlmYk?7E4qb`ErfHU?BIxJ<&LW5bVX1z58Xx9XK;xrHjCyB zY%hZ}<@Q0^!HAF!hwr(e)L&>p=)psoJGC+f$$mE?u?Zv8@qJbK_>)zcP!Y`quL@@6 z|HrOlGsNrq6CwV4e6z)0rg3E|+e0eh64$(>hB>vl4DTFIa1!vKy)cKGP!q949ibzV zQa$^L@)SRSb~|mSnE935TmCRp;h#cn00B?}%ej=xsE5o6^A^&<4i9VNih{Mf1WIh@ zzbzi%WXnEJ4|`F~jNM3)PsHqTSN1+0eU2b6>j-mF6!TBy#9-g2#gn3Jhc)^TbyHtU zL_e%NVBRZ`y@(7g$3w&X#z|Zlp73?(r&r0OpP$uNc>f4DrW1$otAZG{}OGcGH zwmdsR7D|CLd{|x4Uq#TKx15X>i9kf*4*fzW0nYiUsXZ!|oK*HP-`JTrHJRGA5b-B{ zVj+g6Q09u34JeE>svSh5jE=G8z70q5VA!~t&J??Y!Sc61va0#RE-)BZ2X7$tn!#P) z782?({q*~;D6mVP00C!unRYErIeKr_d!BvjZ+fmWQ7m`qn}^#_ZgumeKW2u{ zQ!L4?!#{`zDd+6;7BWvn9vn>1KbKHPhxca)(dBeLygaZ2HD9CuT)WA zlW&U3p<`!UCmWru7NMjMu&;y9+x~Hj<(x4$2T2c%!wAm}!AXxrky>VJaM-P9DOKA2~lw%qRbbKAtbrn4Vo<`@cbD?Tx-yx5Lk7N@enS zH3;F$Ezn7>8x}wbgesYH;++bjK)LC=S<^{zN*l0v5(g;%ty4{!)QRoXfnhuL_x-$$J_0eQvdL zeQ9lIK>aay9pbj{4i!!hZ>8P@$=>3Nh=>ej@9u2lV!o@dS5VX&GwY6pZ|S@(Ih}Vr zuWTlwr?2Wibcd?(7)74&J4D+UcZ!3C4_j;$m_+M!O`$qpvr^%Xxw(BX(I3waYVT92 zjVaw{^Q+J9QD0v8`O>ueaoRB-^C(CglJy}5Z)$Gr9~w-`ikv0%u{-J4?*?ANP-LbSSGlz_xV+!o>i(r`_4`>gK4%921p0h zA({4LUV;;CZiSGYVKF?c!KWzhkBe4RThyj#X?2~GLiG8cuvpp==7ue>KwTYUNX5M< z5^#2xn~Gfc)FIic*B3&Y__#WdK%r8L=B2b`)`J~Xic!K;#`h4K`7&|Q6xFnW$&=}z zBPWmA+foFbM+vyB*(d^(l7=$??q-mOvuFY0#E{_J&ajWz;Mq}d3*YgrMRHASOov7$ z8|ZTn<^w;&(4%=lJ$~2jC?+do>n!W5%mZ%Y<0!ssi17u-_K zqRKrOSaJdpr=n!#R`RWPtT;T0Yx>M7(}FSXj-kLT5i{YVe$|#)@Qvz54|EyUNKs#d1G0N zv+=vqeD!QDMI9oe3W^FYp)piF_X(VZhY~J8`d`F1-VrC2aaWlzdAIAd(zTrbA>ddE z<(3_lmz`TBerjl<(`p1{*uBAQLV)e>mgx7U0WfG_OwL;BK1GzftBykp&m`xnI=6rV{^gXgQwmdhq> z%X{nqb3IQ8V{%c7|zD zN7L8nVSNDs3({=QBTHNkhn;4~H3n6}TqI^y)L`p5)m#V{enb zM^2@=_W9}*&R$+9+KH80@-5G2+X?Ji-vw>fWSRwZHp{?Vf7n*cu7ASGWB%JsK#%~d zF1qjgFLoiNNA=Agb$ml8HrQ22N#h|Jdw3 zNQ6CKZx2_VQqTI|IJ~@gmUT^eS-J# zbD(Ch+xMBK>WdwBR92U_rrL3ijgK?huCb6+>$bZc;txK;qpt7y#?<*Oec8Ahk%R5Z zv2QJmIH5@4i? zT4jFtsY3`=MC5k-gk*lC`F(KBvDu#eFk-tG)@PM0%LTl8qY)$M!0co%%nIK79}b{f zS?;l4L|sN012?#5Dq?sn{NC$!Or6KA_x;n8ovWkCF}x+ei8&aj&&kYt5Db0K%k4g+ z$q3@Z#VX3F_Y-lz02Mi3!Vk0En3!Xq4C0`N+HV$r2V>u4Ky%iBPIey4T2STEE4Cs2MMM=wLL-@$2>2cqUg+iqpGx;`0UW<*o)XuF0xA zZt{Ed3w`=q>8cJ-<%b?@4>vX@Rr?r#RiC{9o~Xfbu&u#%4%CwQsC!j4Z2cO$Xi)+!*HRp0qwh`I=wp~lOuSs}R#zvuy# zUa!ypet+8wQevNyM2E<%y7P$(Who((ci!xvB1IW%?^U}ABXLRy4lga2ilaii8Q%N( zYLFtwCx4}{0e{Rqd+x`Ou-1N?ZxM@2TAVWL?#G_*_HkLiyqE{fF~OFq^QYW=s!jA} zN?t`9dU?xqfz3R5Rvz|Q<3V>}9W0EyO|ZBm)V~%krHJ^EBd&Ho_OEqU*&8Qad!Ll^ zyUvj%D<`+Z#()2y>v&Uf0RR9rr*u(9^hhaYNXRUJR#JL@M^32TaT59};f>hJBFc?e ziHvU|C`kCG*&AO-|Lu2qogdcE>2o}s5~2cF0Mo6Y%pMODbR4;wHGllz1dt|TpF((909TaBHaoh^7$=CD|d ze-AEL*-YSh-Y=YOg{BIFlZR}_Q|O=>lfd=i1RYkU!P)rv3ACH-R3p;0Op?UZOlS;` zsaquDr$pfd8%&9{H8ru6l$6#oe>IX$6KBM)#v`8oUT@0QtPD?3Z@}_KD56{+X`DS? zySR|nr~T|p5pL9|3lE9FE_%H`CQ!96?00irtE%tldDNa#dOo1t9c0Xwy6K6bnqtnFZ+wCfjD(~*j-Ggp6=YUh_sWlf^Ifs&^WGw+SpT8TCC|F?V zSNd5!PTYDZr-|*AQ$G}Gp%z8Uhfno&|%Aq&XqTXn%H}5r#=pLxFUUs-s_H% zSN4*9m06Gb0=IjiucMsbq~2)R*weOD*uFZIabn|2EJ|>XSYP`D-xkim z3&=#GZ0TNWwZ`HZ@*v^wY6bcPG{td+W^!xBXf9KGew{U_9=1g|F1&VwIaRG^lyhXR zUnb!Oz89ZE%OMf^-j|KyNc5VnRLbefC32!FDyUnY=X4j7BSyGax)r}js1_j7(`L>y znzxG(2aMa96J$pWvx?va1p{05W4X~{(shzvVnz!;E{W6cHb$>nvt0-MuUd~$ZQG>n zmNjk8+T6|okqzIwUr3@7kq!Btt&T-AZsvBA0J_W~FS^iAI!CsqdV`-E5nY$i)C>hg zv_X{3-CdrCk{St~6|AkBmMm-F`)P{d^n*!ubLYFG!8m#E;K~huo66(q*U&XrYw+hw@iL|t%hmTkpSOffRmsFJCUJ-)8x!DNKil9=w%iGQ zK0j|~m>HNY@wU9qZHsz8m%Fp=RWoQT5eLJhi+>d&sVhT5Ln~dFh_KZLD=R}D1t_ox zD`lKCT-&Lqk?5N2n7ruT8T=JOC~ z#y>Z6*>tro&&-NktDsM0qJ*{1-@wh71Wjf#F*Vt6sDQ%gJX5MCMhACtb7uYRWP%))N8eLrfRL&QNEeUO+l zSld57c=cg^k|3vG*RoZPPU;X}Ot&$Uu9T!(vCN{~64ovh__0Vntbzd&Q3#n9fe|hOR z4x&My!azbQG$_Sv&li{@9GIFDaWe-iCy+?=__b?GGkgVy#r3gAE#)*2u(n~L?SKD0 zD-eHh;$@(K31@~k7%NKBa{cw1k*vVtUmzYg1joJ+0EI$M$<`_)Mpk&a8qQS?!pf5I zlu&1mkssOfR_bCd?!4-TQn?AxY(JFoP6!i#K`bq$6gP39-OAx5KvJB=)n!ZitV2P9 z%ZTSZH8{K}o*V*^?>|i9z5MP zm(YZ&-%l4?f_J9sdu9q*ezD@AyU%QY>w$}Boj0wmIKtc&(RdwCFR$JxH=VioMpB7_Zx%zE}zEr zDK_UQ9i5q7MO)ohH+YXUa5_#T9Nx$RFdN2T2BvF(4Lrh{Z=jW> zYYOXG1q3k3YE(ZW$7o8!V^BGpZVW9ZZIPNI@O|3r_)W%m?8mU*t{BHrm0#;PHq`&0 zK)7cV5mB8-U5^v){rRFVQvX~%^voi{zqnTfXQmQo+2Ay&HO*~N^<;_eRVHa6+g=oO zsv|^O5tEJzF3=eY=!a-J{};iz@d$0+o|w}cgrVyOctAe^{eNT;e}j7nENS^oZVY^0 zj>;w08+B*T*pFL__ae??_WFQ3?zb_JcFtpdqFXuFDA&JIiA#5@M36s0 zmVpD|kd8xwk#m#HtJ52@l}Yb9k&-3Df98!t8!=vWD&Mx6nx;=$o;T!U!=`HP*hx_n zeXEa<&C;xj5V52IM~$xc(tRV)uz$@CyFPT!aux@1CL&!8jRkZK5~x-iMIEF(m9E!0 zZnK0o*#F~pc-=(O3&&JYTJ1jpJs6~p2s<|OmH#j66#4<>=k36t4jrpj9#ns$NhJ9- zVofT#!iy~RoZ*t|t}x4uock&6YaTl?3+K7{y#_kqzqkJDd*wxbDMW2EDl1Vy>|M!jv)FUD;V&_XfzW3`X!jK=zaM7cA zceN7q{n?4h_?qTRE(k(G5+yB4{T3NFV%jg;s zkY=E!iblluxo^{bJGi_BS{dd{>__)OO?BV$gwQomQBpSRdsC&kIpOtJW-+~< z{tzHBodi;fD)^BBIcPyfkayLxbyZc#)f)Srm6gnP`+v_%DSMah{|J;chf@Q|1ECS) zC;!SX|NdT7{|>&>vNP-QBE#>xXP8-?@c0f!h_m{w8ZRmf`hO<%$MKl|8`pZdYA{Kj z55Vbsl)2x^>t6r-?vgso6FSnWeWo>~1qY-FMF!1wJK_6Xx!h-28Cpzb&Faw@@}DKI z6%Y{5IQ!RjwGK>FRFuv`De50N=oGY3@xEEPn0tW}=Eeg{HJ_se?zBg~$A#dd9k?CG zrVBQOTq~fQi(Zs1tp&GL5|BIJE@yR;LW^*>u}koM^fCSmLX>Mbf6!9OmrZ z`K%EbmmHop!H(3@H#?*7B!+U!hg{o5tNSQKbxrd8SE3w_eq2o^Jw04;a8dq8?ZR9} zHLs@M1T>?SXifzTuQ85bbXqZ8`@1sjJZbv(2>X>)d*LUSCIb00=@ej%Q@FtU?j0Ah ztgLLM)wv`w>1T^J_;rChvUICyp;JXc(>~-XdNu)ggHXq9t$eA$8D6Ayp#gJN1FCDr$Qz>m z`WGu()ark)84MvAgvIKqt)M5-FMWIq80F4NF;Y3U6FkkkEZtS(ksE{q$yuJimXW2W zMyM7a-NoxrhbpOJbhNIbrHfuBH1mY2x&Kv@8cZFv(ni4G2ZLg#=|VJXcO_tJ##iT_ zlDjm)*5qhgD8rSKvt=nfX_&H_f{Th@6&4dp-lPj$kQF1}+tfd@krWJU-vNoZC?PtB934VNxr0bDVFeUcM`K@Dz=1T7I9RFvbYS(2S`JB2ahrgYI!( z#5UDsr9(Xpp|B=voccHsU!7;m*=hT_qQ9p}SU}L#x9#J(0{;GJQdmtbB{BT&>W2 zKW@y9#N4$w7Og8!=U($^ok=51sR;WSr|Jc6S?5!S>z(tw_A_$<0{>5u06s$fhPtEz zmOKwO8ZRs9_w<(QFAY_l%j?JIrspl3cIoTR0!YXefeOajffUSOZ0QTKrBVPR(Ww2h zMhzh}FmWm2+`5vRrS{6y)8_K{_vE@RC}@c3xM}+Au($X42J?2h_fZD3xf#zQ+RE45 zvc{_-(ccN`5Eq2llb(lmQpJ`oBM4ap|>~cP;P~pSC>)(3M3z&rz){+TFofiydB68E}ChtS0<$8t%i@5Sblu+MZsawea1ARk4@?|-lG``a4-C@nH)%3 zjae`sjU$ER=fCZRXO|HSr^8|zknm6Ts>PNjAP`aZ5xhrTEob|{#Xl|C_Z5-^{@< zT?Q99s>RcQ3$2iP)nZwrFpQ-cpN&|Dj0PrPX4z$0qS}6+2F+OCVLo^nqN%tY@4ima z*Od!5qNfG*AsO)KjeA8;AuiX^X zWJ~1`kkz##veQZvAf2>=WLok77pBkK3RJr5>I)9!p9A;s%u9^$#gmbk*3j^yz~#WD zRghk*Glg+-c5W4XT{TYCGUdNwUyY1vU!~Il{&?lijqr+@{H2$#Hyvd#?j%#+>Z|O` z-Te1Jw!!(?`A_`zqcQg~G?IQyq|VM$GiD?*REHPHZ7D%4f95LCsv17Jt^?XE>%<=4YvjIk6Ca zhl6=|e=Uy`{e;blx;t6aI|vvc81)G6)uBCiQb(SJQGrdi?^xdVY)wth4;d_IzzAIsJ}P}?S^gA&KO$Ng4qq2zg&Ic1jQ?gN_UbMxJ00N8!X2Mw zdRnZNcI{mXoV-JqMbCAY0`OtbI!)WvlKZeTVk8k_dTVgNi8*H?$2zE49DZV5Wtf{$ z6mEslr@jQ3Sq``0&)Z1ZUTxdSZ^rB~l-TBW8+zA{8u+df!nX-RenwyZP z+^W#?=adHI{eG#{4SVbt^>ii$NF$#P9d3K}wYWOiD$Fx^S1hhP9Lv}2v}Rc-+`?_5&eF@_ z|3TP;kn63WB4KoUqMgWGKoGIuSXTdJ=RcT2N{kO$5+qodKg$@01) z4t#Lq&%`&;;yqwMh+HW6a9>{|OfJ2bVlo&CVppehC)}z+-kzJBZJM&s7WnZ{?c+0{ zlydMPW-Q%AM3MbnbkBB*Us#}vjpEt8RlW3iAN<5H8*#lhx)8jj8zTr(18O|rCW_FR zj1i3E{Jgve>=AVL(Cb065&d5>cpOwz7>9x|ogdxai4cIrABGRQ<%fHK$Sr(hAjb{V zqi%_Yv7?ft$IYEM7K-jfZ+0kV{D&qX;YodZEpM1#ljeVSFTrY{YgidJD?;W_V!G-? zG&mAWxO6hrz{<&^3PF4Ub((DnY1QF?Ngw4cp#%jiDBh^n;)XL@P1fLbX})gw*sEWG z*dVc3RYdf`DULR2fG&K8Sa9={v%(-3_Hz~KHF}*zUX0tX7|8e=GJ(S#;_MLw#J31w zo_5*T3N3IUV8bUHe&2Cnf|HW)F{g=C2EGs6LL2SRW+3{GYRI&5L8IBz^$asy1~$o~ zhw0+NRi=cryK=Ou!oT{W0zci+t$6*40Kh%tBg8}mwKbu(zDLEiRndISSh(Iiq8LlT zA27V*vr3XkOnC*figE%f13X0k41Dvxa@Q6NgaxE2FsQu+NW7Fm*Vy7{6M~WB4WmQA zVh4l}PBKV~JC6lI(-xVJBY{Sb9?-E=_PNc5;2i=`0>1_;*0&Fx)5_U@#}{w)Cu~~j zNl@@K-M&EIp{h|8g8qUX3<@<9O}nUP1$Z#sO3KQL`uOnJ=As)K8cN8^Kcib->($NH zGb*NTjnV4)HR+3z;SQO&{s?qQcuf@Xk*md!>k3?k&viPkaF&3yS$-#x!e3Mq82KVy z;<|5ua5TKez4mk?05L-%xz`CHIA#SQN96^Gm@LL>;cB@2Dgsk$Y2|X{*v1~dV$xt0 zj5ttG510k)buQL01~KsPjo|#Ee*(GZ*i;w{AtK(wX!lid5W)dYPPGU6btIJj+c69K{MZk$lrc!h`e+Vun5KJmI)6Im~_1 zP6}7+$=T|R$1HX84cm-TnNjx-)oO?48!`az^xiR}4QRImfkvMey*kppIe*7gf`B`^ zsjN5d09kP1^u{HIWNPD&muRlcu-nY4I0-c55es%t!)i>~hwU01fRIxi=r#3@@7})h zrSkDEAu%@Kc-u;cv=!w`>C!`RwE+&2oKRUPUbhnR&E5z0XvKUt|P za_2=t3)LV%;bXYjQdbCUg=f^T569k=50TKV$7PS_@YtIZ`Zaegs%i(5Hz$CXrN_eP zT3(_rq+52PCg_xM)_`AkKjHJO6b9E`OPI(>hGSaBpT0nHoSnp!G;dC?>;)Rp(wgIY zK1+e-$NDRq5~Xd| z8Y<$0!_M6ep|0=CTGe`l9p+k^5FeilD=4oDohN=~4Z>S!T}nXlh%Rm}^sDnPgy$77 zausDF>lBVKS{7b{O*e;p1*c+(4VS>bfEU5Z%giKDxd!Ln3ijKM>-*^wIuiBHedwh? z)rb7Kng;+3Ytf+~n86Yj@Ie=}s?dge0qmmij})oA zY1VdJ3ST04sb{P;rE@eykHsx0t#Q9#Z~#tQl&tC7mukrt3ClX+sRXhRHY!NYcAs`p z$&O$z?E6Ilbnr~VOLD6v86btoyk}l*Cft9eq$2YR7#PdGsS%bH4jxn{M)dlRUEW&Dn&|vs z)V+ce)=Mr3Yn)I3y%pw?NQYuOIOGevoI8pqtYl8e2vRL?g1@&$aF>p8jg#5e9cr`Z zoI*xGKwx5OikG5g1CyGW84Uyij?3q!_K8RuWpcJ@3OhnkW(JB(9s^!g{1pm8m+fVG z0L_M)m{!Z15ii!+^dK&>`OhRnjb@EA@%q>7LJH8i#6|tSCl5x8HC44CK6jI&8^gKS zPrAJ$*42P`U1+O1gsX>`hVYim^(;7n&6`q)yfyFXUfAXvlBI)2!DuhTdb1V56UQo7 z>N*_%GkPm*{Ypi6dR40_i&Ph4VZ_UUP3(7#UwmFh)Avzg^LaNoHlbnr0Ah3u zZ_G7Y&9Y#So1}f)=b*D?=^9PzU~YE~KQcW&{5&YVXCGr&>8;^KQQL`5L$8ufm{U2t zAriTgV# z?1g09-)`(FuBhdVHN89cW1k14)e|H`;n0&8LP*y|)tYxvVqPLKqCvG@OiVz-R>I3< z#jQs723*wdF)7IhAxQa<7fP}+6eTS6a#7y!k5E%*R@K~Dl;DKulFPXP#*yF*w}J(! zrsQTAFjq|piq?PYU5E@_-EhIxuO#$^CGUlRCn?&vezXLR{%l>IwX#MV1OHNBa(UoH z$p-Avfy|DnoDgaGclwJ=2|2!};L48F<%x@7B04W+k$t+v&VVfxzmHXeX;itQCN`+8 zjpW<7L8D!Z_~G`E3MNvkU{oOlm7SK}2Pad7ZBlh#<X*Igde6><#74 zL9LCPP>O9lfEE%Fv4Z8s*~vNw3(@LELVgA$K7wPTasKW-FEY`QYWRTwh;YI_rGV+_ zZ-Tpdaf2o9dBg?=gmU~Zq$40wTf^}9By*9oYKXjNNd6X^LQ^gZDK^xCuHMj=3&^e= zn~T>GHZrSOW&M%IVYc1^oc`QLhZ70hOE~MA`)&2yzH97#qKZw7w& z+<=rI%{R8&Fb|)jL}<9#!jDY{Z{8lz9ghai{^kbvC|1G!(`8^?usV*E{Nw4?%ykJN+3o>}?c zeOX#e(EhsxdVMJk$!`_1aM00~(`Z?35xR9cwD9&i&T{^J=g;JJfy*4G%@mdBUXLZ_ z>@x=Th}^H?{f68<1EPKbpPhb>-pZn0OkP6{t-f6@!61DPP&jerN20_vCl$5Dd6+4z z)ousrl1zB~NEil*UYz!3BXLWf3LJzUmJcPF^Gzv5F44#wei2&{mz;7&`%{~y8gK`F zTl+hC{79Or$-Nt;@xYwz*LkEh_}LvfDR8}w@m06);X^oyCKa28W2cs}RQv=+`Fg{v z-j#grK$*k}Ae)Kr=FF}5#h#J|*QAmfkg>2=Z?>ba2JdS6J&{su1C;x(O_Ny6v=w_! ztdPXP>qTCzT1!hB9R;`rFa#lK6GAjNL^J?{*btJ}xSfsZW(I?(A79|Ch4V%fYwV=* zP7)hDc$s&|kbP*(@-yG?Auq0UGWwPREABmNKt?{;bvZP;Ok)V&9LbxgpOZNWoJM<^ zTcqeBfYi(gUyisY?wcEn;^YZRZ0`V+^x4pW-@LgMXX7_%CuudOZ^2Kw820kfBs9G>$r&fqViwWBsu40>K+$bQ0r+)L$z1mL2H# zyxV?AFKVeRjSzXhYGYV&f{E5sDHDL#&s)=&et3*Mr#Jr}$Act%5Th3@eFhtPLZksd zXn0d$8l0CAQ}_YSiKU6}5vUUwDS=GOEgNcgBk;Cw?XRgfmkpL@CsJ2VJDsym2CZ%IuaGDGH2gu;L0 z|ApeIPwCf%-A)t5dF~-?0$G?J$?MVo=fMEAh98T*;R?p@OBrerKPXN`vjS)7aMBR~BL4HLfeC;; z(>~z{DwOw-1v#~C4`}O7ekAJ|Fs|U1k{ly43!=rlHS!qfBh6F9e3&sEX&anqVdj4U ztJyttboW80*IJ4K`XrTnQ04j{VLVBYj(2q?!R5jZgdFl>Oxqa5WJ%bNL#g|Af$~Zs zZ?}zQzLW6GlvMzO6JA4;j+c9*G%CoPP)2d#^V;OXqqZC0H*3g~P7Mr!`2vg}9cGS> zVzvgD0Uy^u5Zb)xBQi#^o{kKlU5VF+DMsZ==ot^} z&3u&)(N!n64Aj0Zem~Iyl$m1>sSvj)I3QZ-XJHPNDtsVhC8gF;$7xM z+4{|>a}mNti!Gd;8LDUcSQDwyM~UJNX)qBZnD^#;FZD^hB;&!0DG$t_*f9Myu-!LLjIF+O2qQT}UN>u(pN?j&<~2P#Vd1%L zOKPmXG!NB3w6ab(kM1>nFHH_cG?VXMEh5ysuv~%l;1F~CyCL>A%;>IZ2JgK0g60+J zQScya4eCR^o7}-5iwSUz0kVGt530ymhz&I3uu$?o=`3OrTL$sRtC5#P zfEE@gj{P{27DLlLV$RWN}lT>i#Bz*z=)o~0o4j_qNSr_TGYlf>t%1fOOnVIML zvN4@ahLDK}RE33(zwg{Puj?GNWf2BJdQYOPjkpv2q|vpr$p`>XhI116-7UJmHrM0T zE!w|kMKHi_izDnSs_oa1h4MVJ1^V-ir&?Vjn0DaLDJW#aNo?Nu;1Mp%%^|gaKU)p z7g@QqKO)}H{9F9U7rHx+VvcHKsVWt*Jm++O#dy!2sVYa80(7@^e<{ihP05TLDISs% zIuK>G$~Y$E<$cAt5)5fr`~9VI_~3apum|wq#iN^3T7|1Qx4-Fr@Jj!-d58q5t77-6 z0SFKgASV22E&)mk94Rb-%sf~o0!{jVe?SHT^t49)VCM0^Bhywg;Gg6FIbQ)6I5CY% zj!cR0za3=q0av0-$bHLsh9@G%56DQa#n=6()BoGXL5_qFu*zBKr|I$ECK8~@Ap#`~ zsYFkF4}LAg9c%4mKFibUXm=651S&?B7T50Md(}FLdFM&-QFji!>_o-x-xy@GmSc0U zIERINgHqp80Zn98b81YRfdMble5?OK+I`s6kR`t$} z?l9_C80RWkV@4!+FtW^y&i#hFOR_q?+g3+U4@P-7qKM3DT1E#o+J?tL9(Hz&a|?A% zw!`^OR;}-MqIC|8{NbBG&YgqpL>1aZ<(*tDHx)7M`rNOZx<(8loD*iuR@T;?&`y9X zRMhweEjnx*oX!jinAMO>sd(9r^`I9P7M32nX7BDnM)z>rT1T(3-{~@X8zDAo<$t2+ zxlxtXwN1~XpT$`~%&JzlMBa;&MfnHuA61}e*$|pP6^Fdtyc(rn)!Se2*IH*hcptCE z7yHnRrYx8Pz?lyXJ1?M}mQDvMS_(0>`8H!Zt!ap&Tb&XiV$fV2aG459`NEB;Lfc@> z)3J|!D$+NfIGmP~ub>I}gj~Ts#2>1yC>HYt6t(j|dp)A|nZG)fDQGDK?(`CoHg3cf z;gAY?9Zl!NRcbf)S+G!t89?4CD=7v2`$v7T*^a@5_uQ@^A&Mwck)IFzldx{N8>8ve z<1HE~UOrdU(1-|zKrs4YaYqu?S5^b!ZI`QbN5;nPkgt~Z@oux`e|2=Dzc^IiIMDRJ zMS`!64oaIn8+$L?Iv82nDN==hThtw_}JIh}2aiT&^!Kiez9NfAYA*KC8{VOPvxc8kcv zU+X9ZM&)AIH1aHYeO*{<~v{pBq(<{~+1sX{aOKdFCtzSlyh5UYfP z4EwI$RDD0dFuu2pV;!AV10)z27$RvUNfI?9k|9=?#NVM$r_qUIflN`2ut}@%%XyuU z@G-hRE}?HC!QKlFbKChqfxr7;txzPOB} zaPsG%-h*)=8UZ9Di(qftwKFHAq3Z?{Y9YyiiC+U4PV&!81h7(3q6L~S?|%|Le0**; z>1#8JKsuQdbB!;O9OP+;UNK=coCy&QNum1ASMU;VM1D%X8{h2o>QyHvj*;>xnVOpF zttKDBg}a})Kb|d~%!hO1(9oh`U6R^XZzde_tKYNDh(UzB5SFQP2pJ3QI8g>5C}8~vOwGg!@&+a6N1@MLjE;)g z2QQrzfY4hsJjjrk+1WANq2-9gI+H~vPLwfjO@nSYFr$oi#i>Snb}hnJR5A&OQk4Y< zWF9&$qZyQ?coe-)WIT{Xa1#fO6mc2rS8cXD_=qxYE~u!WR+AC$-FRe@>9x%}`1@#f z>)J7o&lOM^QSKn?azPQcZqKGx{bY8|%m`EG&u-52erC_|ylv&+YFCV7-MS=M|G(h+ zgM+`2esEC@7B2(SwXAlKXFifDrVceD+}gl@bErx&8R^UWq)-g$|M(*FZM+5rGCIzEQLOTi=Btp(aQ9WqWx&V-F=#w+79x9lhdYK!C@DPpX#F9;d@jr$8u|n z+OZB|#QEo11)jxwxNXGSzom=6O~Cg}j@$TyC@Aw_Qre_Y)z$lTH4#eAK~r59ca;}c z%k-924>_is|Hx0MXqB>Zb-A!z(?3kYUXCP|=FMT}&X(>t-nWdwHTRFXtAI34fi^0P zC5-NeNN0W*rM`@Kuia_$q#Aic6}4=S%2TM>{E-3)R)cF^2Wyx9~*_2c!RJ7DrYK-T= zZQ$0jv)c)t1+N87e0ASK$xJsiddESQejxgviyS80VJbLNz}qNx2!Uv@Pb!$R%FScE znX#MA#AOFqH-N9|*TMb${R;~V zdrn=9&X+19A|eEQd_Ej&Nci&ccCeajdP7ES%Nm6m606BSSuTj?2Bh0_U^H?zLw+%Vp%5^#Zb5Azo~FTHyj_?B3f|Qj{w*;3MCZ zzlD<^;=1%!Qto|5rUiIwAV9OHGp+w-HJb3*a2(MEeyX(F$P~eaP!Wd~_P^6Nl@Ek7 zTVoMc(So!6sbPBm^@$Qlso1Z5-nSdI{v9q>9^N}2a@p#wcmhB4(m-ji`+hn*kV0GA z;6V`i7uNxRQqjVdgIE|6PCQU5WbDeggpOXYQY>a8gX|>=)-NwgJZL>#xMejERka7x zX-;uR+jWwa5J4H!t{G;Wt6*&bdjYqisuAX;OUXnMv@>iy4u51QZOKZD?f+96L`W$B z$f_%MxFBcPgzB3oIsm6J)LUVpR9MD|Nl4Z_f3ZKk=^VjY81I6bOUa!~fhLYUFwO$IiG zF-w!N)4e{Bwf0zBKcY6IY6Y!ZBBq&SN9x zhEmv?7%T9c5n_^^5ykgO0uih6F7BZ!e`cfN?JHPFZ>t9<5PU(}gzH78$@kyIwH+-e*kLlV^J$|GX&un}RQ#4PN&IfYpOBLjqSam>p=oadS!mY)~Y;txLbZj2)UkDJCMHvp`xNlH#eIh3;4qs8(`)39+bo8fh6y0JO;5{Uh9MefawCC& z$n1J};HMW+F7LQGYn>lEd7rOACZBeCgtLpN+Y;_-!D)pjwdD8>w`6*^I@L6U1 zo=FT7YC&r33$Fv;cgC;xDe#kz+qOII*yfJavF&t>j%}l3+qT}^XPoCe_uuf2^=Xf>YOOVPRn3}J z^Ea>QK{owEc4+-^GeYV5Tqtlwo1N#u`vFB?4=+G~_e$rfx&;xDf4AFd${;AXNtj&U zA%QL^7CZSZXo=sBOubXxKNYfCL|DPEl+OlFd)!}oLJ8Fgz3JVSOZ|-l13teeT5=%R zyu|P+5vt{3&uBu$z~ufA{Vw;95iUDmE*mTM42}*rA95vnHU^L9>eBBMXVdSKPgq!3 zQC&TBa8NvK7ZMR)^ec(uD`IteUz}C5)VqIF*LUGu;C$-#WK+T*Y+|zPKoAfBorfno zx>SihZ=aN6ysIKp<0!;rqmCHG=e6^OK~dnQKyV2awa6nLn_fXmAi&=QsNm=6kEcP< zAT!%;?~)%Xo%wF_HiV&h=Na#YBx6R%VLzi%4Bz7~!7w1k@!GpU8rs~--uUF&svcI5 zO|MA{q|12d2wK0b9x8rj$Sa@32!K9jd`i#{t7VICH?2TmTQv~zmB$hu>hcfJ_`njH z#KqOU$H#^r3Bz{fn4R*1O8fO&B(BTCxff~WQ5cZQ>g?k>9XW*W?1Ov-CAI9B|9Vy( zAV3Zh*$K{1G=*GPC6a*IdY4-3^!n(2)#%akalXH#T_O(AL1cxXQ715SquE*_g)m{p zbLCPC@1V7NJ-(o^H517&u%@IX$-G?SunyDyr{jCUk2h=v19tqG{ne|#E9XmxUy1i} zHx;=HOSJ-S@Y$ZP0#6e-23xDzY|6W+@xM!kZZV4~LK%GKUH2+KQT()@_#Cm}ha+G^ zHq^pA2;vjj2DdM8e)jI}{;KgiUwkrjgHD&+OI#j1LbAs|Zw$Q98z990~!Qo{7eO%D(~9u>0;P*7r`JOqaXjYr(1Ja9u8wAa!@qb0zTH-bMUya zCS-8fh#O5DvjrM!qD0?wUiCC<+RsTmy%BgR9wLaW2_`VFklS%aHV+%8&W0z%b`I2O z+!(eWd5?P=lM+)u7_Yy2De5hTwfoTyHRrGrzi(KkdMY3;Y1o?8c!AK`RIZvHv?87S zmB_I`z3uK6td^jqr$xw+wxp&*> z;i;Y+IO6ZtEG^r~M+Kn-=-?0H zdvp_hZ9T8xS0rb9w&i?~qVhzLJU%czH`YowuaHLlritx`!G^I}(y}o<|7t^TQ1v)} z9p}WCF0y3=fTMS#FM>2j6A7)twvQ4`v^!?U@(i@g-!ezbBM$XMKdnOA?fNwe;cl=n zk<$RQu6s5@s~TG-ep_MOxE>nktVf#)2zb^BGq%_%z(qd<&8_*TH`Ka+yf|QTR|=?d z0o$1%sOz|dj#iY-!DnXEZm%w5H;1gBu}LAZ^O^vJRNN3k4TT3JW1|?GrT1|*41GR1 zRo&kam@FEYj!@CR-RHqSwGW3!^%U4IJ{KtE3-~Xl_yuI%XwP%2hHXcVOlHb<1^cfA zk2}n5k%VsN|JX{n2j)UQz81A{U-ItDV6h{RcveYxNVqqyYN90Bp^)p}d-aQlFaj;@ zZ9niCE|!NFBaJ3Bo0v9T|H$FR%sBZ*9$g9AL6SXed9Gc7;Si4t3byg)!l%UzUr<{G zUW#oaHID7$gbV61xt>)9Q(S#h_iIL94oH)P;`d;=o4djW)?t2E?r@|@Ub}}c7Z%$Xr_pW z9SPk<@GS?Az0%8y<0SI*NK1t%v*|Za8Va(?!;rwo%qi9DS^bov!KUo8$?YuQo=e zzbgD1#2CV*|3azZqjp7gAW{V zbP&l%gyK%D)0uQKsm1^Xjr{<%mdl`|Qs~Lv@hxH6$}qtnaRO2IKNyw4dr0%RWyl^9 z6a6Qpu$B0QOlKGDRIItsKB|;)GQxxFQ8s-?FypsoT09a7abTnoaBy)wja}4aZxG@8 z6nq27IaQf3SxMpfrEBtyYLN~GGmg02PQ2s!?UC|zVqJQg+wzd3;hGSbn);y}i7`~= z=f7&hpe`e(=x0Nygh93QN#*0nPDn{^xal|#V$pvX4-+&yvsWD~6M{!T0e{BmUdKHoYx4>ltr(Wl2w z5CXJX%9!$x^#d62ir+MoCJlabyT``WePutvb%2sT12MmrUZQhRdLzni7Q~1+6eYFLF6H2C&!% zw*az{=g=ZEDDC4I@_5LxHrSOUkjx!$g9Q>@Js4Z@ogkRYBxtW3zA>v9zXwX%AhzFR zv7KW6EOOxW$`9h(P+JSiUMd!fhsK|A&lDHg9MQPG_8Xtnu$HDb?finFczesACXtqy zGIK%{YuCbV-NUA7+Ae$K@ADK9;C1pP?=(p58^MkbySffx@UkQkR^N%7fMq(ov{q`O zfP}Z9E_jb-tR;s{X^<_6@CP&F+T4j8fW6)r6a?I1g?X8qGD#~Hs|a;>nj<$oM-3G1 z9+f>JoM(>eWjRH67{jWglWl7>!-;5!r(!D zp>(!9H#NtRV5Ma)A~un%x^768YPK_{NuDobW||zVR<)s_@`%; zzrXb&J#G(--e2xFe)+uI3F+zK!N9^^4`3J&va+ImRe2%^3<~n|y}gdHw@@0A`hk~# ziBnp;XgCGQwba%TU+)nOH4!7WmF;*LTwODd1i;FoTsMG~DZl}tBFf~WUSeHEsRb2M z1sHpUn7p=eY2HE7yp=~N+899+O#?~nNzL!dm)Xy|S7IY`vDjfq7-9ndwqXka6N}~2 zsK*P#n2|WQqaIZYj;Gb<=fL(+7gR9@ott_t>TCOR&EsoDT9y9M&5*e-Jy zueR+MD@CO)_}!hwIc2&;jvvtNDfGjiqHGSVF0U|thl^I$g}VeHgh3g(VQMt}#zty) z)TI2p=*G~e^Mr-V-0;c$7LsX;^~H%YVspIU<<4QXSh08^v2w$!^Q||%+b=A|r_R?y zLX`9~zr+lNkA<;n(nTFx`_Ewlb$+*CF~B2ASY9AhuBk}NHtL_(pq1n-y`&e@Pe?rpVgmQv4D@zJTz)-j{f~+`z_;25tXX(0Iy^8#6_F3tMDA$Qx|t z_+YiBE*7;`)Km?HVey)Jfiew&s9hYhoWlz{KgQ5JcC$cEXM%R?dSdn!C5d0@9%fj>8YalaFl~`OhdPUA->1~kQ#@CLe$=mf?qbtN+WGajfn{<;#{*1A( zDuW+AkVuwhH40*cm!>&QH!h9xg7d<>lp#t&cCuX7X^hoXYRh%5h{6S0@w%k;QxUsTm#K zR9ryKAO0&H&U30HqbA;S^X)?YY<&iBN(76eYyjv7%t{F1 z&04Q^AJ!?=#C9Y4kIT|WUk;v{8 z5Tl|YpV8rMP+>T${)nUdCw^nG%IX{riwTi8IZ!lXlg=VV`rv-T)Eyqtiabx^e_E*a z1%~aYwcK{)|2xa#BqO>O zQNoFw@Z3`oX-B1hB`Q<4p9~cNuQWsrHRebab+K_LJ+e`K{$QbZ&}eNME`cGQp$`}L z2keRC7ZCIdOi-GAc8H>{&YEQv48enmH{a@%-{!p-8YBpk(AI(w@<$>|;ex)Jbfuc> z6yIaqeEApq>91qgf&Tpb%sPU9>4;VTW5T%Js}h9JRUJVBmI%9QBiCH=<37o>SKVQ@b{Z3;+CxRBr_eg zQx5|X=1D&2nXYn7?ihv1BmwPG_dYUprIBP>96%t!^Kb)n?>BW2jnlUmII}Zms*Yaw zG+~N5CQaOA5sRfyydW-;Eu75ZW&bFQiZ_D4T{nd3`4_!QJQqNNaZ11|t=O|$tkD=h zozSGZy=CWW6AE#Nz}diF{}H}`fmKeN+s*Dy zZv@K+&(|I?av>Tpw}WK$$ugq#7Tv^3mqqV-AC%*gQGS#=fXdlF6a@JNv;ihq?ji zhz}Xy@~0WE7Ffy6r9^>T$ovu|=Dqm*`TKXb_Pg)uqIzLT%@f_Wi<{A>YuZn3GS@3@ zYY2N@U_aX~+*)}KA>EDhy;w!hTm z%7GmXay5tF%kIP~u7DNG3T_{H0O~%isvivY`cS}L1^8i3FzA4{YLTg|w0ci0vyb5Y z)LXBaS}k!IpQOuOfJ?Tp-C}xS$M}?DT(qK=kYj#OJ~6c85xd)!p?btulZj90YIPEg z>3dIb#C1wNaKd!9qNDiqplqXgLndWI{j>E{x^S*dEu|Ua^1}`X!hjEBt}ga4{Lwzo zo?Ft>0e*mg1PFV!t(XMNl7j@|M>achl5t}p3qedD$DFL;db4^kl@CVuG=63TF3nJd zdAJd{cGW=rz4F{U1TXk}2aD$>gi#CHMdtP+QRllNb1X;5CuPMl`FRFxF{~Xv+a7YM zATXAKtNp2EEiEI{M^oKC`*P^e2_JUK2EjrxSH{E`i2&((Anav4x;H`zbqd7ac-r4x za}5RZo|%#`0>V{<<4N_s5~QkD0wQMNMBD2jGCo_sL9VM&mcb9yU;v5vhXXP<-!XE$ zwm%{7pZffjG@RZz&NMvb6ho|w1 zigw-9S~aC>pUTE;{h5Ap?_hny|Bg3;%-x!W6nWLN$o$~M%SUQ8S)3X&o_5z_^8unB ziESiuDRx$4jeIaPuz%vrNTwSc+yIM&J4opC(864rj8jQt_CgqP`S6KH&K0tl2k^4#geC#vH?F1%;2pho9o4DJr8(?n)MSJdbq6NnK-f?QJ81ro5YvwD`y?n>wu`)WJGA z@A6b|*%Dqecxq(irViaduCCYn4yu>QW;1xPK+~R;7#7b#oCS;P=Hj?OAZnBa%;uqa zaA>l?U(Z4Apcns9p87TyGQxpcZznJ0qDALygnVUZ=>S?!->vtk5Ii~0yXM)un8Gwf zyrkPOb$1-$*YYP4mOYvPh5jtkzlAm!V2L$;?mxPSI8y&YWFpHP=*)fu`Op*mil_NH z1H$UL!+0|Sg6V9Dau+2Nkx31TuQZsa6_MP43F^JY85Be0?xYbG**ZnybM18@ z*(QkbL_^`qslOvl!+1kO1K%OHz*l*7LknV_SHc$eE)%joI{hUKRe+xUu+^jtlf{h@ zLIJvfB_DrqL+gkDC$vx%Om^n3j^A+bg#p14ent_lrJ4i6AZcWIL^oKM69cn;&3clt zUjf!UGjoZmv|x1l!5dv_oR>aqrN&ne_Z$5G-wJp zS{~S?n0+-iM3PnBBkZ2cSPLj*R&}SAWaH~c3ydkioA zP<7$I&;iHz;$1rMwJk3@U$xs-Ld5H%Bd@f+N?&x#Ev_MS3{7G(561s_XRqs=^DqF$fv|ms)jDkJs>!9MHvn=XZi_US<3~rY+IB34ahORf0r4~AS472;`r>ph z%NdgTMnU)EiM@;B4kDY${cpcJD%^6XZVFBSe?;WdB4PWb^V&dcg0f=lt^Fg-A(d2N zU)>R^`1aqDCoXSMDSa_Y+ugG>9As⋘Yuf$^{Ge2-ppGuNvlTn{; zOs+|%tOZ$I`lvStV?aA^yV$KUD6~kTvL6c;bUkOK6(e)I)X^`VMW~^^^*xBpt8EKY zoq-3sa}Lya_wW#wlKTb+(u*B#WRWqV>!V|mc+QMp%1bdz1MGej$+=^blnJRGuM(L? z+dG}QpfYwSyPNc&6A&Hy-*mjL6aKV3q!E}TJl$}oYyXhF)pnTFexG}%b8{I@F@!-2 zfL;G}bu|~Bu7S)^Uvzx6Vz`lw>Xj;&nLUpiL>3ks89X?VMoTV!F(+7u1g3cLubrF-%8=8ru4_Bg?jxp+L{cxcTV7L;u;gdS_&|+DBd7_HDARFfo zuX^NN6uYS|QAv9m9Ne=x&fle#FUggYjR8zH8EcZlX%w2o)UV)e@BVUm!rY(gplWoN z;>Hc^+gxdLH!#mK>P6Mx7(61OgGQe2T*HiF;EpI84rPrDu+Xvv0?&I;47@b!Uf+xFgrBmyuCHuIzf zVtJd)v<(J~fyl{gUjI)f0uh0j(k!>B&EZ+%$Ot&v`I_(W2G*)fRz7<$FTyB^Gxgq$ zVi&z{Vq60$l>Tz-Oso0MJF*!{OXN##;XL)PCw6xY4fPo3syZromVmffGpFnKd5{EbbOlT674^Z zWNwNzy^D()Oz@#sFaH!5{}Qis|I#IC(Th-0sykn8M&#n+a^H)WXrHF(yZFZ4DZt}; z)_t+w_Kz%)i-0X>orEbVpoaW&OH46Hflqk0nklB0lT5jb%NPYxB&}&F(YueXmQU59zQ_oT=?>r zTC`=upLGSpZw=a2#$jo>M)&t+j2S~j)YK8>yh#fotGU+5$W-?Mz0f-}7ZjyaK#7FjxkB^THaXl)42&-}Y zeAf~fTa*`{r6xlnsgAQVtLNiI^wN^19h+i=4QR>Ll|39L<2MHfhX|s2P*NftK|%k; zgQw?bm!ngs-69U7&(8OM~z@gcH2%}@{gd3^YpX z9}wUK@dXG?VyTSWguJhNMR9Zs|z5it%GFiT;A=k-L?eD7pZ4XlP#dxO8vN@wX z^Zo^;&3xq5*GZnL;-k5KU!B_hZC3O^op@{j))kt8624xD8|#$eS{f&s{5t>p0h4q9 zZXeWJCs=(*?DFy%Lojd9pl!QvXY%M#;&i&~G`iGunFpS%n(J54Fk=2n@F_(I3odds zGp9uy&qR!z-$F?q3tV)RbkxA=Wnc}Waj>HytT+gABL??2`J#==zlmCFE4lXRYDBk+ zXD_}IlhPj9>*JT;v5)3+xKG=cVn-vO%YyVkUC-VA@EntNGZ`z^C6(m~7MH>Fmh;4= z59MnI`k46z}7KBbwIxOu;ANUBusf`mY)R&$p&$Pmf?v z?kVt%1Am68l(u`l40;HdUAG6!KLW^jMva={TCj{6w_Clgug-rV7pe4Uco3o zcR*J5VJ+Q!V#m*H2Ak<{@&2srxEx?t_rJ(te_%dP+>AfC)3#3Jc6;)pP|pQ1yX8}x6pg-bk*8uybq@XT_fSAIO|9b`h-T@+N|N8e`C~_TD8I&T zS=T36?D{lfo4((-$If=#dwD^f6rdQvJI3c*>9-)icfxG1Cr{Gvoj}NJcgWWh;%Prg z2;q0oxM?RSh(ZbBry0|P{{f^?K=w&;l}E1O_pOinQKVy+!)q;H(*`G=oIf1h$v1#0 zpN^;rF5_0h*a)^QP%>-8-=YX&aYFBopLQ?p`j#v@Men_gI-h*o`F3FRsRwy+kT&nY zHuN5dV5D*FlkEl$b1H?-)EIV(UKu1PozgZmqBCLf16HMq2Li;|fNYJ}cn#U&-XDYB zdyopfe!hNWFOa3V*hoRGS=RQOkOW5SYP(FK>5Bl-t#+N9p^nA=+Yd{|{pvwB0vqwh z+s9PLbC0K?3Y|FTYdibnRLty@f2`F%>w999|Ii!)$e=X+78h{_=?}bI3k4;R2lTQy z2cz%3;8dt10{A4MKMb{!WTu(J9w}Jq5rlB_?~46d$J>{JLN2@fd9v4|1AC2C6VLks zAkf9<^cFOg{M)q&?k<-5WJde>T3d`Tlec5<@47u$$hcIPe!~BKU&=BFTIm?OeuC%8 zQ}f9ETdtIo6A9x)Ia-dhz2(BsA1^5dZ|BSZ)SN>Q$;roeDl&#ba_hLeNj! z^mH9mg${`^AG-F4eH1AZZ0A25<;^**-Gqsl{_r0IfdqT)s~g(AsetS~@}UNMk6!G< zOb^_a;;%`X9V?y{EcmdrgL=W-j|G9C3ki>*RpFE(tn;$J_5tdha_i#U7 zcMYyybUu6f+}1-lFiWfF*s9I89hVI65ir}5eNLrkQT})ro74Eazod*d0btbpagQr}| z3L1rA5kVY@wGRX1^V-x<3F3?}jIPJ%UbZ7Ekk(djxp# zr4EoV+vhbyT0k^RE+$T|o-+Kp!Eb$XZ=-v+Mi#fN=BQh!Z?~qSdiGXxdUI$ldN%R9 zkRziZf=S`~6K~Trg5#Nfa0A5VREn=5Q#eF+_G7m~jCt_A&WXmK$G;`@f!gk{uDJ}^ z6uyGUwJMU54v_ROomhd%_RL7iPxp4avjX7z3*&p-FRFO#GY*y7uljN6dx(DhYazf3 z_m6N8qre?M8HRK_ZP!a$s`aJ72K6$QRL5*zt8j(IZ+B+zl>S2y`Qd~oe3a$rF+XkD zxKYE%fPX=byVe2bVMZ@nVouCPpQ=y8li4m`ZM{8>uU~IXadRpsxh?81iSVlxi>7$4 z6jS@aANEbIT9;+rF)z;)(a|w-8zxHPhdOt!?1VY*Pm0~?wM?{J;Fb~bAFPanJ;_q_ zk{~EEr+I?@n5tJpL>ZBsO3lIYk^iKJNQ)VM>8`h&lvmh)1;BQ{R2WsuQcUnu+#@Rg)cbg zFWB3Gi$oPsG0GyG_!@Fb!$H9eUPcMT2U8m6mR8(O<$H2&W)_q)HxsH8e74Vnmge}6r)MY6 zT;&6Ndn0v?>Fe^-`&BCe*OB@rk%;+>nkJ#@+IXtK{@j?nH0P~K*B0kj0syfM+8R2eP@?JmxkSx zh$T`A8xGt1D>_OjU=@umcmGw_Z7G)Mh4W6~s}wy8V^Z8aD}<~3zHaSLwwJ*~Vsg2m z`(-p?3uq(`jc$#xvlg2NwIhW7%Jxb$0{SOxmr$Evd+Fb+Wl^4l*x))UY%uKfqO-YL z)2{#zw+cmJJT-@SQ)?fxh$dRmo!qfJj%S*|i!iMjIkkb(wO10`NsH7#8r0nKH=Fc5rQK*nVKOL@grYwbZ^ zOmuCS7_l`yCQAN>!T|bbq9z2r_B0v4mS+hL$<9u4fISA3&#a9o>B9jE)o3ZCadLHI z6B7C>2VvbNE+&{s$Ct1vh+|{&CHo>MZ(5n&cT@C3) z*7$n-GnZ3F2^%<(fH^u3?C8j!4|JC-wWwf3w2*q{Dak;}xG&7F1EHtp}j)o4Q(0FIHAhw3TXUsHs6`y(r)%$vT?TWEnm$JEa(jEdu>_Z!Ru5(%L;P z*WEVU#-1;@4mZ4nHy$#OSOOXvGzylg;|csE^)HyHMtVnSZQ1GBaZ?`1U(Sno2ocS0 zuSj;rM0HaHNN2#Iv_xrx04Ydg0;0FbETcE4Z4`cjtw6Z^7-`#*ipdAg`;2$5TspV0 z1E6gNTTk^Xo&e3<1^6V{b#|nuFpkg|GNGEtDD~@!_tYED-;cjgAf7tZ9d-fRE-*)3x2}O+4*{^PAIIk|ked_) zuN0sibP5i7cGdS&9DK+{%CDa?PUc=A5Y)EcDpDlMP4r99e>T4&<RfQ2~yR*ij)hCN34plNU6X+BjCjPdoI2f zW#Tk4D&Fy*mid1&UJ7N~rOK}M^LFA>7CqJxG;!$(lKJY6Zo77}oZrIih!QGJ`>JzD zzhkKCbw%VmI=?gcq8>tV^-l}$ZSP6Qf33xN#u?KV-XXFzYX~@FuUjwOCPm4MSemS$ zn)>uBVXKc_!bOk!M2!UC?UMwPcsF@nR|O0L-}6a<8Rzq(#U3@X>uL7(-G?+u`0l#- ze6I;?=IX5VnmBB`L&1sk&TM zzvVnXCQ@FV{_$t9@0gtc#g9r}Y1iQm_EhOkAty8N_Cb&^Ndf4uO&P4r1D^M z?P8Ud(~XaDeVRhcCdgTqc{V0>09vvc3*%%o*6#nnf@+2!25pkqihzdF~bE%3>$s zs1@-|@Ox;?^#8jP1NL#e|YGSQ)ggqtYP!+zLs$1CZkLwiCIfGFyIA_{;!MTj3_7`f~7 z2#E+DR|-1lpqcq=8p!Rlq6)Fp;XV zCebhUAcDB1feMO%5u|5Z{G=8?D<{x4bP&?~R($@u9dNx2$m9V*Y;3Yz#qF~T4-RQ0 zjYF$uJ-q?|kii3)!m`RlC*RIG6a*dKGE&1wgVY<&-Z*Z)cY)N)2<=o$^e+M9*!HV zz-kTe(AUnjV580`>GfNm zRAr%@49_DJt`EgdYf_@M&Dcm>m(tuH^5>Y?q`^<)NGO*%Gf>-2$quMwU7KWAVSWyTFfNjjRE4z&Kb4AYDo$)f>r=Cq1$p-7U{>U0j=5t@1D+Ed35MG|Tn-9j@lv{EI8+JUG>^ z|EPA@sNF?>XBN*E@$$751hLu1nB)|8Z$L!NLHO&l#K>1RPxxfSRE^*BE+vhP=$i)% zQpR|EtOsOy$|n~1DF+BhPXj*~QQu{M&7Vc$U~)qR1r!%f2ZWETUZ5rFIj7RS{|j^Jzq52btsJ2@6A@25aV9h$9?s= z&L{%$7*#niZYJGw&6 z&H|aO_(QWl8Fx_gmnXx?zvzDS`pc*xQ1JIxBwx*9&2;H>ykx&Av5F{LIFAF z63h7-HG1KhMTTx%SCCM>CM2-AMvL`(gZahfLZtvC{hoM0C6D6Ax`3h@M&zB$#$b)g zj~=ZTBCCj7BfHJemG&gTlXXm&Yd=OpaEq}|I61cSgtolPZ*fc0pbIsoEb6>sw^LD; zoRGVGND zqvD=O%ZA|LYVoSvIycvQ5|OpN$q_?8+mK_kGs6{Ah{#`7hx^=DoEmW5HTg^`8y!pZCL5dGXfaG{s13GM}YqJ<}`Y^^ESQ! zujWgU;b08Li@-Ldll7Pzy0N9NFl#_$0@`^Lpz}%Z?ENRm*}I+aBv~~HSsqw4&-dK2 z=q&F(?D?}LsP&pIqki=2T)#xDiMEYhE(FFx?gg3ofv`;z@zsZ+TJt#s%pu;8YmD_s zC>Q!s4!VGe9nR@up@coAbK!-$)tdKp9HmT+Zwi74y{IHo5$!eV)uIaza*9QpG20s6 zQ9WrrN5g)2jcQq>WSvcmGG(pw1v<)>;u70q!v?iz>3NUF^J)sj>@^6Dr<)QP>m$C! z6$ka#ZPs0aGj0k6VmuhjDV>4iXOazTv`{%gIZ zFFB%NYke{4I&3-XO+gbKiaV@DqSwsBxV{fYr`hA_TY8r+nfO#L0QSt|UDfUJcf}8W zb*M0IjYqbKWd0EHg)&Un5x;mYV*GQD%Yx3zP|qlCMNG=2=AfX+6EnYo4;Z$WtamE$ zOjr8(MuNP2LJWjvsN?PK3J2&1dQSiwo2}vyf{-9mm8J2PYvo?b?Kvu_M4`#+U2v!F zc#MK7?J>!Y@tT?i)f=WO3ktM|f{>;urbHEdP1A}fpXm-kOs}mNL0)Jq8jG*YyP4W|C@B?ffPi-s-MiB(J4qNAX9XgLKQ;YXs)$xhZ{wqrO zvj>Fvrq)ulV9CVr?`=Lji^fV>kOdM`{VuyeLrr1Y-su?XU(i3);OMy@wcGv9pE)VW z68;M83}~hs4px=DoVDJd$5R%Yks4*J1bDFctJM1|LGzjyL#7afpNHR=PASTOK92Gm zveFrGC_u>XxXj$$eP2+4(;mPlD|@|xX%$ilFx;V~W`gwZJn8;aSeh?*5A!@ZTM9p! z*z7~)fqnl9aVtaiA=w1iH!4S9nMRXaa8u&FPRBiRt=No6W_p9~K*)(mlCG0XdTM5@ zvJT3@P!Xy0UxmBz`#%UogCUV#zK#~}%B_c(- zU=O95ULnyFb^Sd%t*m~BAEE9GQ$hsp*#UA=BLrm(6%@9p{PmCIM)G&3I(_s%O2@aP z2HFbFUy?`>1G-m*a9wGG-J6eiiWjY!sl%*Hr}z)Gm^s6 z?O|Wc<;69cANWr$TYDx4vb7CJ`y2IK|f`k$Hz`r17_~$ZG8w~ks&vY=7 z=0qo!aS(Y(A|z(LZo#ER+_8KJM@j=i3d6PKeS`f=o$xFs-q9xD=|!Wuu-<{PLYknb znd|+o`V_~qNy*6u_psQi+}RKY}bf`PlAGU}5_$*l~B zu^7D%3ol4GKF;{$BDHHTYx<8CKxqK4JO>!1RBBjyQ7(jV2pOJ}aBIjin{QY#;y9s7BFyn+qeE(leBr<*9)}mIlATPU8AW^0Z}+mB38iMV zO_!rfv5bJjJ7@g48l|~WJ9w@-%rhyo5xbLMUG2g0=hsv;6b50<-c#APbx8(<4s}e0 z-sCw%O|g_Cl7?h5IEoq6!Y1=f?0W9)gp_228DGglz6<#!9QC|fv7m1EL3jsVW9zbj zb@7~>m=8m0Z)nFRL=7j96>8Y6=v3jI0a!e>nx+hQ=v74MY;H!p_`V+?ulNJSDZj<4 ze4rBKJF%36h<}tfHwsM)Vq|^|sO(X~pR6Wns)=O?2sA+uRKJ9=C!opz95upte&nIL ze*j#+$P?UI)M>a;AqJB@j$<_-;YCd!xDppbLN6~x9fa8KMh;1uaugXmlFHgb?Q4o0 zd;X>mrLv(myblsmQX^bEFl*X6!U^RGq&F!h47{zP3<1*@>(&QM+VSHS;ekHMDA z6jkhCCx)DXgN6E=JZ(g!I!=2E+vGwrnnxh&&{Vz$LRP$t8vc`^?sgejf7dl*wX zE#!NNVq%z6A=6mq(6Z$vmymRI^Z!+XEB*lQ8t2B7lXX zp!hd}ycj9iwFr{D@}2`HW{XY0g0)zM8dz1pcNVbSpe2V#uJ}}sV`(~KE|u$b@dOMg z87+ObE|CCAU(zOuO6b5;wVPH#!l1>7U7vOh*>9;5H)P}(5nf#e2ygKhlf{57w!u<( zRFA8buLD(0*(u+7I}cPfF|w=D`)LYB$&DD{_AH#g4)=Sw5EUb zwa;9SM+P;AVv~nZY!lRTCL*_Xmqdim?PqMvs0l(O(O(;Da?m$1G4+SPqzY zQLO^y2Q{dBA&LFLa9||bmmpR~jgBq85l;$2(ZS=(e`j2%y~d#sB0w4P9Rbl;byfBC z!nt9;2Mu}I1k zv!yrWG3#Ps#D5X&Y%|Rw5o$*3ozD;aX~-R=TS8!?<0K_B4bS#33{m=)Y-e9@ug1Z4bDwWF(k&^fkB|l_%uE)F&92~kdFzCk+!{K zAZcYyXfU>vpe8JN4vP-ciZE@9VMq2S`D~BI21c;PKh|QQ##T$k(TtZ4)PABkHi0S) z?cc4E5PISWNcK+jw8W=JTdcGN7W}=psGts9pO{S8fXCzEv1Q~*X`Qgxr6pxECkhdp zMyH}qoSu(ywUmOYCV~zi!ev@LomNkW6zf+607Lrp+wjPL!3uFb8uyF25l$5iRr?4)CxZ=7^&p5Fg|PSyU-<*Bi*MlIA_Rr9$R^8xd3UPMqBz%dPARnxE%F>WwO z8ByL|Va;#@!rhd%Y4Fjy>ZhUjG(UO%2!r_Y0n+3k5^5$b49dV-aM(H3(p0q|>4E4Z*r#M|i(}x`uu@!Vhl4A(>I`-K!>vWnnu8^I0;&n9cr>2H=4gB0 zaVSGPs0Jk_hLx|2C4K>0S+cesFy8)%k7^Cu-JJ0^@lldUDB9X#Tx6h|;7p&VZ5C*oZAyuYYi`_BL_-oLzmhUY?)oN)7{amx&4 z4&@|()S|u$JXSnZvZ0K)Z_?;C6gNx7Cd)=VI{SFh+HoC+yQhWHwnl|A*$>#7{8Y}? z6%wS_x=&mYAWT!~T_$I4{aKu#!JsDiCdFX_MUvvnR4zDWj0||6~l)q$VodBPle4asuV0D65%c9(!6Q1Pr zfTs%fut>wLh+S42JFqr7j3mavO^#w}xzp*#80p-U3=LjyNqYruJRafL4&nt*)LP`9 zc=&nk%|E_geR&^XpnWv3CjWfWJHSH8EM3)QJdIC5PPaDaWb=WS=2VM$i3zo_b0)}A zeaV%=tDXPMg3^_xj)>`Lb#)VCheUHc<>nSTP&hCQ!xVY(OmY~g{Z5!pqk6H1lz753 z)n(u$Dk}lRI3E=EwdUz&#?n*W6Ip!8$AP-P!PrcT7}sJ59x6Oma_4?q2|e}bLD-rj zK;=t>tEoknOtasUQxKDAB=$S$E_HBbx7Oicl=~*NcN2{2m?g=ZR_r6-WG)A1bn|*) z+oB}{Km}^;5B>aQ2|X4E;)YCMkaRb2=thNW5p6w6G_gYE#*oB1enRbHX$;~NgrMg6 zqm|zep0Sr!^aA4fR)H5#42pt$&}5K%sbIP%<06bh%Pfe&!Ge`Wb0078zqS866Wy;DoI7 zP6cSH%=wcer~@etU4`?})4VW0tb_;#ETI_kh>soBVNl|+Mmr_yJQ|_6F=yz~+0L9q z*^u4~&bloJEG`2t#Xq%T1JceoJW6361bh9z@w=H=-!1uNoQ*kOb2d)bdIQGaT&$sA zvXXD7YQ?|a@jI^tA(du6#}CnUwe(64EmyIJ%5NW$!?$9;Pk*VW?osReQmZk@mv`o+ zHG^Hvv7r{`AXpt~Sv%bKCo$)UM``lzZ(Pz>994o|1+l=u0(B!k6v~aer`sz35&g$b zm*PauZ+_0LJr&n7;*KgamOGQD3)k$AT9~juH1aVhG^jI499?PTU9u9M=}V4k^o0z( zdtjqclB`L0567)4j_z@s6#JimwQwJxWXgn?)kyy3LN?j0cyyAOG10f<wu=~5g6@ZMVit~xnN&@bXW*rzto46c+eL|#Fv?b ztF`E(PWKyd3<0@_OL36qh)%7a!6;$-CSYGLLE&>_4B>yR+AUDB-JlK=4-=jpNUvNG z^@ZGC&&$_oZqF&m+eF2;h~%)RMNnkqFtd0sT@}Yji`WY|dmwZs_akS^4n!0FdL}xj z8kVm?EejL0WJhoo?P8r5C~cU-;c)B=b2{SI_JV6@agg_#NsnRW`}&b!h{GIB^s;AM}Ue*+lA7&yD9Qh|feYMUw3} z?(4;9|vv3baQ&s5FX|z)Bok zFI~KyK49p$hXgD~;D%Wi2Ir0;ii=9)v7S-}*e|!g0O5U6gt4Sm=8U)aEXX6Ex{N&^ z0+s^z5dUctwfDBZh@A!yvBL!~4~ye4ljk8ohB1O5nZ%APxwU?w{X~4N~TuC>>e1 z@lzn>w41awXS}0ex5M5rxCUE#F);O@RA+vJChzOA`}vWyIoJd5(D*t6w#kQ)WX1RE z0!_50o84@AQ8|tc$&rf6Yu!P>hU}9ZF!zcv~EWZ$8RT$O#Txsu~8mEmq;nu}Q$vDmMt+;zbw&DB2dwOvP$gYI{ z9qKq?2?QT-d?*}lK5)Qc`AsCPv6B~cqO6;V`Z}hUe86eIgs57&rPI)HW)3-*Pm7B` zs4mfW;j9EnX7M#Av!47*E9O=#lF-p9w!I8#J2^kErYWz%Z=V`DPSqtkU ziF7q7qKyG~ky_ObDUgEKFA+5QS%nq*_x1fOitr1eF%UiL3{T zzP+fNZaqNPd>_Dv$$x{GDdi7&&3VJnzcKw06eLheg=!YL z?xwEY4DJ?T`|(NU`4S+sU=qbcu708o>=vpAh)((S%ajg-J9p*rI!cRuv;;2JKMLw@ zbZ{S6ok?T!Kucn1@hNtIfLj>^H4KDuFP-2@^0bqwq0VyGvl8Ef4IfXPAjlI)1@lPr zKH^xL36)dCU*uBEicB@UMjf~KCTc@eXUfp;QJM1jML+%gipKW*#>d_{nUYV9DWgxl zm?#^xZXMz`M^WshLsE&8UR4pnwTc_VLB>n!sz7yM+~T>}qOapYM<|aE-Y|XR-zEXC z%Z@pn^CExZV|9$v>Xy@nV-$LKnQ2-N*|x;6RGJ;|Hp8ZuwKezE$GpnJ3r(w04{Ho9 z5@U35mmd%*@sq|$-i&vv71oKEYHhEP@y%Wv`f;9FXe3!k{nExSU< zBvFjs8wf_$jl(b^Z4P@i%Ji3;S)Y}t4D+IRGzT!j~jgh9Vn2APcYQ3Lh=*h3>TE^PE5W<#0;NqVppMtoT$0 z$`$dWjuE==QH7kCqvfD8CG;u*&Uq8*Ndjn4j#L** z+I1{Xq;8wm&c%}g?2r7zhk2n@){HYBVxgyy5pnnC-apBdq(*f#u|4WSr|}Yo2TqE2 zi%iJjC146~hy^xmqKdcF}QX5-T#p_KF`HTVpsHqdX(c84zWZ4@-a|FfxgEmR- zZjvazGZ99QPtY8^Gez7ltKpL=$*cjeSPy1E#a6sD61#+aK6wmh%%tD2>Qb+!@}K2q zc^euvs;Lp-w$dPj>S#ss6xLaj@d4>43U`?C4giiwUTY?<9G5%wTmGAp*N~uCWVEI< zlIlBR;mivwC)$1l;*`iAnnc!}8}pgplokHLa=lM4r2SbE?X;+ksb5fz{i5v4o}zr0 znxlYOv8sXKgqQX4EFQQ4v52{_y&%gQ1#Dg_1P+S#J(j%iEYy?cozX>~odL#_48wg& zIPAFLY-Ewy@D+|3q_*kawrn8$K=czkBwr&?m(&3erJ{EHZ?vO1T(R&mGjnkhO%~{2 zKUjVv9nYrp~wQAWk%K`ws>P_T*8kvx%w*Sr#fpB}iydhDqc9>tDWfbxY>J%hb_wx5x zF>s~ovt_3AacjTMnBa}Z*2UWPYNGikqXQ%5$&FM25_Lc+6#84C^l)N_xV|TX*!_?f znC8$9MVKQxSr6(Dqojrc8R2<}6`wD9*h#-*{)v$vMr&bbsS^-^CqIb;x$3$uWY*N` zB<1gzSAc~pq2Tq>{|w&DCYNCoMd?>$>xk{QhhP~@JK-}LH+M$cM$8(Ia5m#|A0^9y z3#g{C&{lsX`6J;5ulIaLZo-BJil~md{c}G>n<#F-Rvsi3UBaS|!c2Kf%<{wEy(L`nn z29XV%vC&A!5&q*wZpcluz6EmN;?27}>YK-`#HJo((J{HVm8LyZyB_j(7jMjrTtl4w z`GVp#S)xN;Y>>c(j<^Wfl*CT}Uf+uW!_E1+autjOe&K05lA@_x<;rI+PS`i4Dy2c> z-b?YJYf8`GJ-My~32$Y|jGwvY|K^mr7UxH#0kWuU)NkT+7|c=(aK{XqK3E?*%8Eb3L9@p@o#E9_*-rKCa8x``M9o19O)u&_<+0L>Lg=9l(UH8x;p^ z7sYorDtl)LpD1$6k6#^v`BgxnB_2l1$rskvj4oFv9da8!hW(6_#>`E6@9-AOD8Zgn%S7!j3j)zy= z3r00gmqd`N79gLQCz2^zmg2w;-*lrZOJwv9<*kCzH8lYnvJE9JANy|SRbIkP?nP=R z4&E^di;Bx%?5>H`d3cpAPQGnP0Sh#d*-L7f{;;{H(8x-4z?0^HY=8m4L6gOGX2erN zg)(+o>MKm1k=Dtl%1;K6av|>R2XDwLZP^7a55{65!`|sVQ#oPM^okyXJZ(f$>eV|g z?uWED^c_xkz9Mmu7o=)T4wFRRNW)k2@(W8RiYJ`8&R@;A)CSY&KwGXrMUE zZuMI6NoS1lk$%zPBMNQvJ@H+f6VD-X50jfM<0-ONjh)#!p?T@`mSWe@0QxSokaX|J zoI7)@5?%mlg-7K~+l{a?x|rTTH1=u0{d5c8N$%}9sQ29&C$>8;-{4Vwi;SvIw0|~Q z2TN0xUe3{}^c`rtu;`zgHny+mVYKw@SU6dZ_1uUN{g{~`1@L`)JXcAZU}(J(lYVqP zL2~x~7tx9G02Q%gxOxFyScTQJL$JW>2E#9%gQP`3o@wIv=P%0o*8|m`VXIgBHy(T+ zxb2I8pC<}Z!0%G^3MnykoNJ{ zwToy^Xvm2FtDGSI(}xjGb^M=C{6F0PA!unq7|WG;pN9IfO-5YN`97)sT+rn9eXZ2= z3a59;SGDKDs-HE@B)dx`$7L8ml0qLA7Zlg;`0o;*TI2cE5!?||9TTItfSi&f@#fUS zPJ#6cBlh>39H5Ds&c!KZw34 zXkw*wOT#7uRV5oWxAs@*dHguX4&qeL(bkpTWr+OgcjgQ4(*E?xA-~4ztNN!+es^Gv zInGtP7HcyaZy4%HHleP)b7Um2u*+UeTCr#iZCHx=+u*V9Zwa2ER7Vk+u^?sl#KcwR zZ_2cCDUN(KFEyhKI-_Sj>?X&E^K$xRvz`=q?7A}stJsuF#mY9}gKlEV19Oczo}C(0Trq*;gB8+HYZ7 z9ai27(9pvawM8I%=6SQlC7jTM8(Cdfj%(WxUCM9>US@0yxG?m8m3HCxd5(hY%%!_W zS<;_$ZydoSaN)&-iE3pw*aA##sYbtj@^v&co@w(RN8PEPgPAn9$c7G({Sodxi$dYP z8kUcPV{el^<=TeXHvFMr)8ZxGM6{h7c1$@i$H}Lx)`+}9obdcPv`LZ-2hD|C?{HkP zz2EB9V-wrLFf|XkH|~Pg5beKmUai+q=RjPrhH=GZuGgJRJ#>8D12gw>8&csP6yT@= ziyPBUTJkICFIB_j#zG`Q!g6^Hv?0~Av0Zhr$6{&Asf!%8kd3hEDoSHaU|2f_@ULwr zYFdK(k`Qit4qrOE3s%JA%nxe0osh18-)Gh8_)a$Ga;K1;XKUTvc^4$8|5 zYA^7aEZO=nff+dfHjiyE)Mg7?Z)5ECMaJocN~O)x;lux$m+Z%Ec7%$Rg;Tx4cS9Pr z##Vfyo-#wPD+TOEVG_GZZ%g6czR|H)0je+9dE;YibBDc5qP>4YC5+Uo&N>fI7DoVm z9YIF83C+(f=w+cA)gQn4XmxX(H$yycK8`Yr|K)<=`)?e6P=ha%@e#-z>}=p}_1u7O zc^>0LA;In&W__8kusJzy#)tc&WT7N>eScv?RaC0E1hQPig;1N{VtMgohp7_A zE8)3IEN_u^=zokLcvudTHW2xIebPTDSupbme(y%PmcZQQE`$7>#pE3^@{#_>69(*H z1!gF882=XfQkLn-+otRchj3aDbl9q2QabP?A0eGMU`oGS&CGH_j!A^Uex5rA!sCBG zzJ6`s|F%Gtey_GWAV4;zIR_B}Qu=5+y7Azp%LXKbZS+z#ya`Dg0lx*YKZy!`wL4e-qiwArhFt zSXuaby(X1X|0kx?|EEvSeh2$s9|gq;2F3=-PbDzz{NI##r2q7h;N$&7CHK}y_8y&- z#BBz;&7Ki>tp4{4f_I7jz4HAtJ)8danh+F=vp0a~D!_x~{3u^B;JKWpFX$g&|AXv# zabZeJDKrHhCLzha$iEg_j9Bf*JZ^U`fnx<3^v;*ODfWrhsIHyJ?WS$e%UxO~M^CES z_=FuD(YL{FYZ`6HnXAEkM}hxmnP1Q-FVk)2Z$$ly0pYudU~tr(nWd|S;Uo3EQM2tF zB)nGm5fNu7DZEgrqYq-Q?bjw&xzG}L_bAL?{Q80WFzO5e;P9A^lA&A(I&6&n>Gxx# zTe=|>v0Q`sECiLviARR%3~LRiu1iz}(cYm1+Fh1pw0bqx>&-}dKM=7wuHp2Zbuqz^ zaly{wNSK#04@xVqq`WQIN+k25tLp_3Q7tBKzur9J3I(9ERdoB;gYzeQg?oL_84GyE z>pfIIw$c0OtN#BVa}5(5x-@bv?V_WO6KqIYgp#Yx_v2<8t$pu`>kgHb80Tu6XuqCI zKk+0Zv;W|1IA@h;$mqKlrFHa68m86wlP_@S@wqp_#SCtNO!szuh-)ul)z##hdWZ{t zrypz6)=$Zk(`<*}-54hri-hFmmB~-`` zxYLJ3hq!7mY`VI@i0o4wa0UA?nkwb1H+sW-edJ^dZ<6~efE;}<=SMn8yTlAyVMP*> zf47O&T}W8G3dH-)$pmSsfs(BstU%%$aT++ECl0tHy4^yOc6S<)p?)U+jZ=jHhu{q)}c%O9~ z9kQPt7r?lkJjv~k6y7?)k^^C->hzBzfByT3y{S zp)qp5M0xQ?VVe(d6A*T~3)D&k7JSECgn^;!N>fjUhE|X8+@gAaR5G>K| zx*Jx)D^}|rD8RMi>XL<1CEw6$D0}HUY3uhKs|yg3ebmU9UIG!76=|4om=VkCqi ztFNGw?$Zmk+i&sL2i}AI9HfdmGqVZC7CS3)k_PT!59|K3jQl3!PIbvAmBmawZyXzD zR(K^Tm|BB--@NnZBWRS8tT>30clC4W6k$kS-T=7;=~*(T1^U6RPGX4*@)9n^(RDz^ ztimt!NJgryWN>Cys@yX9GN(9#Y|!akikprX#$Q*y5W&hM_7CD^;V~EL()j#v&gWbz zszmXV$8jU6-`WxGzLfo4E_~>v-%&QV2fuqMS|uYv5DhvARc^Z**8e);|9VNs#eS@z zGUtE`$MT3I;2a&hRjpd)#eVH|I4&j~a_6;gi1B>IklCtLwMDHew>#*+*k>bf!Vl4) zN&A8ArfaPoVQ6|)Vuz{aDRu`-a<{Mmc1HXaSp(iX`Szhj;XXyRYKU6CzSpzEO;l>Bie8BLj?N5#YzpF zmfM%o`9+#MylPIh1#&Ysrzm=MIBQ>q+3n=`hve6BzP0k0h`p+CrHM#1l&!tEL^D=$ zqY6FQ3WFT{IZgx$Hcldnz$_nba-TgG?G#j*MdD!psRPAQ^j*H5iuQ%Ln-7vCQq*a{ zLgTk)RoU+fGk?$K0B?OH?X}>~775y-<@2HwUm5#2|MHRjuqDKwEj6p^ysCwYj)TE( zK9?hzg>!rmAB9=&$yL*v!L!dQ&yb+5V_;}xuOR~OR&8V=9lxGHuCKv<{k))x2RSYS z-?+LsMJ%a=VXA=yLkp&J@tRFWVLt}iK=u8$xOLd&&XIY9YPSq41e_hB!*S{m5Bg3q1DfL(1fBY^83gUf0^I!J}mQ z=bE8N1y$e^<3-`R#Sh$nH2oRH@E$i(1&^gg`8+`yDyku-wP@Q;(7gNQB4Iu!d@n?kjP3^5M=b3 zVhBR#vWd{%wtq+p@0yASRr}_j2{7_rLxz%ynZc+)!LUT3SCC)R{fB*uBGS*{J#Uor zn$^a?nEE$!J&PNLLWhG(S%l~?oN-f%2qC0V^p&gxdJWrgCTSI5qNGRGfv(Kd%HSN6 zYfJiJ&s+{e37XI$q>Ids`bZIB)={9lomx%!eyA8^urcMW2=JVaBP*Cxr0-V15S24& zW1SCXlMo2$k%5pNSh{w*k#>rBjh&6-&FMQ7gvs4d>uNGB?hi03-q>G(e{ zH>n@Q#g5_zM-oSi?H_$##_o5E^TFvWA+fpNp|*UtZK9s9M)Z({qPSK$NXQwXu72sK zok@`dxleZj1wQ^7U^#f5V#Y-;6;WK%@xQJ($oD)NWubKvF!A4U3o@OCjoOB?u|uy4 z-Z0PLEXC9pya{#R*t4KR32f~s%fADAI{Y1u6YZd(!vu1^8@}g<;%fD@%vdgR)=@&U zOC`;Id<%Q!32c!4JGnWNuwU2oYqCeC{i|zknK+~a4n+O+G`OJ>jI3oh^oDy5DaKwA zhaYbfy6l177H z^=&DDx!nb;pr{KyrFmFq6M8(YgN=S6epbD^cTW|w7gt=ulH1_wjDi2ngGMF^l_K8Y zzD2j)^mo7D?n>&*}{mF{XQaHWu6X9H*CDTs_Kl?H9Q* zp8bz_sq0`X>a2qp-#D~Cp%;?wjR=!78dB;mv)YJ7r)~1AFY5x_j!x{|f%RVrZOMy@ zna>ZiT?y}*6dcWoxp&?PLaNbu!1#eUl$1Xqi^dD-ItsP7qE#45`N{iX2}mMZO2wiAGnea`({XwO+8v(-Z_&}#r; z!gMDN5^uMd_DJijA2Lf_Kq)C})jX5~I6840G?x?YF&VM7F^va3Wgu!x(T%`oDoR9T zWauMN!`=6exf;=rcKPzLeE&*y;<@M*)dmZYNh;)zCHZ~pbl-d{;?Cc z#mQ7~0FmTnVsQix!K0-TL2w~7z)|6FJ~qaOyM1oo9lVgR9F~AA zE_Y_gST2nGo+^ykzoe$`Tey31t-tB>o7vHiyUw^8EaEd7uwP_DvCxTuCVa==fd7gJ z%u9y4$l$ukq{6A>v1XoW>y5L*YTxUQBpiG=#iV~5gR0q9x}eC-?r|V$`|DObGLw)s zv7#Srzz3gvbSix1o47x+45$A*F2+q$0(Q-RM~>;&0O4%r0qj(wWBxjpavqQ-d!m2i z`}#@@Bxbm9&&#E)ff=Kv;VEh~o_;O__A;#&Yom%@7@_snp7!5o&I~OnuTVv6yh`i% zPyE=rUCGqmyeh})k7;k$>i6Hq=%62oPZaV+d#+)K) zab?znx4~jrw96lCDDFw5)Xn~!`f`T9xe9MtS_UsvhJugY>LOCB_y$o*O5+I$QlaUt ztY^K=AfnKIUh;=~-$fz> z_Y$ausggBa0Q~!kKa$o00k-c^U{W$hbizmdoT?gCC*?@~pg)8aBJ5V}+8ZS-whT@_ zMwGNnsHRf+b_*)v(KIts544#>0CK3LIqR;{+?qOad(1N-Y3JG7f>iwIBl1>Mn8EjG zxYg7M0~4b}C@X?V*dX1XUvlYkaNFGIT02itW!HfiCrX9UI%+5TMtVa|uZ4A^Nr!e* zSseI?*Izh`N`;~K-@gHM%w!B)y>aXiG5EZWDI|B{=-^y4_lu7tPPr@v1%IF5>gQMWS3Kr6 z_AG`}RH*tXQE<8+W4s*C$T2Tc1tw70jO4qWyK8#@ z{p67yMxifU=Z!rUWT{-rHR8i+tDFc5-hA-*#ZLC7^&H0hb#O0dwu3Mp-R#@b_eP!z z#=`3vhxL4O_c$v|RCSXM8NM{i|3(W_95$HAc2BRXR z$mO7MQ&<0Vz$5&&V|ap29Ibxdi5aUXFD^Dl>#epE*I5>_n_0_l#B-NkjAU|Ey$6SmYXyIQ^5KdzT@qYv9h3j_ zuk2-XRt#kOR#2dlg0T01M-8<@L;(prDa|V@k{lwY-xm*NdpK~^_IiZxBtPtBMc*$; zEB-tJu7>)Dq}LmbcK_SkaLqtm-?nJTDhS)IQC)fuU80#k#u#Einm9P>pPf}MIcL2K zXyWfTO=}=GKAkO2wT%kq(}G=7EM-kStxTCI7M{fvP^7U)mT_ zdW;2jV11<#b*O`wN#nU1zfhZ8C1fZQ^vj4FN z+TNOfH_Qr;$|h;J(i^{48^mW#J-F00w~!N4w8uOu+*y>67g}RmA%?*Z68r|nuFK~| zG<_3SiPC5uDGN7hwnmS5T6EV94EL01Kw$^P@cPW?GZW%ttz`sce!inkaSglQ{@^gB zGLWdAajG%s!;9aMHjmE@%6qV;c~7nT=Gq}@McVu`6-CGoC1x7!k3yH}56`=;lI1r) z(qSWseV!AK&FKtiddEWs$c56h_ryr7{+`qChVR7Sd?4b$K#Lez)Yz9BkoinzHHNaB zAzY$rZgZ;T`Qc$^;z025NeTP~n~kl%8N!YE?U2F{8()@m%QpDy__Qe32`IS)Sq40I zHe;a~iMs{|D{5>hT{swB3NX99ENn7CfDNzkK*ZHi3NM$@c>#HA_+lTL>EvIUtF6p<{Fe_#2sFmO}p%(%HC(?RtcZ?@qZwdZe%7 zU#`HHgB>EJAqQql)|RTqqXLW*spo}H4eJ?;|FL?Wg4R4&m#9p2waTVVxHClhZa&)C z;ALV!g@N^vBQx%qWa;}0OietrMhV)Ja*In(vV+5@SkuGY+8PwscFfKbuDY}FmBu;q zT}^B<3ny6m_g`@k=%5-ORaS^GV%jtlbW;#|UgoFF?c0))zc*2uube9^h=bUs^9Ey& z&fv^ppOe2h>Cofw*g3QoMSqx0Y4R&}lH&o@kOZhNeJ<-iOku(o!md5+#*Qr!6|)5( zE6XIzE(~c|t-yI1D0?N-9vEwMs#n?GZ>TC;%UwggA(qmm+k^5mR$9abV1zx|<)%5Su`mRXr@PXiiIvWc+g^Ctuv);8 zvUop9-f{I_Ln^86hqLw>7hKv-?z!?SLSNPz#8xyd&b4Z~kXu;luM3x%Ul=<&NcbWq zf{PWICbef0q=dwbE3{i|Z@6%S4<$3xBZ&~A{i&Wd_Txf8sL(ilfWg~HBq+i-s%IEk z$iyBl-~f(Y^)dbU{o1d##bdf3*< z3e~{~9!m0=2b^x}H9nVC&jSQOKqXPdBvIUNVH6QU1~r3X9EpUT4H3blwqvTOQ)szD zuaQaO*hTtvqm{nDC|4f45T1<$aNQ^Sh9V+}^H?|uP!X85D3Ow+?~&W9Q3p6?j_+KZ zRB=d?cwXfLka*QDBJvxpH?riX@_25GQU#DPU}9uMQ37QZ6^F(PD)kAf80q>CPWpl@ zUi9+>qtaVfqYpz$Gwu!q*2GMg0}oNm&n~O3twjhftWnZ4H6g&+gwhw{N1tSCXqrg* zSzyHuUh^_kp?FifZA(z)tE*>FB2Z}vUe2o_J8pprRq!|=(8mgrno$|fZtOe{6paNj z#BT{X0@~YUl6R-<2#K0SW{&W&=(ajDc4oiBToyo#82dL(Qp$jE)uH;cr)j6)ZM;jtF8e+OvgU zKO#si_(#KVv73h{h#8Qq*W2RqbT3Tk6NoJdf9q*_NwP^&rhJ zkmBNpt0OtsP5a(VSzQB#-Z`?I->46V@oXARfXR!=Wc@w|sKYUTi1{huBz3uW(?Djh5B*;{I6f5{rzq9h_cvdCqeG$;pwYoTA#oPptIo%Qb`?R+hla6oOGbaD z{!tH~KH{*}DmT{6AvVWU=TOD@Vf_d!Qa@je4qpZK_X-Sa>j~i zfEYZyp_DZX(EN>Wp$v$T($pjgc*)Mn$Au>w66SVGN+lxR6B>?7S-J$f(p7}VN|)Bg zH>-nUVT^NU${VH)c2`--$$wO*CAZIJ;lH9}Swi*2Ub;3O!V?*S^f#)q474Fl1~l&g z!hNGYjonf2kNZlqJ)0V)9#z_JkXl+&@m1fEkMD7IC$Y(liH-m(9S9;*e*WQUTvRXv zqT%eS@=YQ3Qz~Q-9$Vq=zbLK}-Uk$ee4%!Yio3%#$jvnim*vwT(~x6n$E)~0{?DtZ@oSEl_fPlQ8@3A z%SKf%mr6>Sg4J?y9=2+hju#)+G%1U#I(G86z*FZlyDv3e(I#VKgqFc@OpUlxbGVD5 z<_95+QpD(_I&|TI4kSy%E*EE%+QK)YYeG$E&tnZ#u@BcoXXxs3sQbvrD+vD@4L42D zq{Zo477|d*@js&w(NZ&6@oXuMBdZ=CaE^lKOc_5RX)kamA;$fQ{}mZALaJInI%*Ja zeXee-tO{n8=#o*1o&LyqEASFAzTFLzSVJeqXVorInK(2B1K`j#RBZ;Jn9TLBs77x| z0)h^y%fuVDqXv)Oz3klAHV?ysGge4$5l$T)Pv{4Dy;?t+nYm#G{1V$Xz`8$f^Blx&v{f{9txV?#Ya@Q=?{ zu)L2Kx{l*+SCpZoe;%`OLlQcieqV**Ya33i?*QFxRVdQme=WK{HzMZ}Rg7Qxj7o|a z&^#xI5Z3;d>oaON?*u77gP2i$#jT;%brd0oVMU3CWE%t#OKFnxz-u0-xD zeS0I)kTM=%lS4w&&a7ZwI+l78TmIvApY3_D6W6DFS}F4$@Z8GeIFTb_+O6p=PU##= zmQf9h0DqLacNVZMxM!hZ#TkQJL3yqi&!-?ezf6NSAheI4znoIjKfHiXp)bEUu&7Fjis+N73vH0_s!i z8GZqZlu6>fq^~7m4n-950t-42Ov*`IxV%o?4Jpk zH#QT6X%9PzOdOHvp4T!ZI~VQQ)r1Jlwi#_c%!Z>-!)5s3Hi(=Wp+R8_|L{{Kg_yRF zw%3!XIztcit8LpL^GAUBY!9M}iQBG@HcTX1RN3svdcrN$CuqIm8#q2E9z&#lL@8I( z^HENNy#8IL)Vs@=QauOK*rQTF1m3CUnXBm-{nQ909$nF(^dJUUZ4VlSgl}0n7^C8O zqXcDVboKP&VUzxkxPT~lZcWqTk>SL7F;(`@hL_B#+|ID6$s{58$?81k$y$rTw$czyZ_vT~syQs1zS&{w zjOtvRMZanZIO_o-rfp?^CMaNnUT#oyk2S36Y5Ex>k#r_PSxEJjaAa1vOWQ&r1};tc z^{pWo%xKw@*Bls*rdC=&#}5DqF&RBxXQl18&4#S`KgB?LW7Y&M_#z5GNCzi{J+w_wuHOZo zIB-T_8*_y<^b;5~`Ce~jyY}(0-A`w;v6;=URufWi__37N4L#vwYK1p! zKLE_C(pgFWloe+n6FJapQTOfe`EX{N;P0}5DJ(3yC(0EqwQg5Na@&TBu8W0#Mrv89 zb)ytER}bEoTf}|b$3j_Ms0|a|jlKy4LmB%gPb!W5pFkCde9crL@=JC2P4nC-sONdE z3D5x$g>RwM^nyX|xwJPUx(P+5V+%t6vY7D#N5G8P5+4ZJ}8kjGtxwTASf$ z$j&Np@PgvpjrQiRSAju{fW=f&<@{z;S2;5lVnH~PrpLtC$KUxz8VYr&pwA?hcI6#D zSR$zLDKfjmG=q~7pnZ&coB+p#{OX9s+!K&Mva|IMvU+DHx7%`ZOaGV5+v5eF$JuA= z7SdvP0uX*h0J_WqnfaKkX-nF0Z2@NMPK$0QzPO3Z)mq&_#ZJGP+!L=^FIHV<=096h`{nfaG<|*$%NM2? z(E_3D&_SOcl%56w7!!81R7F&Fn$Fc|}f<^Lt{UqZrb4Sm8U*n`C8!SsgDNi>v6InqbUj@o{i3 zc!Zg$E@$uiF+t68<033>TxDefFlN@J`{T>*!%U9BAr>d9GWBhLV}S1V<;}YzAQK8B zLs;MccBF$^5XKP0fQ2Q1#*!g7md?XBRT^jBdpx|OIuM~fY=7XYt_e6V435PJ?7QEXCKk-B@(nMPxt^)CIh(+J*SPyidluRu$ zW0n`JL&a=HE7aj=r{9 zWUm?1NRL?8AlbWBcLVn}qtd)$_sgwf1!0V>)m98>9$?uC+=}C000gf~Q`8DtIb`(a zbt@r4z+M7%EmXWGN@cTN=Ln>F@xG}C-|c|pu8y9zmOS1TR&1|ca9>XshQ=0_CI z&E9%`!^tX_^JE@7!f$3wpf5sy);^i@$`#S=u9D%%$yS%jt~Z!vK3|sDmOTFl(9}Fi zrQa`V^Ux2%L+e!N3aUF5eYQ7+jbw)7jOwd=_FocXWR{GGwBB-ekEunr3uCQvjcqkI zN=yRTI!Pz2NlRKUxj=qLsu(K+4Y}#WXNMHFl#I}|nVKB@iuukpg?S_<-H71ipU2bq zL!TNs|D)Q?k|1^KH^qax>eKeNe-sN+JQ7%+vU`EWdgwY?<*{y4aEmhkH zA4vlrFI%1DNuPtS+azea*N$w`$JEa#IQyL&>$j+|oQi$yyM-D+)DXdEgFfCPlun+BegP#L=oUK2A^EMC7d4^vYM;4e~BXPw-v&3G`1HplPQq06N z2z@gXbIHRKJLjBAoan%>$+J3h-siXZ%8+VtM470g0xDEecs`R`se^7vzXPA-H0PL_ z_!k=D0?Kdgv$Sh0E|~wt)jLL45;pC^(ZtTgwr$(C?M!TQCbqR>PP}6$6Wg|J`{eo7 zde3>EZ`Y5#fAp%myLYdv@2a}4syM^5kX~M((qE+B=}kW-K8p0ue+O9HH#)wKsi1zS zEdg0+g4G@O5)hPEJ(oE2ZPR9c_D@>RGnnItTR)!kzA#<4fM~iWN0=CZXqVUf?E-8S zGt@#3bwcbHm(!ylY|cDhKakt1&*QY!GzdyOf<;t4?*V+QVwiv1MsPOp`RP0dJkOGL zgA}iM0~Gf?{qvLICgMqGU=8b4>o~Lc0qu_%@8m=(R$WRm&p_Ui$4NyV>^A({qML=T1v|8^r!t$$0(cM*N|%Ik8J2?M4T}q*YVY z>GSPyiGFlU&Zozpxc2#q1a29T<#;;-6ItzZBRW&oO6uzPo)rOc5g=!#0D6!@3B%y# zR?z!fJ~R<-$Bgt%Kt#U3S9j zCQ^e-G6N7`@>aAbQ^Ds^p?H$q-NAM2l`stBiG9?k5Z0qFI(WoXwsb1>Pt8qr8Re*^V_EX)qPXsc@dGl;wd;BrmUwk)RzwGnob2{Ts}@T@w|A#jF{Yq2=Cr# zgtTok`{)fl0^h0(G-5f@d!MS~0ryBte0#aKb4q@8rIF3UGbZKtyyXZ&SuT&)Zt^C| z3i{>&qt2AQ^i$;MdTxFDZDoClYTK$(_C;PIjH9@y!Pa?9MV`kj^9QHB#eHX$-^5&q zNd-KjiW`56{KvF*p4kJ(pq3XQ9Me9>C@A6~oZV>7gunr^F?uJ9^+!zbv)aaHAI!`J zv2X%|2!c?SbER`cJQJ~Xi~Sc`&+h}w8(ELTK+=9YT~Ac8Dmq?4e_;TGe#{{}n3osd z!|yCv_mkOyjAML)p+R@$Gqeb&K8hXNn+0>fhB^VY1#9bCsR4F9ZVf}r0tYfq!9{P< z#wqAWUR&ALhz6(+3!Cz8j2sze3n{v_kiL`};aHeGBjE?;vuhK6lTZ=NbzW#YqAt! z@G${Zl~xa~Oc;cF-i3iu2`_zvXS*%u&?37yTz}I%D*=z?LNCPle9a2Af+yjAz5#X( z%i7B`(i#}3eB{?*jro~`lB>>XK^h#g`YVKxYT^{HpbI>(8w3||0l2wud;g^0njCjT z)?KFf(>d&La=Z?)U-#U`lO^ZilGw+*(sUGDk{Sy#&8cB&-&z+IZm|AJ08$QqBG|4^ z^lSW;M*#3%(2TZ>cJjROZ>G7B;Kr25uQZ&%nU`Bi615*DUG^+nc@@W`!hyF;>6jL% ztXtSq8}DGg7?|3SoyW>~DVKQu9gy9+iQXG?`n3DJyf0FF^sgtSdS0NKh&Cg3)}|Ik zkgmFS6Y;-Lin1k(+TGvMG16miK>`yQCJ1LHb9*4F_YT#fngcu-RerMx|39+7=4x>^K^hmI`+yVo7 z7QS7{M8aR;9~b8Kf4XRXj*ajO$vwC*Akp~L4Hoy*dB6ip!YNlEl(DrMPSwSOTe z2fNHrw2;v;&vg+igfB^hjg39}-OK(;xpIAPFy_~j?<%=)soMq8Qlowb zPpVd%mE$+4m+b^ggWWlffjTXFzK3~L{B9zl1gQ^6@$Gvtq0{5_Ru}GTFEEN~uBN3G z(7ic-W1wU*W2VM+qf zjNN$j6w*Pt9>dR9WIOY9I0bI2k7chYH@VU|T6|IPW(sB(wBfuwWWM_v%+`vIR5&mT z>IgpRv6T_6Dz9@+CDStKY;WlB3Zc?;Ej{t>$@x!kFADp+&gj@jfg(#^ji$5&J7cnR zxyI4J^A}D567Kd`0ZYyu{G2#$|M%xY@}f;u6e~;!1z2WN{6Jyn)Tz3viHqBUA-Uf` z&)dR1XXs0F>$Hn>BsZns7C>w2?401y(27gs28i*&jl+~Xd^>{aQ~s1V-eHQ-;(R`! zrG{<@AMG5jNK-bsMWQD7QynT?mMaEoQ9cqp>>#+JK#g4ea4^Ws2W75nO!i2)f#U43 zxFH8&_507WEM5rygIf+Pt$_0Yzi0jGR?+Jsx?s&D+y03HImKMx6A0D>`O*0#=klFr z>=i)AzYK%+3F|^8&Ip~y>&e)%9e#fDQ(cr+R&49Sn`bSDjl_+tje#wO- za1jp}lDK(Xc2_zu{A$pGKDgBSQmLQd(-5x|dD!QZc`1K&ePYe}-%b_ET zpi+zL9=%Zw(?jA;_Tj1O{Tm+1d0r5nrP` z#E5#`Z$0jJ_45FI`JK3DxAhF(c-dP*@6#U#yOn8%%8$q1=XFEdSBxn;%lWBUwHR}R z7Gs~jm(8|9{F+IvqVa>g-IaI;gF3d8BccXg2qPzi_!}e1Ga>D{S4YOjDUB4ilau1% zhCg_XpRkspsn?vYkt0WGYB|q9;-v3fM!Y`6c4;7n#^qgN7E-PsAHsU)*6hrZo|ulHvbu$h1!S?6XAuFsKh1co{Srkyg+bk)dHKiZYe%;}nHt0t;_n z&s;dqY#M`*^`skbciX$}w^_M9S#!yqAWn|>mz@vNTGOB0w@W{&Gpp%t8SCj-ON`_D zwY07bU$K?ekX^OQ@OdeWQ6y)3t|ym*oh&3l8^9LsHt=l$5%Ef&?NI|-{Ru^qqUUvxgOs&%o5&ntgd>1bg<);ysigV~Bl9`sqW znUPgDgH(9TemTOWaMK?uEHBvMpnoq3@v+sxUS*E97nQ9_I)X>GxtK{&Hp=Pw_npfb#~}P$L_h~Zkf{_p4f0gSRsTwq zhH->>2>ul7AcP#p=!O=8?aH^rNlKXBjh~KFhp115`)1t?O>iESR;yS4oPn(M+n(*T zO_m?kkM^+TrO{8w?IB{FPzmGCGY`04247TU)aG%uSv@xitQ{6Y_0{mDX24b&t&iL& zct$7YTdsCkqpiliFvWMx<#ch7=E+VUAzIzO2x1W3i|6ASvGXmy&%sGPoHrVy%z=)( z3L7RcJt^HLIG3zo&FiQ{W`FtW%Of-D7otgNS2N&APe9XUNv?u&`#`Sd*EDeIt7I9Ihuzp#7{b4dbw>KbpTO>mI{H@rIJe)$i}S1B1s> zYG2J-nBUZE_bnENLdiPFGvfj3PPpM=J2T>O5?k!b8nb6v={E${t{_=^AWcdo{iLNI z2^z1|S&o#=;FxcdjvIf}ahDurw2$;Cwh8RMNy=bqj8Jeb4 z+e8ZzbD?c^LZSFS2GVsH&P6+G2MEsQUmjB9VpBS}4kMpIDwg^Wph;~D>v8iT6n&5i zrd_w|NwC!IuCL^t7mZ8EC@A^!1CQ^Lwz&B2VQ#UrZFXsy{r3p8Xz_L$qP5f*b3qB@nkp5 z-D2~-QJaA->pvGGtt<(UAt5TXQ)G1D7-if`^D?9@bjV_j5|$e6&*1#48Vz*Nw%PJO zr(@HSRe9XLSn89JU1?vhy!{8q$Vcv^{6_=DQ4`|51^6Q5PI1|Bi3$M$L16L0Q!rjx zBXZnN!uC=y=D6nf6&NdE?zX;p$esv{+w#{YHoxk)LyXgc6EI;pML-XF6cZx1xQHmp zT(k}ml97R}?s~PU)T;|F8t>vvWM!np9lKm<5A!#RO2};|6qSJQDKJdi7QPHItElM> zSm`nJ+!3E7D;`uvQ9^M916^4Z`PUzTR_FJ|VSi=px_Tj}hrSP8s$QXCI&mxBl9DC8 z*7769gxcqfeE9xA_v@oZW;B}$ie2W7I1jb{u}?+*giS=Kgzh_L$>8XmvCQ=g#b&i| z8~{KqR;`AI{Z5{-ys2v58f>thTTL!`jgJhB6gq?EYBK*m3VwgjrmV8Ezaff2%nf{M zf-U3vl6Go`G!b|*S4vwX2k3G|S7!gbA^BcuPh82*Ks@gU7z@H_{t`9mbxIVbGqE(n zrq_|a+bi&g!l@!UsE3fyX+;nvlbVzEO~scNZ2gu`Oe*lib8muEJ(tCUDvwnOG9?`l z^ryP(HbPt|)_EtkX;h{fD`(@0VQN_w|3i`P(B|}eJ1Z{aE5Z+-v)pLMDwvf_&usQ! z?qeS@qp>o7G${wsLZpL01NQwWMFh&l3`f?Ml~HxP44%4MBj0X2Q`d3;&+VmKNiGF5TrUZ zB*$%*dSJfy#qsGE+BW^*`Q(8dhaI^N|)w8d6D?SRe`4Z9tTeTuT0r ztH!`a@qX5)(n8xGL4rF=@Tr9ZNj|Z+f;u=8i7qE+Gq?6R@vJv^PO+K_l01oDGqcVV zImcy1P7SVPgq9(!UNSn9{Vx61j4OCo+fthTgtT^&>!~;UZNB^7V?G_O5@Dpqi>N;> zce$DDUn*bUiijO$w8q)^DwM9uLk2}|Nw)Z7i8!YC=e&;d7Gxx=>ICsa zYe5kjE&k9oB6tx)qz;FB>)KM1jRA*%Q;Y4I+KLQ%hM)_!(pcc!;HPC8nBE+^CUUqtN$H)1B1ub|u|a74J0q5!~^joa?S8bsx5D>e zC5hMj1vK0ZF|loGPH6u>jo?{?DCBr)LQ85nnbW6#77&tLO z9*Ck_3BA+lS`vwjArgwgHh=rOraWt}vp>q3UWj%w>zy74oURiV&P7JGO3Z0FOlR&+ zt~)K@B^2U9FkzlhqKjnOEXM_Bw{MeL!9904ja<;<*TJ7D$C@ocuGm$Tt!ram<}puu z*YSE7@mf4+9R2C`>VzKc!E=fk>?%qOeuHo-fK@A4H(P)>h+5okH@B^Quu>ews zJ%iIAdZVp&f_9tp&K2v&Z zcb^1qDh?)DmXHg1y@7Jto8t3Pn*PNG*Zge#kN;+R452{Yz2Chlt+OMAS8YkJA569{ z=!F_F9S=OW6`EGOUFtDWZwFa6#5R1KsMkPu1&o-^zNm;+jk-&bEe#BX-VN78JI%2H zX+1s@V`bWjr%SQoLXp#UA)&JAZU-;R&*zHI4*ZAA}#FUSFHxB2k=tXNV>btYh&_1@JnA_;d%wQUM;IU7urMw)#5INt0oA;S--7 zb6>Ng2Rjj=!7szlj|66y)Ini27%6L8J=Nk{eP&J65HmS=FnH!PTh|V@A18Kd^HBf# z+R#_va{4}_Kgh&?#lPGYE~ho2eGjI!iaa!)3dW32Q-$P@kj?_5t&YLf5`06yezxfe z12`^PY)23VHt>`jQ`x;n>635?n{FW#a>0!1YNDfbH+4aF~$H!Hw=L{lOQz3 zeNIKaHPsLZQ-yO_NX8dvf_qO>%g#_QdHx%a?;I}R!1@gGDnl6~K0q-fcP>MJvtmiU z+7)=*sc&V#u1Aqwd2r04imW!w>{e!6Ycr$T=#oSS{JD}ogMv?N=~nk_-G#-9KbWtk z)W1GRUjp9NFI~7~nDH1TS?;Wrn~x)xQ@9Z?=m+v7z`oyIAUNIGE#G#2Tp&^z+RS+a z+;ZU7(~sIre47dkEX{Z`DC2b`zFTUhq&9SE=epFkN1_8di_+Pm|74oA945)Fbw#}V zxT!{WUFJ`=5N`ktg_Fb+G{dJBl>AIl>`tsRPI*r35#-oSs<{Q9#C*uvbOpfc47x!I_zl-iCz zXw#&&SqF{#zz1n?!{VLI4zjq5U&fO>hGJ?u;UvxxF$U@JgSYvKE0f2E=K3J!eB2wP z-DUYiI&6o^E>dgYmm;Gve#}wXmp@1F9v6@S5`)zMz_$Kw#sYZjf^4dFj$$vQ-yG#Rv&|`sN-OPxdnfqs?*G>2b8-3cvTXj@#$aB!|t@bqKqU0Pt6U;qg|!Q!{x@a+o0{ZU#PiPd@#8LDFrh~PBu22 z0(0io8%c6OU)meE(kqK|>#G|wELFf7z<#a3L2o!mwTItROng2}9mNvvSR~IsPR#9{ zLPlzfe0^8d(;d1>PqPL)JWyW$)GZ!$t#|5QzQHq}#&ZG8iIgMLmDLI`*btTNf9cWm zJ^ED;=+|#b8X-2viyQb}72S++eJWbPYVPe%7D0EGJgk(qqAQo zgJ<7|p~x1te80HMsi7A-8xk^g}ZWr|0iga6d)Ki^=zG*F7^M|R50eP+5 zlS-F1S!$hSg4L$Pdnj_Vh|!FH7Y%cTxp$geKEd6lOP|&xSxUt%PHx&0-yE1*b-df2 zdTLeb!3(~d{UH`&s!I?8H$+G}O%N6*^4XHVS;m`Fyu!NxH9q4ir+*AFMOKhHBtb{Df*zoY+kT?;k0@ z6<(owBz<`6AWbL>huChjxo~eVJl-%X>1l_6ROEw-?*toLzSaZ=YQ6J3ltjT(j34q| zO_-u5lSqU z2I>e1YL4U7k$~LWeO|Q2w$R((h%g}a#g4mhsC3`Lh8^UeeYl{S4yhoyR8LgIL|xg` zg=kLeVdRj&i8@$9)QStk0H8K|UDne_LZgptBz8LXQ77zIrI>m)dJc1e%{VCwg08KU zd*i0hkcZO(gOPxJGdT@~+NmNP2Wv$2oDgBp?#=-iQ8^0JEKDVNJa?peD~tG@h><*A z2wHWf4_iLf&5fIH$&9zZG3AIZT$JO#8Kx3;;0VrBj}dRDO~}O+ZIg^1%LCNG4l0~~ z79Z0*+zio^SteWZ@E!{t(qEA-os59EF!NG6pa$0UC~d+$K(!@6x5@>Jbw_f240;=% z^_SNZG;pww7Ujz=j4Q&Bpxu3>n3;0I^phu32jl0lRByXkt2$FBaXPj6K+x;P=BUHb^Oqn<(_2^hr(@GMY1T5at;Yra7ea5ojt*SuE%dqhzpdZVd zQ;k%c@9qBBa_rm<8(_$a-A5nZ(2IU;1L;RV3hAmQV1FB1R`C2cCK=UZ{NdE=lV-B} zGXV7`eFxe<6p3$G!JE$x7RsLbrL0+paPY^LsL*HThypovR!#?FS;R-YN7Y-9{5XFO zuc&6g-u-tzw8AAMpb-PXT)ScWt+$UC#2cTS^Vx;01usKzGpS1o4P+W}yF)4aX%)Pxkhk@<9$ch)6n}(eKiom~N2_Gv=155KMMq=!}B`yI? z7Z6WZ#41{+o+PwokXq$qbT~40#SkX|f=G+q+g(C?Gfrwc7nwVIUjC>U9QXuFkiqr! zlZu9A;Ti5}+pKvv!4*abp11|VkqhIq(B zx0>9z=O#^HV%~rlL>XSiOP7!Ujes9JtgI$eCTl{dTzb=)}M;me1VTYzXxkQwQGH&hL2>FiBx5vyYBOIJbMniOO&;`JCOQ;aQ;EdWF z)H6<$K1Xidogxfl4-PF%qWaP1hM{N)%!c{b+-uPbmA7{Dq=FHsuw z+_amF5W2c=WPSZ7(w<*~29}$uVAXBoeXT7Y;GRMo!_DZ|!1Sdc3V2j?|7g{*F3oEc z6!btuX=0J^;u=+O1)FIQ+$K&Ww09~aKpaTHre;}y zA#jZt@dp#Pz3p9}trIu@Y97;{fGkQyv}H9u)~c+EE=YXo$EWe9t4g(q{3V9g1M@YU z?4gwkl*z9V6~LMG|*6 z4=x*7J^~c2vbr`>Ty+9jgiiXX^uOkXUQ%X=pd0@N+ga4n4*G{is(jJNSUDChZs^QW zm7kU=s{6=dPuD?|hU05CVkye~NZ&iJYFDKud8A_n<%3c{~I9Vzb+%yZ-z{%~MD&&pf#@KSCMpP|ZX^fa87Q zOC)s|-HhRpe-gDd({m8Oq~y8yF0~veN!P!8!px9CzMna7zo`K)_pfWT;!QrJ{~{_; zET({r&g+L~bg-Ub4+U{aW#7lJz-a49X7FiGaaF6BfthOo%qemGc5#Xmj`R_38R*(c z4o6Cm%jI$>un}3hZAVf2o?_CtP@mf~%>jx9q)`}LpTg+5rmEr7o|~4s*Hbl8MOuf{ zbXRUZ(t%}!E0s%OQrr7prNRAsF0yG(3D9FMuwxH@ue3{&r7=*%CF^`pg{h(K+&jqNfCua+7km zL=W<=v9IOiO&h<%7gpRRh29gj7hx;V`Hfo7jlH~qpC!X?i{+jDiR!rEo)1_ zRqOb;t4`1hB*{Ci?jmuKJ)K~8N#L*Z3an3;^Rp^z!v}tgLUjxApUOnlT1u1i`FM!h zZXsK>?$&JTg}vL&3n)7L_CQj_Axnd|D7g3S8GKmNXd)um%*XqYx;Nd0z_jp6TrdRf ziQ~(SFnpp){M`@~sIv@qDr(huS3qt>iw|VYN9soHmG;Z0)(x%9y&B4){{xMC2m6}s z!w<%9oltQA2PbMRiVQtonLZ2sTHiOiDm(^Zj=d5C0J;t*Rt^t(KD-~VAP)i)m-a!s z>l0$n#%8#1M=Uv+@gqM66!q8#Jl_esD@)n)1?gep+cSc7>j$(*-K~K(Jw(Jj$ zuqhlKjzlKk^fxL3?@uqw;d!UZBFgLu_R5JJ<%!WtoODr8qIbTQUm2x{1q}PUCu6Xi zbsvEa!)PmwGb*p@sW^JXNFiZD874^cx1@fan@&)G5)eEm?5gk=)D5V~cV|jTO8R1D zK$)AsPNbJFSNh_b2wvjz!;R@TkgAO)a_D_{WoPPW1tKrG?G=vq!QU^Ccs7F7E~1`0 z?5eN9XK*#3T?|iL;&7*jG>`>GLDqeZJ|^ic!Rk6wL~s8iBFl&lyxu60(3Ij^OuZ2{ zt3n3IAMW>9$)4)=P3xL@_u&fyN~@BLgaRj>=^|g~=DZL2n=r@3crGPoNeqxlOoTvc zSPgi_9Jxo%Up68fAU_Vn3B~(ZU9jR#WY%jqoLB2PBjUChFk{gn7-$j`4 zd-5W^-M8mU^Zs0~2=lLKgnqeIccjGy!npiJgxp>7o?$bGjPtv*X(yS5%y_7{&4(GSY2|{%EsMIbzp; z$Z|L%c4)_{MWw6}3=V|AJ+@(mF^N~p`8vgub;9_-a?ZiN1`~3BPjbuzO+ZY7W+*9N z&duCyynrPfYzvvWr80>3Mpk4$gw8LiOC38Rx&N9Yp z62yi}Ym%~5VtEM)#&ues)SjZ|eHWH02|fPNTHu*L#{Q<{6fp5~qOYADtRQt6>>&7vW727_0k@MYFQ8m3~no_jUs2PyKu7W9n` zYOxHvQvQ68gBl302S;Au2XTvvjOX@Xmuar*yQDL`&sj+#by7sNp{=rGr&?AS-Q-8_ ztpl9~7CT`xE=}{Zn{WnClSZ0_9W*E6ui0{pPB|F2B3d5FssJ8w=~1$lL30!hQ;^*& z&++E!gW~*mPJ7YmJQBlwmZf~sk3B!Yd8Ia4nR0H39zxw}eFzn(-5Nb!6@nO8Zdl#u znEc7Qfp^z8rKGI9;b#7rEYH|bAXiEn0VO_)u=MhG)){*bS(J055&&6?cXe*OlbWK3 zuvZ-+$m?nC7=L91xm>^uBr5WyLda&_LEU9Vg-?wr(`22 z$ge0ZL8^M&6UUTcv4Wd3|Lwa4GKXJQZUxbbXnURljQfM$;r*4_MdZO`DpG*`nN*@F z=81GB#O!A`YM2$uANOx-;!#**yBvu*cFA|lyWd?2N;A=xL6iDz$IUv(v;99)$f!cQ z^vE1Tw1tOM zS)K8(^~(k&OH2-M2VIvQ8rr9ebGD}ax6Se9kA1K@VwyMD_?iYPlELYZSF&70%R&HF zrVE>{GrjhO+nVGQxbwAGazcLckYu$?-0-1e$#z?vQjr`SEwHRF8?h@mXXY zu@`LQKHxUH_OJ=5KjH10S6ci&YPY>~<(^z@lA>pZosac*0{+Kixuu)ynZRl<%_)KK zaJpn5=X(JdGO~O{WDUXQ2=lSe8OU!;3!|5$ZevMhtWZHW2FRjQOk*+%SFOZy+`tP==!V}usLzW5 zf}Ztw|oaRGmGTvaN1M<58j+s_cx*9{6+88?W#?{ZQs)Esx zX26nk(}n4QC}mN_-TSG|^zmxUIp^Ob;Gi>;=Ndi26K0J0H^Q?Y84io)M-OK9?(WBz zo;cWmLVY&E6X&^H>ozt)L)(PDLO|{xGKty0cbT%neF}YX-qIVer1>{HcTVR-pFEiw z+4RXlJn2Jxmdsb1WawkdC-1;CKY?T<&_jFHf>1Be9MY<~SlNfA&8U6`EO3$h^ck ziOMas(U*W7S|v|tOL@LseM@@fqaC@OKe;#~o%?5;-8nEh;4MF5o0`E3p`3dL8o)AQ z=k^n>$i+08cA-fzc4>BooZ(~&zEI~lQA^*#$2OTx;n#BW-`iu-KONARiy2P(T2sK+ zazB%wv`3$Xnb!zXPcAmJirzor+!``Ze3VPX=l^8DNi9ALtmm#xg1sC?!xp&ApJffk z!`JiW5;r*z7Io>%0gP}vEeY2i?Xn3daX`f@oM#ma25(TF>O{qRW2`si-IEV=@S>5 zABei7fd`&146kVDTHFP(VxA;`qbiH}6m020w(Q+|WxV%RC_cZI$oO1?0k3B!*8;RZ zY0c;GQ%gy(`kG;<-@Q2D4X)wXsnRl!{+4cZ+Xm2gwca6Qf;Uz$qwpJqw!cqp{a zlCCZ)h11x3DgU}5PuM331PN%udAJXt23x`xy6Ur5xN#>>u%g-${c6^0()%vwXGq-% zeoVK7=XaIhVH6kdywQsHpn-Mtaow6t*eH|f06@_X#LKi_m~t*(=qZp2sY);kMieq$V{(N!lKn_dc8T$;*}zc4GtS%m z!twLNkno56ys8=tOmYx>kFqT5Njq;);oCa|Pia;}8sLGTj^Oy8Cj)v~Mxueq<{gZh z80>i^Es%}2wVfosC%C=n*{;6jBQ#>4IW)%w|4ydyZdOOcwu1zYcPF- zy}@AxBc8xEa1-Fxq><7n3{v6{>r5R&?P}km6*N`#u z3RO^!sWrug-&@e78vdx!yv;y%FTs0LO5^sE_8T!tAeU4%6w$t)N0@x46^Ozxp*lTg zVuGplBuK&`$3+tW=Siluw)R;nhbQ>yH|6);a$ zrfV{uR}WJ-1e$JY$T-hLIIYaku5I3MlPev;{Y@v>qKs5es0r{iQXd~Tm3;hBWyHBW zvXkJNv4GVMJP{G*Rxy#P)C$<=AJs-m{{uW#4KJQg3I~euPQ=mDEr`(d3P6Kix zPl@>Rg}2M05i%o>O}w@AUZEytxSVq}MB>g>TYhXL%KJVC1Fo1dUYbvE4 zQh+;8eywzx21m;n)V+@!d0rVaDQGX1BEtP#wT-uCEG!6bXc04KQQSY=Q6khd4NNoe znUy5kc1$QA6f|bs3pd>`u(55R4XA6R-}-1Owq37@YCEIt;BL?iHO1ivcSPLhRgT0A zW_Vvt#~#D%2WRP4z?rFt9-n-v=Pq<-niU;}0IHWY9b2LS-GngOj60Lo@4e2Ss_~eX zV#M!yN1?UQCWCR6zExEgYm>Lh_Eb`}=4BT8D{{W(Tiu^dF>a|m6%Zxn6q72J9V<;? zw{tdr@xvXQrWnm@UXBD7;YsBai4ia@8R=+&6L!9{5;aor>C z={Udv(EFYA+#jFh&wn$(*tUvCa33r*C}vm**j)rgbe+3;c5bQ<1we_7hfV#0SB5?x zkK-p*h?=F5%y3=5{Kpsw5&V1dFIVP?5%lGs>ut5L>*N&ne<4N8zy0k=EbEN|CTG^z z+IP-11!(5DfjBi$9rZi?ls4=&Yi?J1I9W3Q;v@kDl$ zbrq_tUA2AIaHYoo47)eic1}K`l648eFrbsorAoBok(AQbDW>C6*5&Vd;0I5KV`Wt-{5T#M*@;( zL&jL`5RxO-kVqPN1!LK8oZt%Ngz3#93;2>uu)nj#6t@XNbmmNGq!04d>?Op_-+41c zeJv!$If;&Usnph-L!^#y$)m~l2CBJ3FI=hkOy5X6XU@3k)GIkj_Y=j7MKHlp;w*ia zpbmVvk+>~W)aQt!CyVC@flyiPCiq)9pY3Iq4b=k=Q*8;AEs6wNPGbqVj&jVK{^oK6 zWJN(l0|^z&6wg~*m-A=cQxCck$MzG-Bo)y&;_bzLIRn+~!GZin>0X?z4T<2zZL8nK zCN5eIC^eD?KBh&70l(uHq%QYVf83vGU&czw5o6C{>BP zKVtG8^M?JRfj?&_eh}Jl!GcxIW0N<5TMiW>c3PD)Ug$l>o+G}Um4VUUxob#Bnp{x- z@)_ChSs*s`dFNNj7$9iw}#*!wcJv$M~t0j_jBv0v2hU>ir{`^~c1igxGp7 zUfijK>enSNe5E>m^G5OLv2P0Ff14xtH^=J`@=so5-Zy#d{~6IYV@z%V{QvU1{y&}< zr|>u3&|U@kFV=r3Z~x1fRo#Td|F`wXBXhM@u$W6X`mPhdOQ#Czb494M!AVt#I;Oi` zLarJAYh?H;_f%i`rux-tn31IawPmpSId-kqGx3L0rSR{!<^_)ujPg&qwI^&%X)fqy zkimMl42LoIi?I7=%|o+@`^u8n>rLPl(WS6^gIzVRf<@INnzt%`+z>NWSkjpNxcwf! zR8~cfj@kB5-%>~xzIR=rQUNxg>edwaa z;FF8M0(%tnE;tL1tGa09Mcfw?Y=36TyU)s#&OuM6~ zZ~RT*wf4d&s_PGDcVLiseag}olzWFTt}4FGprNB$E)|#7=4bJ!-ghFpoDs2U1q1*_ zhDA4;cN$d)a)@PTP`J`^{;$Nx z{EBfHe3ZWkXIVzCYihil!97wP9MOV_y_F`NqVb0AhHY~ggq#OO@zzP8>$b}cm&I2I zUdI&z@g(H{wxGpxyh@<9qerbxd^(hoytnNmTIhIlMz{y7sZj54x|g$4Xpjo?UCY&} z;7Cxy*f%<$vw0+( z-lr9QbKj?CghNI!#n&LOJ4U!zoH$RkJu$*d^0EBjYmAkQa1y;F2*ll@j7&+@CT4aV z26X0+CHqdXr&N(von$}vJY}&@GDc7vUlFjpiJ<+e07^#;K&Ko%$rpjD$=#rvp_@ee zdn?;gh3S5B6Q^#YXAf9++^Bl3-m|e^PRG+4bOh^|IiC%o?J#dMi5UyGAcMgLfJj?{ zt{hvB^$_;cd!A&%eeCW-I=1^WLOM0 z+#?9z_UNK)xRp6~B(MhO9CLASV*yFfoT4@dCq&O?ni*YOkuF7j_{Dd68{GsU@&cOk z(*;iHw0@1uuM=5uyH-`z4CyzDr>~bHmBe9#|3a8RvqvtfoAdNE+Q_$^XK#@;CLm<` zczqN--L?Pk_+bU_1)ob6;cuNajm;6Tef&J<`Capb z-~R&%F{~Rmfl!eVe0HhGbVL_D-StWPc%F0CZdA>2Y&||ESP^k&$LCT^B< zKs~uU7O36^WBt?QP#mhodlF&zX^wb)VIhB04N0{XW^fi~k@QJPWq?LJDkH_EqS3tZ z`kuLGcAMbt!aMf$SJp^Lp=shnoT#1RU!C;M3D^9xJ?<_HaS2nHv3YqO;hQNo_4J>9 zw2X|RVBaj*f@E>J7KA}atY6Xd;7?X-2l`b(&VvmH4c3B+YA=l*ZKWd+iavb>C8hrh zMDs|dZ@zJ~a*tWlMeVi?mNT!Km6SPdGE zl>z_MU6wyc$?hm&V*PsAu@CgwaB!nGA)`bk4Cbhp@UK=f--lB7<-+rgvBxBXCq`dM zUzc=wXu4Z31??Wl7kC?A&n7c3Gu@|Y#=*gQ{a zedy7g4m|E$Gct}1RCDNOXRN4o_n4X&-Xk)$S5G@JtnnW#WizxO5wQ?`s?9qV%`EY% zVp;Iz2@z%0K6VtK@l=oS{&A$%-`0Bp?5jyT%1?v9Ym>g3>!`V3g7LJ$tgjnvk#vniL97XLAN zCan-5{Od;*d9{u%I86nuvYuVAI1Q6v?DwKZ(<@4*l|ugF^ryD7#Lbe(*xDS6-pQKk z1w|#B45qXZv@F1WdG-{lN*3ytI!0o5cNGi!GGSqD>`0q4YQe426t-2-T~$tmRBRXo zTF_IO7^c*P-h%lk!R~~4R{D{e))I^b@kT472gmsTQ`I>}R~B_$I;f~_Y}*yvHg0UQ z;#6$gww;Qtik(y{#*J;;N$2bC(fz*X-&tp`v41^dk8$=~b8<(}Kf27TIeB2v0)CH< z;GsA4BL=v1bksq+P&N#y1L8e@AM&!L8`1s_tq;VAy+&8V$l0f_{EfH(^TVjhCZFM2 z-}?JyWM|Cr%E_scie`*4`mN(v)aJ`lPyI% znL*-Bf?KuOGjgq`rC9R#^bSgb9&Fm^By`EJh1!h`Lz0B4@qn6&P)7LATSvQu%$GR? z6fU_#8J&|gB8VR@nLH*nY=s>la<%Yt@XIKMs0&5O==r}^Er5rQo4B6~_EbpVoUQ57 zMPE^kGnb>fTxc+jNWCZPkk`ApVvd&=)5{Zn(T2^0ru<&nZkvVMgROhKk?T3xh-4vhDIyV{C=!6$lP3*H&rnAFbE_d zWR0v*Pv7@x^x`DqYwbF8_s~3i-%EDnbZ(3z%w9HE@Hpm}=ddSy1Cv?ot%w;RD#$$E zM6)8A>jMKea%Eg})9R5bqJ!PTHQJ+~j;)6%hQ1hn(R}8)@o8+mkdeK?2Y4-0`+BOLj*6_bKmoSd8B)S8Ehx@fOSp`Rr433zxX=bxGB< zYawoaPjGo9Sp8=hA%7>cV*kKUkP6L#IhnlRrgpWoAPatcKrHUy zVGhr4w;deoSNTcScM~AUvQXu0L?;eq)tN3*?w!>JX|&p5-#=LtyFRw)q~hi+d0Hs; zxDl!MG~ruX(;D5!34Okf7!QWzY*|)HZucW--rPu>O*IPbq6HKB;e;q@Q1nTfa2J+5 zSrpbU1;*aj8LxBD<+VfO;51JUQ5p>j4jXP=fuZIK(v~a+-BEsmaddaWpJJh_q(M%< zPhMlYctE`Im5q|Wr%UqxSE|OvER_9^YHoc zU8V{adlt!r2q$*ZfkzX6QP#C>HWYi#)%ZE8vd&2o!8ySJ`(acQ9-jc-7BZ|k_beku zZ8YN#zF?1zKcSn#Ft>(yJ3|thai!7<5jcX&q_R75mVs8Niy%f}-y9K5YutdvNoU?9 zJZ@47@t%&3LCD1NkNUl$b^LFqC>##3lmdLnGuybTcJd7zQKthuRg1H(6&wYminHq~AnsQSo)EPnj+T|2-L66Yjt8*r~8t2ent_Vo$ zhS*pIz!QN*!|y%X2gI>}EWccpNG$l_%Xyn{rhGr31jZM4&jk39<9l#wKbl~zt(~uI znk2$0jrEj|{?*4&I|ZG6si|27i;oY)*bcaCY+J8SJa}jVuuv zoms@TcAr={GHaZVco}#HAGW9#*k5iOh00C5xPqr*s5YutHigc<90$p~wOAk$rnjQB zfHtu47W_kG54P8HFubSWJFoAto?iS32cY34{PYCO{XPY~b5;J%2ucS7+Hz`?LO@p0 zB_QV0pKmw9gx-@Ep`bb|8!sc~*_H(mJ@5Em>w?FILcEk`XkHy$HhX=bIb`pLy}q9pm;|h`pIIM z(e+3fxx0|FmN-N`ki`%x|>#QvAEOdU%LWT-X8Q17i42oe|rj+_`BBtXZu1`KRN%(Rw~j1ZG{mNT0LC+-){vF>38Z>{1ZVE|c>$ zNnV9(t66V~+!YAL5#?Yci%bwSl37jeNG8sr+vU-5-aP_!Z=nB)(N3KV;rR z@?9r^rlu)_puDf}yY^6v;%Io0nW@vdyZ0{gRX@!mG0NM3F3r2%T2QBG*d!Y2c3LB( ztp0LzP`f{sSp}#Y>&NFa{Q+Mc;agtMFW&H{k_>i)?%9p?GTQHOXI!QPW z7J+Wy{Z#E>O_Pj8C&78yktKoSA;S<-oNXX82M&!70@u@mGF_eqgsc)uOZU|ebp}o{ zS1Tkjk2K@IPO^0_Whm;+?>3tv)}mIn*6l0t38!&K+@LD4=bGAIhD*ZVa^A5$_1FMo z>qVBS>)|(VxTvgNkCijHhGU=5TBd?6qc}%S`%XtJJ0k82mxfun2$v)ZCZhzhOgFJp zGhKc}rFhX-^>Ia~`32KiW>)h<#LG9gbF^E3-h21c!0drAb|>(r2aq}dkIewOPmVew zFAo7}&2JH_^1W3KLI$xisnaX!%1~lA?SCxzy*?$Tt;GJ~@s_SPX^u{m&jsww>!?JC zTdQxq%Au9&Il=1O5l1Vq(H z3He8XL0!2L^x652YxitU@PKP6TO3mOQx#prQo|<6Y~~u3eAW|~V}79<%-NEdjKED~ zzwV1;ca#M?Rmcxg)DF?^4hugP4pdL_e$vYQH=mt=Bv*Dl-;sk5TvPaIzD*9f5sjuB~AMiCIh?l7P zYzm3p1DeqqTZn(8PFJ9sdUqlIaS~;^n~~(QG}3v*xenx_g%> zre_odEhI6$FV|yx8j<6=5{Vpdu%zR`!xAha1QF#$M&;s086^$$dbp|Lv#pc~>ZQcJ zlr5=~m`s`%+cy`Ekf|`=kHH-YibdV;Su;}{(COxQ3cF(Af$W! z!+(2_CzMMbqawj4(R;R|x!hEv&vR zlRhZb$HmfCGc_@k>IU*1zVChVGw=s|ep#XCLF0dUG4gOE=}YG@??gzqT{TA>aq?l8 z(+F#ktowr6rE#seJYeg@U5oN>_wGXbYS6VAEojqI?d{An z76HXQxU}aN3yha8%gG~Pv3*J2xRq5EGOmftyz)D)=T+Gu58 z!d-_Q%+BKeYm>@8ZJ@N-9HomxkMGgA(j5a2x`S~#CNtpu)(T{Z)Zf+RlhFRa>s<>c zEaJ{&W4;4a%0WQYkcxfgaQIG2eD74}Sk5S=>nslXG z7bUf9`IUYH&nm@7Vszap%$s+!b)kt&Js0cdMuwE2Afa6g#4qxB*&L>+mZ;-yHWM9N zvDve!72gWsPe9&9kT>sY11f7&mDAYz7g+zy<|jd>$g3IYPDB=SHZzOo+jpw!<AOD1k}Z;_`Q6HCd^aeLwAs{h09f8j|W1QxNY;Me^0pnJlBJBPT9}!z3{cS zyU5%(r-Flnv(^ms_XofRs;V|Ep49W}m5!`rEV`zzaG%$gVR&}PEAY}g@y&@V@e-Cb z^Q?%Aiv?-6z#1Wud~3LT^=zHE{oqVs`zB93iqRb>dhx(``RToS+qL>=j^P;yr|B+9 z+uY6O^4J&@axA%g8D=RVVE6p&EDyQV{0H0XQx`^5f)Mpa4;JbxCdhpzL!L>6qq--N zZO4=b*rO8^5Ue|)y%(yr8Pw~q8+9%tlpyj99puCH`tyNPh3=rX050dfJac5M4efw< zS{6a;BvgDf=3ICTN!Ac}+bbXAVIeXYT^tqF_K;|$`qqWEn3YECQ~~+20sc)K^Wl`q zM^M~#2EC{fcVx3%5|k0$*(}ZU?LM%Do?f(NTD(r71+S*n^eiCS@Opysg#Ek9Z|VhnaGk%qYTtq zl9n8X5{JYi<*ocjo_<@tez5j3dYqlRLEtq!^RxflrKe;Wco~MquRRUQ`B8u`2`>Vf zLT&xMSArZ3sh}Rp)N8Np_5Fnsu?}1#9np>w-_?R%ke9 zyf;-JFNe@r4gk<3>G^TLN?T$%;E;RB2+7I+(4KlS(n=3?uZ7yaplwnOHgp~7%{AQW zP+X&?Ha1|bDED9+q9kjcmew&r>!{!X*`EJGZXMJ~-Ht^No)>sWbv-eT|J!{~I>sgE z$o4^cKNq$*cW-eqO;*hW?%GNiU{aMQ@RB?{GOM8oO9o}^2S`!T(ZKR2;l;4DguYXI zER12$?e9kME^lmpFlC9Zh#?UZpJN>1|Ed)eb12d^f%u%R5Yjzd3j!qd1(_%)l zJE4z5IQur`d&0Bf(+L>hUGc@dtcmVoA=-B6weM1p5qNs@d;UZTXffWYSA#^WdbX zj$qV9*;J-|n7^)kMS4J(&y5$Qd@04CyzG|kD`sO&yL01lNx)K?Z+`R8YfUU(5BKF?^ zdK!Rvn*%+M;&y?T`Nt2fxnFGRXCnt?c|(4Av)gE=mm{SqbK4Y%Xj`k=?0N`(Vq^<0 zS6r@q+QFT>^NgVW4WW+s}>D~-gM%lxwvx6u)jEQ|!J!k<^>1Cn}{W6%An zhJ;3g2UDEt4b>Pu%d!5+xFuNBF%ncrS~F6`aQ_ZZd##_6nNRNtNGOJPx2k-wf3e80ZcC<&O(O!erQqlA1#d($Rl$ zVSnT6>$>5Y(6JZ<8&lD(fl>bh{=t%%;|XaD@RpJ*Dg{cBWoWe}QOsmLEeg_k0l?KiP`FfiMo&LY3Mn@R@zt{3|B$zZliHU zJ!Hz%5+t~z%3@)KBEois>M$hP~5@Z}9J4aE0z;b`)%> z1FP&=4bkv(fHhmCtv7Xnz`C}Pn;lhaH-;61V>S`}?$qWuA=EFI+0;)7q?Fxy_|`il zNYHP~6kq=3h#!DhQFZ3D*e||A5}h}IX<%k*EXuNcoF$gK=l~W&mDT>zMPw(TavYdq z!E=S{DVi~=-G<8nS=3Bv4BRRamvx;IxuG3rWhWXvblZ!QBBm)40fUh_!jfPUWDOdW zS0t#{Tq55iYPDD_?*(}TmO>^rE#R=-CwSTTrlMq$o1HF#_vj#zf2T>B(E;MSk)ZPF z%LQ;W2mz>;b}7OXWhxpyhi&A_+9Lt)QZtBqUGyC8^1`YtO?UE{H7tNzCf5*O-te!2JMFMkWgUn}o*TWLJx*=4mPNPy z9DoFQuR;{hOf38zOZlni72<*!>q6o)_J(e-rO#ZDD}W23ps^UeEOSDtn+na6+x(8`M_;UcPfbF@@}^E4yFn&2=PDcPFK>&_8Ap;E zoM@cmbce94*7>>Yq!4$#5_8I?@x7AUqK|tPwz*f|2LyP&d_Q4LmXOhSGEhC^v5h&X zYF;W?ecM?Y`TEhEggMGK!|vNhE+H`doQd+FKTE!PW!mX_;u^!l9kYFb7=_7cZ4Rkv zXYj!jet6jAWE?%+Bbs`m+{V|Bnr&Fy$e1=G^WCicq|szp+3M5#EQ#E<&v)E89oNbJ zdjyUi`e3cd|C~&Y9F8HsTX_?@%~LTg5j9v?SfVZBp-^7s;a7F7wb;j_Ag}C3gBjB@ zkMh&CKwM_v3<_cfdPzAoaLp;yo14*!G8~Sj_glTCJ%#)XkbrLO@C@t9`a|g#)zn!> z5=t+lr=e||U;n#$QqTHoV@9qXRx#$2=>|mqHcmUg$=i5);@vH6#u3$y z_ZK?zwxkf6jYp=T@O2PSv{*h(?zE@d9Y*FBd4o2iAoqX%K9LnxZ$P3%++bOX`SYv) z6e%nGuG>)W|{KB(tb(<|bGrVo06@Ksvr!&wbtEx9z@&Crt?Hwdt2 zoGKZPLz`xA*#RW+E&n{+c{x^*HU;@fO~$<29_5^iUB}Yme=MAGb+=-(f1t13DH7VS z$qF>u3EowvZr9}P&CO>Bi9VpT(*y zC1thjH0QX|y?&f)@{lE@YVy)P;`i0i-(|dRlEEJC8();&Y?pk_^L}at^Is~ZCy;n?=zJ4a!uk7Kp zR%gS9QWwn*VubD|NG?5RF#Go=`28OGFtckB^s0aP=FtEKeIAgOLdF}D6Ij#~3Ca@X zEy|;+eJvk^NXxq?S1D`AggKB-IsIg;*j}7n{)Or>cQ`i>o#gI-kgbh1kneghJ-C4d64Su{8)s;7Ftsd}d1K+S7$(pI0H|=7)8ryO#Z% zndCLWhHIGSlnx4&U{f{}qBIWb- ziH`klbuOFteYe;3vlg6OkuVYI$1TinQlz$cK?*!4Zw`jLFzgdCA+kP>8)0Kx4z6l- zE863EqNISS^={myRjHbmOR)$CO2R*%gf@mv+G@54CzeDls7K%?e0H6_2;F?Ek^Ekp z{X~H1G&I&i=h$Mm$!-kPW4l0lMt{Ly@MS{7R z78%QlxYOHWC->Y>yY_OWdhk>Po*fn1*Fq@at+kb#6r%bPEDviE;f($d`(32E-mwPd z!h^N$%$n7ZPd(CSO1;;6j>UV&>(-LP%y`d4{s=IWY(ixzH9FWIc2sp?DKe%KHhnZc zG(Mt{ZpM(Qx_9yBey}`h9ebh<8)hh zRPZci$RvN=NQIovie$}6FPZA&-zRnFgwNDFj+b72o&GqAx>x$RJD4;vw3yc~?Tm52 zShARbxVXZA!DbekxSP-~J&Pxs$X*k&I0`*1zT}T0%{etR%V=Koxh@Kt3hxY7sOh!+ z9EP&=?-tv?LrVE+?#p7m$VTVoj)?lM2I{&0=2+;q!E-!klXbJ%uZry{i~s#&-;o7m%~m3DLU3Y z$+=VO&f?S@l)ut+WSZV2F2QE=vcd?bcg&?Dk#BMpHo(3UEmX@uRN0jThEZ!kFaMWI zHJLDg1;vW7Xlx>v4tN&5h~9{BzPoc*Y4(cc1fkZt{Q(UP!!`p$kH&nbF)MvXNn{`Y zL3u-K$vzm#sgA=I@8Xh5=5mP2r+EaE@=I4c)@5(nO?1h!#G@zkb`{;KLRX~JVk0>K z_?C^WNan%K&il=&dB+*ek5cPzL8-jSAYZBjqIYcgZ{zgW_#XI@7VHizs*F(eEy>z; zCf>uwHv2J%8t2I+gNV&2D`Zh;bPqa}p)7#wIo=%Tonyz2V$Gy&(x+C}M%8#kWA82h=tzinU6SE?EXs1%31yrNS zm13yyu5Wrip&mbO>;@@W%!XhxEEY)QV8RS#nG)nQ9nNisWoSDx+vlnd3dtA_% z&pOzueHMh)6)-hiNgQB`ol@IA@a~6i4e&PBN_UO3 ze@U3>=B_HmAanT@BZd#NuAb633j9B$Ds-X0f$TVaoya2fUVoi(s?QugzSY04=`9~{ z2|np8msP?T8mU%i`S$Yw2}x#<|HjMTmb8QroVwMu;@M7!;usUkAiXweL2U0~w}(ap zFeX|hdyMTvgT}_lZ12~)$~E&L?-yb6`gYyy`>BYY_D1vOU;l=KNw!flusk`2vpJ@m z6$jOEG{lcuiiM8-Y<2*~ZikaW7HnirSTrV&I*1m_$@Btk9RD7%r7+DV`WL ze2?H5OB*;B1wU0y%2>iurd??c&ujONU0X7|C(cW%rz}C1^Q|qJLtd8H=6JJ7qBtJ; z;2Bh9dBxk+y}Ir}C&5~Q@DE;?7_7e$o>dtgtS*`|T9d26FTVQoD4pSrDPo?2FU^;l z^it9SL%)uIB{PX%>XL9O>tpSkN4`h?V}}H7IZJtktyx%l{)${YHp$7Ojm=8i4ZEDB zf&gAHN%x^Ja!c6QV(M;YJvW3C4V5~0qL&!ZZV-F~>xznVERpEU#qCTEp2$cZG6tKy zUl~Q?sF#VhHHL+(xlv0rKG;Xe(uoiglFT2P(0JDtJ}RJ1xkHJo=27$L<{NLK_KZ!; zG=Gr3m7yo{W0nJiPrSlzYec*IyU8f_y#pIMdSY7Jq4*y>!cCAf1N&cgkZbeggwt>i z&xwiu?1I?+pZ$>T z9&8Wy*gab``OzcCjYk#~)B$Eh0v4!3bM{Xj@}9{hGq zgSAC*ppVI&$yjj}N`aH0Me)wsh|;BLKG7~uQEcX@WQ<|VOz&ip3A|SG>u95VgWSN4 zb+MxPA0@N32828}lI1YJyxvo&Yt%v>RLOClje}dVOV$XMn9eHCPZW}01uu9-8&lS` z1ElÁ?vOFtt0&6WNXqLyd<_=XPg9X!NeM2I9GSrOI(mfE-!H}LcB5~gvUZN0PJ~1}c?C$V3Z?EW;GSjr3PJuukiAy@($P_d!20d_eNu7e zwK6sL?z1ko*`~HE3QRgH??vF9T(C@z=9Rc`%s+o%GWPn?!uc(Y^WRf;E!q#SB%aC` z8Afgo4zzeQN9SpLABos_B%?CtxzLzj?d!rf@J3H@7)$2lp_~x5^jT|rwJ+Z10yRwmnR(-b+I_Ax>&qR-j2+31@`V3DZ_+**Vk{jHBGA4cE97^%HKV=>O2mueQPZ=tfk7~cg{_qut7rV4+7PXhiJm`OXB<99rEq+^ zjz)5d(P*&5arOlAlxXnslsWz+o{x~3J-H+jeiky4>F|aaMbsM#Z$r3ut<>~l`LMg} zfM8*VCF2N(GFVWFt)7ZZ|K-E|pjT?S&}nlg@fO$wA?0`HIDe~wWzM=QQ^ZpJPd0vG z5F=A;z-xdfo}l~}aJr-rk?DG*{wL1=kHP;zDk;RD)I2GqxFhyInG28=`$AFnn_^DQ z7pwmdJ;MfM#X*~fc-1AP|6dd$`4#cSqiY0M#T>?eA})vdwWB;{;t9`x(Z_%i@Cl=> W?e&kv%i#>{>yZ`*iq(i12L3P1ljPq3 literal 0 HcmV?d00001 diff --git a/docs/framework_design.png b/docs/docs/pics/framework_design.png similarity index 100% rename from docs/framework_design.png rename to docs/docs/pics/framework_design.png diff --git a/docs/docs/pics/rl_workflow.png b/docs/docs/pics/rl_workflow.png new file mode 100644 index 0000000000000000000000000000000000000000..ea50231d9b4ad9c4ec50cc9db5aaef724f44c316 GIT binary patch literal 157308 zcmbTdWmFu|wyvE72=2k%-GX~?cXxM(;4}nx2<{Nv3GUFiJHe%KcXw_l`=0ad`}Z58 zt!h-)s3o=LeBb$0xT3rSA{;K9Q6F{0?!71KhQYW+3dD&O z2zc<|UZypQ3idwWz2O-%Ynx(y3A@e2`q^-SFe|_nOEWl+*(|d3J!n@So)6l&3?y;E zed?)4bir~QREaq5GgkInxYV;{u);IS`<=1U6C|o;g$mcmgQ6M#CW`_o)Lw5&Q^hYb zbkDzP?|$Q4pG#`gTaRHE^V`JleE_Rxlz;!CzYjL^0KV4l&V_PE9HN{|MU2ji58?uP z00l1CcwzAeLDEc7)65(Cz!yLGXy^^12TcBC24;*e`Q7}C6u8~rr;tGx>|8nq7%LxO z%M>=xmnm;U(b&8RC3E(}#ROY9q=kd6kogkIt-9q;Nl-t!rZ26Bk|^kP5xx&6Vs zvap&^*45c-=yD-$ddd>B+*xR^LRR-3(~jS}A@Q)C#!?is)Ay1R1^3#{p=JsBq9_*f z%ZM9ltvKo%w8eO<8VNH!6#k-iEpj&W^$e9%-;pv+HU%n9U?@XGBhOhuSP^PYRt1Jw zG78}vb`qs}@*c38zes6ai#6EDn_U))GSywbZ;@8*cPT$Piy-8R*F5D_=R-E-??d@H|Cg(p;F!q@I7wX{qvW%$npYDMGnz5E?a9#1 zIZpGcV2hzt-G26%nuZU4k1elIX>`|0z=3&a7u)q3?2H9c@OJ1sWa0(KsSOc@SeI86|mHL_Iu&#h&8 zaj(0&@m`hr)BI0qoB7{*$gAF8(LTY)vJh)`q+LX0H!DbuW&QDV1XMeub9T0*P<(d)bPe%) zQux;nj(B2Bpmh`%nu*p@e0vM3a+udKUz$%IV4zo!^JndQg=FaN+zkSl*fK)J5Mpn3 zOzch}@aqYU4*eik;sUGPvI8|502h|OiE8iqR`7mbS5CX3ru}%I0U0$BiRp+CXt(-_ zOZqTIpO^P!yYhTEZ$Ym4eM`&}yJWbwf|PO1GhiWn0r14tL*lIRlK(ZK8=1YLAXYpm zzgB(yH`FP^x=tQ%^GvvBjg<%lUFhlZ-m}y`}# zJ<;y+bGlC3V!1^k-))!Qox^QbLFtGMS9?N8LyA+73l9Mgh^{@&fKB+-)J?PT*nA?C z*a5scj|{P?S%2DVbGEye02&CfDuvIldKE;hwARO3-m_`&4_w3hNnTcr8?V*VS(d}2 zgQ25H-wAz)BzTtikslC-Ky3 z3~H{bPfNuxV)<4U5k}+cnlUrO8!HPzn?BbCTlzluyyH(wGVO%8@gu1zYb;5ZLWoA6 zyKUq-F!9aI%q(7?i0>K+g7Y09Rmxs@7M#8#eO&OKW)I*hF4G%v>nCV3ko3@ZkzN}T zR`@7exKcCtl>0zmbG#_aW%|9w>V3D5GaBbPH0|6nPinVI*ix*bg?#wmVp6G+u6{Dk zInh~-q3sPY-|XVdD=gvroB-GsPDQ=%IdNT+Xlj8CxHnN_(mAN$JCuo+j=riUcsksq z`m|hyp7NpR1h%J1P-Hf_$q2|5*5v_{dkNa8btK|{;l-QYc_M(-;bviI6sYVTs_MD^op4Kcp{|^-3iL8D58d(hJ*3ZL?UAI+eg#=3{ip zol$`ND_6E=QC9nfzNUYR)1Gtn447X@`qq*6nYR;DFr~`D2Zq9dE$20)N-p@a9=>d9 zS7Xm8Z8)RZ4ML>}*mGGI2s1v6{D_dS-b+oV8GDqSVY^f{z$YA-^DDOiPJQy+$taE? zdyo+r`}LZU%U*CA1sj2wuRF|!h_(s8#~n7(z0+tbIYdWoS%U5R%SD&i_j%{dl!nMn zr-w=WUajnuC80%*3&e4AO=!Y?Ajl7?(yW~^VI>Yh$0Ot|JZ1z<)$N7VO~Huq2b0c7 z^}24X{d@gGg-pJ@EvMAsWH56co|{0x;V+`jT%wKYq1>~&a%7_;1FrS;NdL>?%dh*c z)At`~L<-&TqAc{G8pq}(ORRxscqSK>etf8ekSYX=W|UgCNK|-{n4^w zuswDwMU%ry@1V2jfbgWVEiu7)_0~5sTo0W+jBmVeHV4w;mw%j68GQCar9BNS>j^}; z%TECest(T(rYXXhw-e}ESUCt6&IL}VrjDL*yWyOgnT5ZKA5B7le8ZM|ELX6zpD-+t z^tw(RMG*+={8Pk+jS$BAsG`iyGd$53JUn&9EL>sf%I}?&XnrVwy4|ygs-3>W)pc%D zD>SFD#UL64*0J-%V30@J7%$aaZoV(&n-$M728Uwv`NFJ0j9|I83shBKy7P}k=hW27 z+gm*1Ntj%h*=yDMqPHeow9~kftp=9cLu4>j8PdD?rHRQR;)hSbnn^G-C*AOACHV*V z@CV9r09a=qyHrC7ed+nBOr0uWbM&1I{r24>JcGL{pYYoZIe<{!5g+WoGj!>>u^p#q z3{*R}dw+~J29|^XEWeWI8Jz*SK$<;oji{KDL{j&rK*U!B?^~~V3&F0WX{s3zYl(qC z%5Ba!yHE-i)vrLz3(>ZmbuPLc$o`dOy)2C$mw$V5{i&K27keGp#0r4 za@4H~_}sLeMa*mJXR{jdJe+gUCMZPZH$3f$QfWuEF%elU^H(ld^5vN1ZhdmR?Rrs3GwftaTg6=-{D#J5cd_4KI1!;JmR;R@NJl19W>d}GmFSCkB3nS{3!m8Y z)V<~-->2-ywwlr(NrA5*ThEDb;+KPPW0gMU>c{B0!EP1D%#nJ52bQf<>Z;|~>$kya zi|ETB{uPcp+l;BREXgOG%*_48gr>tys)ci}a&jmu$3b*lbeo3`9Vd_JZ;uK&C zZ61glJce5gDJq;aHI3Xtw7gth7`k4OvPtK9x@`+9?KUZ4OH^&d5uaT0V2X0Jx3iX8{Xu0HL3 z?nhmf2>D=Yj)iido~)z>-Jgc93>{MGcDfQ?H5AF~n2n2yPD1I9lQ-}rmovJyxZ@e0 zjPSG9d|1c}?bhKHBwfpae0|x1oKBTp{}K4b(mP+3N+*L+xvb}pU)1#xl79{JCvWBj zM^YI?I5;h6LH{C{@z*)}Q~OeCr?}%~Vds{hM{?&3ALWe=D^D@^`z;_nbPY5zS~d}U zxf2=(imrV`CaXLvGffG@xy=tNyGkThofGM{#MT`r5W7H|YJPUdDAHjY(;rxF7NL$Jlhl4!)C=*H+`aSa3jJ7$&les`&gC>x= z2XkKkkHy#dDrls9(Q*T3<84j#t#m&7YmO3Q*cK0X{tWoJcgnaGu92zZds@qR#NP?# zNncTfFV6N~{q)kYM3z#7*0V{70IUg*T~kWm`EQoh@i+bP zCvhlks}X4j5~;6n3#$7QtjEUm?MUmJ{cQlIawKZ zBT^>f!t*lm2bS-X4|(6(*|0KjcI5m~kT#$tI8XHUFGPq#$$l=Oc)#KIvyVen&KpY^ z<>oeFCtsy_ey3bY7a80V7mf)_ZGDVhXnQOAChj2A5hgX6eSm7(fM{YH*DS}|YD2lD zJ#gEt^=)V+yliH{X{h|lT*sacuWe&tG4_z5i&cbCCQGa)lSocTS)Q!LZg3?#8=Hib zNJ|z?H;4tx&G92mR=yv1ERAcj&A-wAZ2#(nh6p>6z6~IrBMov$e|Frv;uq5!&1hqr zOme@P2K8RJtA(FZxaJQW*&IOLkw<2-(5Y9Z2;rAbVsxgM`jg-HJ8MGEMBCF%ab6Qw z^*@Pq>o|UPMS>JO3k)VQm~Gau#UGt6H&`hay^g;QihXxtGv_9_&c^I_ebEd zr7LdS9=^rs7By(2mbccLb9QD3Z08rrek$60Bo4j9qmWx0>5CZ~JtIjhz{`n-k<%g# ze=NP;Qi-}b6FSAiThcW3m$(!Ts{8VQFwXNS(d|{BH`S)NxpCTn3dhqt(2NOPhtnz= z_(*H91WHd{!Q=*64e*0Sx$$@nZ6Eu%6eEdV?9g_PnI*O|G<0=D`k9@tl1ncmxAts2iR%H{ zBs>QZHarYG;zKoN>`gZ`Hx z%RQEbyBz!LWJIaAd2(Yr+NCXJj~{-%SBPhf)PiGEAnnc}89xn8-r_LYSZ2qE-o$Os zntWj|%;K6>Y0R%XjY*-1S^`f+{+D7zV!*}K1T-xkf3|z=d8e^12bWHu|C3^#7H1qY zq1+_R9^uD0Xu^7DIuaGe!6}49gU)OcCIBnZwZBqSBZDWvs%L=FFJ8@|^^)5lFBaFi zGoHRtY*Yv948b`kW>`YxanWJUQ|Olp(Bp#wtKeCheU|A?T6A)UsO?%o1N5*8_b3&- zzL6cBCvx|2K{PseeYJ^5k=0GW;&M)>ePjdQAdn3D>5)ycSB=tP{O{6{Hdg@^AA;2| zeF~y}6txS~{)*7yGNrAO^O~a9R>amk=^G*Tyrx@do35QUP=rS(tiP~OTJach$l!GC zeU~-yY>nU2HiZ+)!LpaY;<3xBzW0exu#2YNwZd!mJF^oc%Al!a+p=Q<2<%>crQuMF za1yJ_q+q*FwRv7}u+fU!a$t}iV>0Vu6=lAhoV-x0eJ@A_vKL~veA+1lvIY_B@_GQM zw5efY{kr^nU!_P(gk@w5gbTKh((}@)=s4WJGv4Fyk>5Im)=+ z?y%wh`*t16w1rb&xSlhlLXu623%S&v?LLE@euGsK0g;MbQn{_SN4Ukzd1Z6(HM`Meriva)6T3)+0rk$=;8 z!AgKbMcc}B=I&2tIkpDdsFpBmY)j)in;iq5@=K-Eki#j)HdU;y=iW1Yd>sD!T{sqi>^b?1;m z-64@pra5L`Igsx;6^W#yqBE2gKiu(~Oe4F{{aqaN!I5MSA|IO?54Gbi+unzR@~Lk* zccAcbB{4tO=%Ec+<_)5l#ni-8bDIB)s_6W>Ifp zNWcnb(~XOp%nKMDzw(NseOrSq3fdPS5B(-3XkXi6@+X**RS-!#1s5eEX@5|?P)+X4 z>}Qk3kPK*S(8ur#gc%v>Bw0~AA-o>KTi7IESYY+57~0fMe(sCpL8nYS5$~B`JP~3N zK*ss<046j4sqUv6o?f^~F4&m@z=OUk^Xj8I$*oIRY8CeDktAv+RA#^i5QjBPj8at9 zK#46-cGB#C=29OJ)>sK?B*Ed;!OJu9O^yL5{J>bn?fESbz8N#I*ZVh0&Ax$^lO1a8 z^STJhyB&{zfZg(bPaO_p)DBdf@)so$Lz{Q`iv35)%eq7dr)UNb!J~n$* zpBwiy3GC`CIhQ|7)5aV4^b?@DN*?1u{~4VlSQmv$wp@c@0L$7-i!gxI!bxtLX+|s} zsgi{KEXfT&k~=}e>fQM#tO{9XzIDdt4vQn~W*3T2u7~Q>$)FN!rSIgZCdHG*a;p|h zS4;2MnZ*E-#Rr=BivnoOU59JM&yU_AJ~!Yh5wy6JO4HXGEGBEz{MqQq039vH~4?_JN*L^dP&xsmQ0i}wv)km^v&x4f}R79G@OOa|W|^t0#xu$YPU zvC}AMy8tSFcMQ+NSYJ~Kp+MFoYq@NtH_l1;h>-efoZ2x_?IDZqXZHX+v^o0DN;^Cv zpOv;Wwp^}!QVd-tEBiPzd${=8Fy>wNIxd2gD`}FWHPe3AHxK;8qL7NiR?%z@UgyX9 zir_SH1Oi1fRM@NaP7MAegDfK$PQ)_zL|gFK86LQ{3i?8jtfuQdGU6}sole#+A5C~F z95>tT^uPUg$V2vD?5*!Y)*v$RJ7BCyRHNw*|L!V0z2mWfOHlR#YGuCmF_H=Jv z$unKTQ0Z&&O1oKD2(W?rs$)KP?*@y@ea|3O`B|IsZT?@01ICDba|q?};QZ@lOMdmR zzk|ZaVCkPQ31M4h{;%Y{WVZzJwMcQ##;uPo!B6Idee8rgc@9iXLx1Ry#WrA~z5BBz zOzWcN~^MOQ^<1RDJX3OF58gDTGn&~)|X`6iE1KyK8^&~F=lbRn+8t8QBSuE6zA@Yoz z`q>g2X|mR1bfi^MTh-QW@p@hry_nC4V+C}g|4E_t`Wim{GsmQ~uwjgdTFW1G#NPO$ z^Vu>Nm(T%)?Z=7c24e+N) zYD9Ax`R&4_g{|0dl-a)qPxR}sQbQBB^wf`Lqf0%z(HaH+BfLNSqF(6>g9Atcy@JCf zV*&fBl~k%7YoIfrkW-C#t_ts?OVeaEOTcb_J6@F%`zSVJVtY38z-~Um)jdK~=JUV! zIpo?#$Q~1?N+*QM!Pu6rRH8+7IB3B=15HT*H_)l>9By>3^^NvXD)(^Gs;p#OuYqtk_LS=j24MuQR>(z;k!+ z;FEiR_oooW2Xry{7Sjl4KaKM0s}x9JKnKJ^>dJ^e`^>h;R1NZlaQNn#s-piO3ew2$q?@}n)Xs}gmr7j!Lmxy zaDuYN)1I1Ng3$2$FiVpG-}u<<2g{I`u;spFE@8)#sRMvX5Zpv@Xrp-E@ci{fVvmK} z=i^$+d!W#z-96IHdbW`17Bs$qKwq{rrSCf_qvSz2)?wJo$5=D6Q47?@@9)qwXB~?} zEXRTv4ZhQ~vGrCL`_8`#EASE51WowVffW_S_M67fHq09CTURCXN~iyEm8pd&&Uqv5 zz=?*@uO{?Cf-Db_ifxVQqxKM^9HMS3vr>ztH}e`m ztKrW6k(#D+e=jZ3>8F|x^qReEP$P~-$y^=o->mp%)z$}+z4e|#)S*r}7m zn%5Uzm|jziFrL@v^B=tYx+8$$z*Pf$^@#w{k-Wg4B(*cy2cIqj^tu6$&y?OF;iJ_D zF-+od&;4E;IXoLV%Jqj}7xlaEk2(!6rO9I*6_~XC5*yAj8e@cGrq|+W&*91J6nN-h zY8(F0HzOinL}?KEo9KJ)g63LM{v2zO<>Uztm0$PL*}0R&)WlxLZOLIr_|g#IhCm=X zckx49tf$>al1cqk__31XTZ!l2%!u!$`CK{pMk)NWfP2E**0i>8!b^(W5KC0gs4n-= z#V3lxe!~f~uHjkMp`97dYZfBxkl!rh%;#>KAVD5SbK*qRvvx#65!YRTW^rYMlZXO6 zcxKb%{T{EI^@yV0;3pMmUE2BZC1Oblk%S;x9IMkb6n0f~h+De<6g1!rWW4;FgIX?q zZ`^zk25WfKDJx<|)`RQbu^xO8jd^Hjmf)(Y&xo2S8lm3Mj#Tiua(wW$Sw87BtFC?!?@Dq}>{sdCX5Vz&F((e+e9 z1nOR@M_^xA%!T)4!VmY_tiR~wu#27R^mI{8jW(Q!-VYR*ewP0GfQp!H@6@BBZxyVyhU!4r#6H3o}D2Y2V&oa`ly20>BMk;)M3&%IF@2uoO z4x6KQxL-0x6m4E{$&53S&vF#?9rDT&?F3jGNEHcK3u;kU^1bk4l0!`?esUmEOgAo^ z9|?QAn9z5K%TVE`_=!zdcUk`7?~Hc%#l_fJVw%_SS$n zgDsz8yZ%O9fQq-zbYm+^z$(-7{Yh{`!hK}^!AU3N`%XIx9a$p_lD^6A(i0LrwB5-) zm1De~MgRWpWP_SED_dU6sibi=@!fq*uFS!qrG)ps6&F+M#`l}N{g>O4u!-OwOdtuh zOP<7e!k&~o$@qOwIgg0T8a@)m_I*y$sKmmq6Y3KO7l2rd|c<<+dMBAW|V{6yWl4bYLd_F*ip&Bi>iVme~?K% zjzMBmRj=i^=g?mEp3X6*$bwjX?Y_I8F1A*BP8|Q(b`hf=bQN`+l9bX+Czbj2ti-Q!r4zadGjrQb7Nb>*svE~zLHf30#~e0to6v>7-^O{%w$Tx2S# z+(aAfkVWlr->5m4C~rU?v{kRae&h~(3 z@e~rIs6is`%#LULs;M_I1t@*X~tMNLSV+;py;wKhH#0v`d^(I}9Y0slF8<}&*>3p<_DIpkYJS>aJsI`KR$ z^5*NJSisfjhw#5Z$6wbS9tr&O5{jUy#>5fkn%ksNie<; zB07sy*!?dusPCw+>%<_pqtwLtLwC&(z3-|ULPJ+rLJlKc;T@d}d0#mT=K7$r*FYRN zd;J>{x)ZXd2$hoM!U(H9c39Qrw0A%l1gm<*-EN7zdVhV;l?Cpizc2MLEDc?iXxx6L zO6nhwHE`jwM&E}Z28Z*j3Hbf168O^RoxZyggpRu*@H77IhG~MVQAc6+?)5i$^;2#$ zn5CC6mH2lausMl>2k4$b-hM&oFQsChFb+C2aBvZ96|-RT=bdPv z_sjSX&9Y%0s_E7f>4GCt#1_Szt2}=OHYdb0=sWJUP8~2+oc+j zo**LW_6#&0L&z ztDCaF-;hY4QG04=MaB*)BJ%V?tU$2yhTeGjo;!wSw75I^=gmTtW^-t#$nRd=MtJ_- zSoO&sW*bQQD8267qiAYy#=BXX@+wE?SMEztweHfq1&kCN@C^WrimS4H??KN@>dYU+ z4*(P!Hk_DlyZi-QsT-fa={Zs4a&$Cm!QZxtFCWw+Pnf7X<;+f9e5Q@i*+4R5U9RjF z$QacoDb%HN(cQoHJY=)AW7x_LIF}ly6JTQ4s<|hYZq9!{QrK9y<>rWWItLZLTmj_& z0jbQbX8P?AgNUcB!eU`}VSRaMm-3=aD=y z7J^z>blvOoBQjbsQx$dRLi8s;y!f*O36n%Y`}Y1Q#xJot4&HI**>$od1`kcr2dHmSNWc+d`ui)D5dH!&{r-<9?#?RY| zTr8ILy3SAi(%q}zf&VD5s2a>E+UI+UfL+XX!wPJ`CIclb21{yz@cTWmr1e%ADbi?co!J1 zBgL7cZiqQI;z$QCc(i{dhvYxqsHq7839w*J+Ym_MB$p#`0eH**SmO%UduN^6-10OL ziS4ZvElnn{7^#&(>Li!-6A!U{(#)m>v-6`jg@K1%pZtBt7?wTVXTRwShNpnxZSk)% zFvxyU(r-ocFPzPTwfF2-TVxDx3j{q~DxPE6g9Yyrx=)*wlnr|>h@=9%5n4KAPIiU4 z9IriJPkZ;R4DPC~p7h`+f3j>la(+3kFq@4N^2DYEseRL*hf`oSU9mOHa`>M0%(qX` z68#Tv1|%a{DQFmq=X=Xq#PWf?Fn&0?^}yMCEN@2N-!lgQQortfA7qT+TgMpOLyM&y z)`+h8l<0QEjM|0`@~{RWunY7tZVID5(g91e*f{W;-fyDK@5hctxNm3LjcNNpBaJ1Y zn_zX1fUL{BN=pWYw?dSma$(JlEe-Sxl!o;NaA@Y5wRN^rOw*LoWrg%hsDCvvITcFn zy~&A1nQ%n!C?f`UEEt##Vdjh$Bnr`ZOdakAM0VF0!Me<%dtmzx<3+8yf93Ygg9x{o zM7N%0B9-|F-H3O@RNmyf(8FpfU$uohsOgE#1lm_Pvw;dcenFUTVM=bM)M=Y5!^WoG ze7^X2)2Lo&;SJ?_7x}Pd$bjVQgnGww=Y5&DNGu#Of43B2BV#KA_#oy3NGoOs6C@=& zj8zPf8>>d<1aXn%C$xfu_pwPaOMbojjFHYz%yZR9E!E0+-e(?8Ib0|eN&Uc!X5>n- z3VMy;>I;CqA&AgzfYy?(dxct2Xj&_y>9oXJ)UGDMK_;K4v(tglb0vQL-9bp6Sv9V5Y zyGf=mhLe4{HxJb?9@%ea>}kG)1^j2U;WWb0r@4} zd|AAo{j4n)+FSWE*tVP@)d}sC6-E!Yd20UxW?mCh2126#UV1j3@M7rM(e3}eb^cNZH3^pe7$v*ib-3%ny$*{i_(Byp&gcu6_PG}D?)ucOB>{E^ zSAt1zmI#HjGMCFUk#S}X`a1tZ2e<7<3MgHELxWh8i$~^ltNd(a6xR@a&vtmexjmrFOV8)|gqk>i?kYXY(b==@R@% z1Pz{~%IASl{l%U0YPzyKDx{&v?(@s;?9;9D?R%A$_|J#^Kj8f$3-w}`;d~L274lfY zWdW+pJ2$ok^)QIW(t0?=p`zO-JTuTZTlr!^)oja#R_1`*MqD+-XZEY16)=0sku-y} zLs!awQN7>D!8Uq2miY%$i}(0hlY*Xy>H0V)|G1!bOrhK5IZk!YM~EvxV36j!YnGu| z8y2Fgw;}I+!kZ4;Kf?76psw%68?%ID?69K5^(0;&STMAsqZ=vP<@bnC(dPE6v5F8e zx^1fVTqLv2O=IP^skrNe=J1sq%7RDR6J|$6RKN4tV(8klCbz*|IX>Bj-F1L3^>b7< znI34z!N{+N+nb8V^DX8-Wb-z3lx%5q`3aV7rmw`~o$9W1CZ|XJ0rK>~b0Qa#gWL(0 z6pYLLX4`i-0%%^tW4?jq-T&o{X-}P57DHo`09w`w#;T{+#g4Vd(aA63GKPY2g^YrKG65fn}qvA_27kG|TTYF3L745+`s zg(&iM#r_?J<5-}63I6~uZgACf59Ndxs@$wt71_MB<2t2z!72OpvZ`vE%dHy9HQHxF zo+qz=q3K-p__q}Ll}&-~?l_A zh3E>@=%3V8{R7Z7?Z~-y_W~SMfZr?w#Wb-|qYK1a+Fel*+KpA=(PXv`WJUcs7JjrW-!ceVmrz!j z)l4KwW~O)u68uH&pI+IROlh@;-{!5SSijN%il8!NBszQ3m;lP1j~xX+K5sRj_NG{L z_J)xhH^^Q~p@@Fv4zr0&y*ZIu9WrI@nndny)Q=N)#DgG2TiM^5+z3Bo;+y3=(ctm1-1&;ZDbUn(^#c!MlWk9QKzrI z!5A1K*m_65T~$2>ADs$hcIu=!rfRS3%s2lSmA<&hYLW~!SUZyHDtC*Pz& z?{p)3wz?9;i2+^q;48n*i$H=eIoU=U5HDi~bYY{y`7(LMpKBQ>;bixKyxQpqXAyd4 zB0u3v4*9zQ4XTLeW>r4RP>d9rd2uZ$enjLVYNC$=x~TQel1?!V*91khN?A>3- zWON72F_OrIrSSf3C|p?zE^T|fptM2{Y#m-fO)X-4uilr_p#oF6ivi>}=qIjtM)q96 z{+AwXCXBVKj%O0v)5odOULj-0nKkjdbWN}|dVn7j=V#PGBzI#fz@F(L%wAWjshp{* zg2qy;5rNtLsgsPzRNNQHQ?Tu#hlqXDt` z_SiEQJU?<;O__)_iQa?@#+nnaw)#=~epfIVxYiFm&HWD@%fb@CsT8(qf+Izbb^s($X){eYKNIRfGE#EgJ7xhx~}7+K%hilQ=9`Tl+L zw}}ypldETHnaerCM^H-R;{$r&jU(_ZOvu@6HP_E0nr}}#AQYMh0hCkdSM&Yu3 z*fVyr^nJ3EOnV;pR44TbE2in5GU|5g`WEm*8a?)hpUQ5q^gqcVBG_dVH`tG7>-z>b zIEh5NcVdxxLukI;<)wPRYFgf9x^+x)`$lT=%Qx8Dan8>0bPH}kS--Q!m(!7)hcJnt zyGwOTH(d{BXy+1#T-{V)PKOLvJliGx;t&J+&fPa9)xh3pcM+=YD7(wj6Pp1fs~E-V z!$*(~#xGN!nI<@stATbw{nrvR{7ViZg1!13gheCBj%GLEH|`KNP-$Qe)i2<) zsS(j+OOM+z_5|RRKc`!uK@XXBtrGRoA1TV&6AsBsI&p}WIQPAxdO1^=8iqOthgLh@ zk4@Rs^y9lr4d~!3g;c{CWxZOzh zXC-93FPZ*QnC?aQuj77hi^`fzcsWgJ+!Ak>B4MuWd~SMwUpd z%8|t1GNTd62ro2CeoN{f5->7U_$8QkV!P^y#lkAirpyQ)_~&^L69H=1@;0jU54Vm_ zm~SE6V$low!yIHW0v5p`=fdSzL60W{`NO0~Z+@=(ON|GIT211p?zJHX!MS{*Z~;mW z)BJ0o5zl7vCv(7{(!`UV)Uz6&I-Rn7Hsq(Jrtsy@LC>5wIiT3GE$i=(7~3*cuAQHD zGdlHZ$cuc%p>=j_8t4GiK?&V1CAV_E1wPkwCDv^p-oMYh?hMf@B>vazaef)dOv z4G0Sb$0fgCH-KQ`173p~3MI>w66|a;I(D|wuu6s8S+CWgmTrbDBl{AnBh9k%rzX>H zRi{h}015G?tR3aSo}2@lU3&tKUY-ldC+6@+lJD~r(2{7?j`w_SmeI%U=WI36rjHWX z(xnomau`Hi#TM`WM3fKc!zO5Sb+VthY2rwev23{_auY!^(?EEJi?Dl{fQ1Fj%4h3_ zLy9p3d~vBP`0cVKK#%XeSH!m@Knsq4!zdUpHg!q!TG_pS|8%a!d(nh8+^X;-lSzeL zYiB-LUPtmL3;EH@jW?qFZo6Hu#o^-X(!k7mklVTF3}~xw`S~wB{sfyaY>@dEYDb5Q z09P(a`^;%G52Cc%^=uc5ab=|mSK#4jr&*ZOZs=Q*g$Q%}>*B9Vk8oMgceq0<6}0(1 zewp8rkE25sX@7q%cD*WRcQDiL6ptIgaWhz%s=rB#!vE+Dq0I*tzAqf3=jey;;Rh2R zU&O2}Fl1*&dsH7*pDFTsa@5n9YcUqq1^w62)9z+Xz;_N^ct#>?^xR3A z|6$8qX^K!d?m2=<^LucHiTW8u#OSm?(Zeb^E3O?^jt-UNhQMu9-A$K%^_4(4Ew)c9 zZ_I^pOZ%fNuP(OAXp^o1jZyBBG5^LlM69BEU%ZvY6EI|al&l@f9o3Sqq^UT;;m2by z>G^O`DR%TJCKsxgZAJ0ot7a&J979A>lwMJ6H7H;h5HV(U# z+ACL8cUA2VQ3(yiykQMO+hej(8FD&enGJWh(&n4V1RWZ;w0*M7tnUn!pH#)y56WKU z#O=?QcgaB_r~RRPpRx5~kD|3Ih&an1E>&NIP`f|dO&em$=HFQ?-)b#=`H?^6g`K!U#9LPYG=+; zdl>BeK2YK74NmVqP<`UY_aDPI%XUFs_|8#~o>29IsqVl^x35rnHbbnVUvMk*0A}2? zk(1=YbD>0h5=y=3X?n#ZDwwQ&1p~h>_#9Ie=XS!P2@7s1&Y7zcJSmACBs~YnZzS@; zKpJbhzcv2(lef+o{M_8?hc~8{brDh^=PGdI12wW>nSl}@|HrbPZhK3c1CY=?vfzE- znn*j!8M4aMum>BnDt|M0z~$_l03>RXomo+h{A*~(F=Wp5JuKoEH)y8h&;+(nr}@+a z`ECd(T2ILrc#?B8Zg&UC5b#gst()NJSUaNY|1hfL*A3gzj++ab>_k^a+>Z4y+;E(C z;ZaZ~MKfTnSMX#AR&u-|m{eskUy{|G8k9UqqZ6(VME~OJ+|#x{{CV9w^V^9u_3Pef z^|%Mfdt;Di?pZS3DK?+?j)8dRR8;Nom}^I^W-89Ftwx6Rlb7vmm6p8MwsMil5B1lt zg!j3p9?rO^vNsL+L507qj51tsz)@Y@OJMx}(DqhAajjjrZUmR$?(Xg$JXmmd3-0d0 z9fA``&_Hl^Yuw!}IKkch%+8X3?R|CXRGq56h1oqn9b>%jGdzabxOZS)0;@xymlUg; zW@wP&WQVxkn!qn6y7c^`(N<+Jzk+;r&-^e@jvQna8h1ARzVVYrXyersJxazJrx51w zZ+52y_}n5U&cy!b*&Wo7ZH zfU2^y?v}7XatvT>CX~%to%*J;<*(LK10@+~L19d76aHgI%MMRu136X2Hw`TiydPO{ zWdGTIFe5D%-q4P)lbCf!a_OtCJq+Hj>Z=^9w42ssh(~mtGT}9(`>+CjdabsV= zCDlu5l57W5gegdY%e*`u#_7FAV$E{ZiN=y|J<{28b8L}H zs0i%6d}uM<8K}8`_dEoBjPFYyT=+NygnYwZsnog3`>7s&;zrM@kP9g<3`+uGbY)hOuJHjACGL?yPzq45VXJGsKRxX`_M%|1}8+A6cG8XkmeW>D`kI3tJ zKlR64!q-)LCg58#J-z$6{cSLWDWOS4^tp9Z2cy_5TBce@m9s9Vr(b;{+Ip~;v$ZG8 zHC>Gu5IUU*x`X&)Ge&*P3)cJ<-1SBZoR*WtQgwzH5P3IcS?`Z%kc@va2TrslnA0zX z6>&nhkfELR$3yjwEOQ=xjd=~VuWs_r*ftW|V_AWHFjl|MLsZ8grFq$Y2XQVWRWO-A%^+7+|4muCA=or zvoz*BN|O*rP&EY7kHzhSnYLNoh~)RR_{ezvo=#6IFjr42cFifHJ-p2z=jwHz!Xhec zpNJgotbBy{YY;mX6xmp5$}DNVPiB{I4pN8g3N;hopOsCSu7hPAV)H>@Q1Kcjm1w`) z({~ln7$~vr#bk$Jr{lMd>;{me8)rufU-}#yw=#v-q#tH{3LDQ`>kw^nE<9v@k@uQUog?>@)snNJMG z>^y6q`A;ZFAo&!qR-%ZZ)0wIJ(IDbPVm5H`SU=gew*~XwW(_N&mK=@qD?Tz$ zB13YYI4NSP&`7Hyvjwl@tI3FGISwBW;XE$H7X7*qP*ao&25hY{S~J8gg#3T6*S% z+{+v_yUHyci(pr>lzdJS)T&sqIGRjz<7l6eDphnU+NMOExw5tjIg@8H@;ihMETwL? z!NB*(e8(BC4pl*or`9z>zjH@Zn!R~K7BQ?e&mDY{T=PH!+gs=Ojuml6(>!l!_xk#x z=byS>KW@R(3@TYU3}2Y4;#h4xQu^r|YJRVBkRhd%D54MB=4`ayz$GI(VnXtf40q)l zkR*^9zU&(GkqQ?aAnb5^%timXq=?>zJ{9ph!k0kQR(XiLjKbPF@YHPyV=S(rQWt@{ zQeO4FO3qCF-HzNL+f0`Pf`>813*(%lOdr3*_p1$fEoHLw1oo-eF_(QG@~2cZZM^&% zJMp#nrVDJ<>gx|U<)4sm_k>_*aJK-o28upbnsSCzu`dQaP>t&-MH!jzxYrykVBYhm za4iSKoEb_P0Y0kO37|hEzvR{bY_4kbBEQFigp<{gX zizN+Ivv{EG;Wv+3@RM1x{CjzC?!v61=JgnLzp(E;YoQDpA8c_gXpzk%ipLxm9grzw zX)aH}LgWM>w*?Nxlrz_esM(6#zA!tVQST3z=iy=^cRNv~#7 zhDMtFnk&(}W)aE{;!}G3)Or+Y~fB92nZZ z&|5po*!`Xk(-Jw9a!mc6fpgnjSAGA%^;LRz_3C(SxLiD7Pau~ZPPiz$`K^}vx!wnj z?k!@Z{M}L3Vop~M>J#&yV@LIf9rHktvH&Vo0Dc~0_Grl~FKT8s{Im6bj_KKR}*58b<2N-J`F_zmZKp_RFsi{`Z!&v%&C z`}c@xG+{o4ejXEa#rQs%WBLTHQ^SV_CfdZGHFkh&q3%kjh`Dc3 z)Ja{07r0hzS~YC#L;|LRGyX5nRJm-Q!iP7?s-<+`v5vW8n`tIs8})yp6-3%(Tqr`_ zd56rfX5xyM+hTmk$u8{me=q(0NAVPJ;ezbz(~hqDgu)u)t};fB z_IH=GgG^hp)n-=j1^OIzC;Q{C+7d)H0}$7bmFotwHMiJ2mUT5rn1=~%vyj4jv|H5& z|3mn^H%C63g)X>OR@?JSH}5I@B2#e${TV2~kFPk2Ws@7e$9{lI)=PrC1bkjbHX1WD zoLt4A&%ctEHx!QNS`Z23ma}%miIM-L<>ZnvgQX} zAx&eHNxIZCT@|29x?+P36_$23>S8)Wq7T|M4h+Bq&8yQCYQBK#p^9Xj>_zUzD zrOHQ>&Xs0Z;U(*>mbL9s>s7bBKsWnWGZg`jyv}Kuz@iwo9NH+?zd{w>e-1m zUdl8@`?Z5*ptE+++Wx%%zG%wnFUg64k1fNs z=6s(*e|_DOAtqs9wABj|a!0BM@dXW{aum03xTcYPWP-1n(RSLO7=Dj!!sL72Y4 z?V{ye@i(spj9<-{upNEOobhj#;X9 zO5J_H?n@BESfH){qu3XO#)1*-@xP$$%O$;8vt7?QlF;DyE3lpf%pZsEWi)wkoBVp+ z>D!@-PYSgg=AUuXp?MuV61|?JB_=50>vEUM&j}2no{x_XPk2o&!*mvj9!i&A;N2?E zgwK{r+4s4rf8fg9rE3-L3T1{Uml(ML)?4BYXrl67ANeLfy~>Bus#t$y2DZ>Ev?{5mui zi9|56cgd?GS^0D<-{U zHK9CG>LWd|@h*Qw@*8NrX*|)Loau(u0JwIGIpU0A-=)eUc0t0j4L9rF-o(DE(}SlF zKspe5>Qm4ytfu;)O&BFR$VF!or`2^XRfR4ahFc_cUklujAk_~r(barEx3P#n!O61- zp3?Bf+^o&xwJ>1ilw9H8hx*)daDPuMmz{0&*!`)Q;BU^s^~TJY0$0~9@1eR4xhdv2 zL|Vm7N94WL2ourImHqMPjUxHq8)?=vX|(-}RODqS#VIWjR*Lh)$b_LUX?QUS1QqO@ zS#5xcAT?4!>xvQ=m!TrVx!q?Z_RK!N3sVMAOqEZ?)alMB24^6}_avED@Y*xThY}HUpHeg};wg~8Sx(sZjM%tx z`!$3sSfk~?)!%g=>sq<4L$CV!(?&vE6dHf-xGG$P@cpfmT9(?oLYoWHo?A{H20g~3 zpZ1zVv|kv^-=+wP8H@AK;tF-=)yxJq5+lwJnkh8q*od1cr0mbBXa(QTpR9x>gKjBl zJBz}#{VP~YO68^1ZIJcao!4+j7wG2|wfiQm#j5iV2>>BTX2n@Qsvgx-OLR^5`Q-RK z9GI(?7$C`oGVS>xUa+^ZaVk>ueDxaid@nL?P;?lV$xAlVk8j?+H!q*<#xkbN)hD?V$Ng5LV%Qw-Jr2^XMFm{n9%mqp&-$VZjFzuC@{UhkK$b<~ z(KCAc245GUx|S1E_P8P~*xXZqk77o^3~rSh%UN{?dQoqlQ)r2I{2#aW7rw=K^sZ2FV8GXaJi zLpZ`91D(cq=<=?aE5U0(E0wTQcp>WC?AK)?M#$&f%@J}Nkg{D~n}-2y(+2o+Z{X)ny`n=Ovtmy#*iDqE6tG6BY+&whQ;f3Mtz5MROY8rkJ1Yg6|mP5bxwGoxhe#EcVoEYuX{KX+8v~SZ# zS7Nt=>D_A}XcjU)^P@m}8#m{S`u_5FVGHq$#YZde01d)qVIA$R{(#Wl&LHCRkmXrN z!Ls*YXSYX0+{#DMzQiyr3}dkQpnIR3&4IxQfQaqAq5MHM<((gC+oS`?_)GLY=nO%i z&V?oYp+bjU4nO1dA(2DABAx|cswfa=q}9tCi-5J^)VRm4182A=o~6rs6shYuLW=EH zE5Pn7svMQQv5&okTuO&bjfuR>^i8-`xAY(Hf*8K9_wGH$;}?5~I7$CUmC^R(t33rC+W| zy!^A{fdIOtxP_h?_+J6v566IJs|(B_jNw!X>%;|dW_9?!w!Psxg<{2Pf)U9S5AvZ> z7+BJrAZ$3kMWh020$x-f5^*`fkkQepQ+lf)tBpG{ zO~oy~97f=)MRYO@=$NynD;TUG&QZtGr!yrpv)N-)R6bFs(>=U1?#s@cCX=@U3=;Ow z1gY?{%v_B*OqG`!fT~=L6BCQzvztUjQ1dCDjvrsl>${gBC}8mRga3>6*1*I>Y4agu zFn!#9kGSXw>bYv~dc=2%eSB@VgU6hrwq9CfGjI3;OA?6LCgP6&>~A-*q!zdVNN&ts z49V#m^~N_Hi0FeMsS6xPsaGsfVq4X$Z9x%zlch9Sug_Qk0?PlaG4xb%Qc*fiT@$9F z+D>R%o*`$;TiK_~o?5sQ=6mf@p?Gr=l+lqsY)=)*sS4w35-@K_v+3qRo1&NUWkFJB zXb20OxmgBm)}r!1#$b$GfLZjv|BVclD%t;lQZ!;(;8~2D#P=~vsOFnsPG;w7t(@UP zvbT*W81}0h;O_f>m|K7$(ELBXBj9^6f&H(&1$aap1$e3a=UHOO|F8ZD%l~Qo5Z}At zueL|>3Dpa2w8rubA*yWtJ@{XT$)JF6I1j`F7;hdP65=_nma1rdLqht*G3xlsueNwW+iLi`iDil!GI-WKWF=+n9e6nyN6F_w6mgzzE<4MB%(dyHvf1(srftkmjJZ^e<3DKdRM!Aaw6g`p5*k{Dm#35@3nIZ2 z_|7o$0{wTz!43_05xJzQfp{c)$}Yf7{?Uy+O^!L~$p7Lxfjv2K=&LIPiJKGqkrAJ_ zCnWt}=!B}1G}_p;qOx|**e^F^Dp=LtDnOFsJrxn4M!q5tS5C+98idmX@7porv!!j~ z`t~31h5i|n?!}PqT8Lz3E&>10_3d8@@J>^=7ls=8S(?C-zmB~Uhr%&@H{~ZNqA?Or zjr0Q}Df#YsXaMk&9YOU})l!u?n9wm?KUKYRr@G3p=9wCOud_MqKB3m#HYEOQmM&Gr z#`6M3SAZ1m_ntWSS3v#2zwn!J$IP>{G8uR=#FocM~&EO2T(j-|9{b|bgzXjRCUh>W% znh%%g&TN`WNtkA_1u0Lcr(w_V!wnpF z-!uul9yiqe`?vQCjj8LPJtJj2@#AiP)UeLX8tNqI1#N9;ZJ|8sABsX^kpWO}$_6K2 zp*HhhZvAedwsMQuqa^cgaBA>aZw?dz%sWIf#c;HK z-(6Y4|HaKViFjc--gZ2sSq6pY}L_ ze93ePgEY)~bA#)#@jP#A$A2Wx9*JtyljuoN%R*ZcfvGp7iE5y?2P}NeeviDfYIJw? zMZ#nKC6{E7aj>^ceB&OcKKOekcY+OA$xpu;wwhxbqcG5Vjb4?x6)DncDOu>?ZhSQ2 zIc*{W>yM%K^};{*{~$^sAX+zgFmztD;WoGEc|Mjo4gSQ2<*(GQ^s37kL1QQvLhdENe|V@eh2Vpeb2H+ecB>N>NB_+GHCbF%;5u?w$& zx-2S&16alSFZjAt&Euvo$k6GlBF20Vt$w5=Cjjky#n2cRtGLHey-`JcFSz;iu-yNz zlVCgh3C)r|nz!;-4bH5;RW=O5_&krSs>$w`pig-3XX5(3Tix=ag#Sa7!7R?Sr>-}Cx2^aHl*fiB1v@* zkxVtEe3djmMPpA?>Da#$$Z*rcN?Dcs>VbuCk!zVxru|y>*iQ#$0+~03;|SpOffZ-D z6C!ja4Xha|i*V~BUle{0^7lfQ8Ipg~zFk_r&p`i@{pBjN*zbc)V(rUB`M8lDl9Mjd zqY)^zub`SRKfVh)3^oPln#d5hLqC+o`AS&>;4}Rr;6@kG0p#0BSDMNMp0>rB`2#lH zsoRS=Ab7kU*N*+FKZlwu3gL6WnCG_!R(pb%@&jFV7w?gM+t=$}f|oZ)8zDeXgyMHK zT*nf9DfN!nTzX>UpwR8U^>iN`LT{AycGG$7x`<5|=Vg}Mob0zTR&G<%0uMH)RNU{5 z6q>2RJVnR<=_?(r#4UvIKu1XcPmuQ^ zA-91N?W{wue5=$VaE5{4HchD%J&gFk!Cj$MMD2N+BAk*+&9ur zVY&pG74+Kh`@#TgRiST&$<~M|1O985hrAjiRlT0t2I8DgI91$}fg`wn3BVBOI+VhX z96Rj!IeP{(Ttpi)SFjUHoC#cjoNB2!Pj6UXXXsNj{NoGwy zmL7_MAmolK@fTeLAu5d63#J#&^Gk1I3ei=s9^H*LRP}3NO;r&q=J6X2@kBK?Z*594 zSVzs;2J%h@HrG9;8;>b!piPX4UJJ zyi@omkDd(0qk5hzA`gT=yjd3+)?a5*#Ypr-k!0@YI8)D2JGZ9tu zXCTUk-dtB(Cc#LUSb)deskLd=K){FD5`t3dSD$Napd|WeWa{|9#!d-u9VcX8yUBsB zM%bZ>A)`Ujxmj$!dVMD2*pmF$H0uw-I@Q?a7ZKIZDt;jKPPm6(A0BwGGqqgvb5$oF zqxhV*EU%-zb@I3R#q%4|^}f`YJBrOe)F}!t$ zT^=&%QS(sdRJs?J?_(bulVJumOgGY)xdN%qv5nU>eBt6TRxP<1X@gGIsy68m@=!c# z42_Ss?Ch7^fK1`V(r%;YTft1G7xlI5YzO6VWWqkV+8WkY+nAsDV>)o$6>>w1ti7b2 z&+-Zk@I@3}_V)>xsqhCAUPECZ8DH{d*jfKWeZRl2>Q-pyjmhnR7MN5S=Dx+AuW8)k zAgFam+1t`e%p%Rpl&OrOi__QGw#I(wwOr^f*iaNMK|l|{Ky$lybIyjKWH6X^u!MCI zO25DQv@F+_yiQ>|dJMO6zs76gbQoeto^2Avc2D~> zH2k0qpqmn6vLn=OAd8&{vInSfZdzd|`7zNk(T$wO7-s1b`z6ERD+>>}$O&0TYhS<{ z1uA2OwW$XiN7c2(2NjId;niq4NXix?B8fKrITH!O-|b)hI@HRr9C%|>zzvtaBB&^> zA%b6!EZ|S3wTjf1I_WzYeZfKQ-(Ygj+51}Ah4aPxA}^3=;Q5ud(mnZKSO2pr(G9IP zRw_DZPb1&Nj29$SYI)WOzJ^J@u-TWlgj>ZCFEcFTN;2i-i?rCOKXLD$X8u$SW&}CJNv5WMUaeG)5*z$}sLjuFzcZM9 z0?Wo_uv+yFQrnZIDvO%k@XgA#95wSvFjXQw?AD^6o{2-W5Z_IGzS9qm;2GwV5A?tM zjja?vskvE-Y2djoX{OT!474lu)xIwGigZBHNeuxttD)Y@Qd?ZD`3q+{dlORYUORmI zYaO^x08ysDDqv3rnszlJO-=e4?}lr4iXtpnEj)1+N%Cs5IM`t3mi=O`c9xwnv**ia z77M@iG{86zI4)^5@SzI%9c|;zDyglb{Rey*`{Si&pzWB*xP8!P7{j2|#{6rkaTQv7 z5G;~b!^}6^t$E-oms42n^C8G? zqyT)4EYlC1(u3FSh0gtiWOL8eIva}sqhYm^tM%Kp;MZI21mpnwBZE7R+O6f^j^Z(n z+VdN60c=}uga|#~wnpRWqxv(T4ROSfYSp@bRh&EELWWJG|A+130$czLk|M2ix-Hcj z%kvvB+<0eHt+t_zx*G~u}d0V6132uE4iLt?1KSH za9sTw!oD3`aN|$+G*xtd8wWYMPK#@oF)rT|@zU(c`&ZhXI__$%-w@-XUr|b?nBYAS z>^P`fP$acG=6~0Bh+e%ZtESQV9Xw(mZNYObPa(9SMR>n*NGy1N%9}A7Y=5y>Z-4EL z(ATQTh>L(j(};c|xO2D(dqP*-CQ#w^(TGR2&g?G@*b;fV*+$FbV_- ztvCYl%kJntG|Ay#NrLP;J0R--bcOz=Fk%DuU6ZgOD{g5Mb`rEuZ`^)?DuZ1$P=>eY zU9mt|u*?J{k^Qw?D);|PDpD}Qc$k2Z%1*B_%=8S<Mm*MO+Rvq$CXlS zfK2b~j-@|PnK{^yyxMukHWZp{myV-$?1d4Bm|5|1;hmC2g9)jCXuIIVJ#Z7FxY1Uq z)KhD#!H)!%LjlhI)S(napWdg%z@p%&(nZ1R{AEnN!`mIEu_RYk{Sw ztz-N_@s%88J_0EvEj`Hj-rm#k#4GYye%yQ>RZ@!aiAvo;?wSd(TM{T&A{en#~vtYVYZmc4M`G@MJxn(rA1E!XMs{k|z<%j8iQChIvf1J0x2HN=$mX?yVPfW2dcxi15COzk~v0 zg?Kf~`+6}->S0w+LPkm)V@}vkM7<;n_2pO}xC@X+^W%*$RhywDGh_dYHS^5&0l#%_5wPp=b7LA9vtO~@^Q_6HVdd8xTIho{ zgKPmnpsqs;Enbj=yK72L7PGBpr+7G+o7abRp?ve-qILrT>`k6amk0KdnX0!FPDs+i zH1T1QQcb%h^c%|~I#da2=CH`?@6>D<*(H^DD>ZLM9rH)RWDVNuIMQ4@)aJ{6#qvFm zVXV{imS>7FO_E7XZ-NHhys@_4w6WdqsDGL7xz}d|jt!eK zA3Jb*>aIWSoHaX@NXpt))ND+BSN%E~Tk|@zpXgMBv&swIQ0B((i)B`I&>yA5odS*> zHyH8CnXaI5gS3n~wUg8LliOZ!#xe;$3~ze}96%lLraa+(+OV@hIF?|edjYCB7cji5 zMixDg66ws1X0ck1y|c|2#B%(iiJLH+oih_0m*^2V zx6Ncb9F$J>}hCPO{_HF)|3|G zX^Nfwu-kX+g~AppND}*aA^7}UcNfPuF*7>A+=IeK!$a>@RX)nMHmOuvd8@gHhqC!KZ7 zFJ0_KPnj2VL~B3|8#Wl83%&S>)HR7*nivij!M0);?$GNXn$*f!>AI zq9kF=6s%soTWki})0Ki4mxImx5Qp<-+;jJT++!PrNnL8!XSZy_Lqru-vFX*ErM%H5 zfFU>Eu2P`VR;2T$>|LZKf{#?xvhn{_%7WRW2XW?U^ptaNSMmJ;wgcTz@}nPtWfd=! z^(z{CQN?sDsJuQb>;Wdd&HPk_-X$B;AqO4&dm$eO*;HZsHTZe$hE&RzMU*1-Ptv-d>rWx4gf>TV3qF0>-Jm;4>Qb{o z@U6aaaU68R^2OM+0mIOT>2p_seVrThS);$&619Un>ck820sz0s2qpexcLMcMB1!lg zVT}r$x@@XRA{-e2&aIV*z3sg2*6{4=9sTg(@BX@BwE*G=`1QpRyAJyRzXls_ffQ(1 zRi;5^-wEG2EPs1jorD3M>pL3%hU*WKPCCN2ISA+EH9AIO7jM>Yl<0@M{muy@k@yc- z6735J)rGj%>ojL*U(Lm)$l+`Id@4@Enm9f}^gv*q&0F>SPHm9FN z$H^&>1915@+NlZ{O8{Lu|6RVe1LoZanHmDEj!1f?6h8BUrYEhi$#YR6WtY znIE6Y!Ll0NsCYFbLh6`3xb^1K3}0yNL=;-_N+$uhVZT5xdX_IeKJ}30lCs=1DR;Xkry|K7rc zOI-gD=QNT|`>Whow(bz9Y&h|c_B*AT30xnfoWFC!Edj;h7A_Pmaer)9Gh8lDuL0@E zzhV{QFf80Zb+XJ-)&} z$?5Jkbshi35GeN3;<|t#R!-SJReLPd)($?RJ|6uyxW4rf1ch{o- zU1S!|NmZNOXg4@z`Aw)VX|~NP6~zs6vMIjz&=32xWCF4H^pT7wHc zH#PDTB;NQ&W2OYNUZ>8k*?jR_m&(HHdO!U2epcyWpb~KeuzR{FQ;kH3|5qjg6cK^l zwI-kL4H;O(xh}Se49zKNa~ObL#4L3KtJm?G{H2f(Vx1%qP)waG)bN}`BxM2tCl^F0 zI@^l}K)H;7s;xIgj>0IN)5gt6WyxJ`12)9Ed4jDfzy>I0h?cCz{Rfd)A_$J+VA)pQ zQpa;%9N37araTAyN$(3g2*Va#`-QH(Awcjk47)X8Vid7ZeM?V}*}g3hT6YDQ4H$sX z`JwN$)Dn?8;7c>X(F!cs2_HVcQarvbuZ;P)fcp{?mxTTf&|myJ>C||N-|M6UU^Udk zQof1tQ_3RRy6o&$yp%^YjXN85EP}MWXz$k%M{PS_pp6z#dg``O=D^{v^kK1%ju(ub zmzPR`Cpi0oQC0QfL`V*%v9j3s6`f_Z)PG9qm4hB_KpGUHp5VmL?SxpwX@boH0A(YD zKUZ+(OgNyeV{jmoIWU04n0cIE|H^pv`-i+5Bw)6K{ZoolX&LOPy)&FYz=sLbwI$3Q zz29Y7==^0bq=5{IhpXQ|>~03bDfLPIi+@z(8B^3jV?GcHo?C4V2se3c1MH`1oH^|( z!J{3abl6`Lx@%iHV^M4wI(ZM^wE6OkdAe)+$!QlkzxOfc8bGzY5B7)s)#f(p z{X@nAb$ouIO$W6&w}&?NP{z=C>QMh#zG14P7ihqXalg@CgBg}4qSov-J`tY4O0)=t zHy$c0i|+cRKyR~FTV}-G(3f{IbP5vz+NkORi|?p0$e5~%8{WA5&7;y9K{ZD~YV8{z zA4hK8FP^vCzT82jI{YYqq~aZayr3qhC;L9WDljg{BCl&M_X_Q+CD@EpquPVzfm#-4KB$G zEnoaL&7`_IS)d(4BeL!DT2N~#PA$f7bHeUtC-j|`*Rrwt?2KjMrgsAP*RF>}v`GPK zfa|Z4Y~QLpGwog@ex4588wT&4HxxPX)t)lMgsa2G{CK7iUA!&em+CA~qMTHr5pW-MoRQw^j zC3x>8B(hWaWNus2$#VI=s#_bZ8T_W}I)CJfU#&Gh?+pW#RxaETZeYEyZ5EA2E=(F@ zf(O;qpcv+YP8k}siYt2VZF0U)0)2DaY1k9zx>o#z#%2>@S;ifRrGB9PNz)l=D`SaX zoY>i*tfOFQ(alzRr77sx1Wz4E0tF@op#Nz{`GjUY%I^qu3`Nh{KhQwX_*Bc+D=f5O z7fA<)(|5hwHQwr{g{G0W^&9(L4e0Jw0kakVZo1&6c9B+;b%u<+@6?%Hs`)m>wNX|T z<~mU{XkvB4`$XVzmtmy*q~716J%#E;!s-9vOQ>=lT)3|-P>CBYhe_;&MXXjrMm4AX zc)-YgPM9)Z^@LfGVk9QloU+&!Pk;>K@3Z&bWf@9$COTF=cqX^ad_k7{RnLr3y%EO% z$rX-5?u11MG5mBxzYjMA0NjNMCx_UI0rxhUaBB9e` z-7-Lqiqfw$TbgllFOLIC-pYfW0I&(s1_=J|3^M6*x@_#_obM3Ud0x-okCT)e;#fxs z@p8>>@dgWGwly1ANXvR+OU_Tw>)e%mdI!2mSpdDniVDx0;+9ODgNGN zzd+5$C5h|1)?xa&Tb4E-eHls1?T3=C;+bX#9iZ_KIRZwlhYOiReyBFe6S#(9XzO5X z%|Y1ZbqwV1dUZyq>8V3q)+#T*!o`m@1yqq?rw)#J$db4dBEtDu0Y3dx4UG(U?b1*u zmAhzupuyu4hOTyw5w_h@sgIxK^=8;ujM16bqY}#8?b-RY+~g`aQ=m`AkmsWJT9)G- zgo?P_ze_8iKWc}%Y_FrV?7DL}*1cA*eziNBVW17NCE-$4-{%HOi1kGh?L-X2 z;?0sOyiTx^Pl=e=_lbDT=$iY#Y2!G6O+kRD|CibE2}4*-yCQ17%F>@ z*qn~SMSVW;Y(1Q!?TcGW>F=3U-1-2%&p8_Db4-EGKq=sN=evbDLteVMR1SI&tpQ$I z@-rYq{AXMjs)2s%jp@6Eda-XVY#SsJw%LwPU*Mfx^n>hJSL5J;Zbtz0qmT2#QutkB z(0a9c_ternq+dYH7iZ9jfCnmbgM9tTIzh^yWL?&uYJD1dzj(@xf`&rah(P`P3#<4a zEaVX%1(CB|?#@2qt#{j_zJF-RwBr`Az9Pv4=B?Mf2Co&(_X1<0(}jV88VzjMsw##} zB*4G%!MB0EyQ-&z4{3RvO7U-NsOxXq%&6&;Fz(*}u1{%wcy2Z^`%i!XT<79usQx~} zOgs@o?`vZt8IbmRYQ_ChIy*c=Hnbv=VqUW_A<;wP&ks({33k$I&jWE&N>BFqUwM88 zwpev7E2g*nKqij_#+0}$53X7>Yuipvco|i;E_tXPR7MntRi^F#B^jHeqEYBXe$$gbu@wQV+HzEz+@gO zZb&Zkqlg2pA1SH-lS#GclmM>ofPv?a)(hu?%l=e!)|HeAdWWdJx&;X@8HEjXZD^tw zgH+nE4o6IvntFzNR(HAa1PfGm3xjaJ{FhLkcwA++{x*3%wP0by4axqxO#7n()ogKH zs0sRn$$<{`SfTO{AeSlHemJgZjl8tQIq!!voD1R@1*`icGo?yP;>WS(5+__|o-@Sa z8|qd#e=xD5{`-o8UaN&S)PAQvMX7sJGtww*%Ub>nO zeafEFJt=dQUi)Ta5x7XsRo3WNcKxK>>VCtnl0C!(FDH+3>nB8(YFY`uo^EgNUkcc| z)QuGalQ8FQpVBZ~@SI9kA?6yi++lGv!=$meyI}T!X%!~k$dSDj&83jQ#@gQ^WBq-mKP!;>1l_tBQ?Lm6&q8{ zpwHGHgw`Bpla=0(0yVJINX1QT#psXsI+}o|5YukDnkm4%+bG3QIW0c z)12F%jfZT~h#2G#S9yV8l>staNh-JHmu=KJlj2?xcz!eoln-D(-AXrgiO78H3U76c z3z3zOE&MQjERg>%$#)cP@BF}2#KsLp!+$JVY_5BwM)*`ZKyPGuicKp{$BHJsSY8Cz zSOsZP;#WZUkQl6^e8~q63ncRR$=ud06UDfdfrPiE&)?NqaJwBN@aR(|lGo?AC? zYk11qv9Lv{oryY@-Zq%8hO;sfrq*R$!3*V+4u*bk1KF)1nxNFO;n>bhnSZjdf+epq zO{ZsNFtnYH7VUcpF65_1`Xi{A(l9IKMhSZHB8(1h8P*q$Jb2d$>yiKu594uN#c3!3 zFpLc|=wp4o{&=Kp3zvm{0olBki1-j?u(*dQ_aD=hUAybujXCcE&^4#wg|sB;z@X?E z`)#&npbJYhi8dr0=gro{YFa!{sFBb5c=&=hG5&-WU?|0H>_Ldv7BPS3Rh9sj|3Y&S5nGKvLIsO>jlEyN6x%5!)hpBYJI8vJfbPA#r|* zxf*|K0>Hh{qk{C4$gkf;pL{{a0y4 z)z!}-P}^4k@_ZrW*Z;%ZTSryZe(k~!5+aRAND2rdjdV+QH*7kU+O)KEgGfq;AV?#d z?o9|t$ELfxYZGUI&+mQD`M&S3^Nn%FVK^Lvfo|4)ueIhq=QXeEn)_-~Us#MhPEiD~ z_@`4AL^VF(_OZk5I!UamNV`4&h3S=nDCalfs}36K8#9*?rb|9OVd zY9F;#g|@>VJ1N)|W#BOlm(1O4RrL2#Qq{oTWwbH+K0ep++-m=bFCR#kNksl%^C@;D znxh0Hv+m`bAvGN*qkvE)09{%#h?SGvxV85d@_yd7|~u14DhBU4Ie5?Maf?byOp@yL@;AU^W8jVkE>gQlJEn*agbR0~|-LETF{Xq3!|l@tUF3{ae$9-1Bc z9DEH>uJ;a^)K=Cmt|vc=P`3aigYbx?1B0T(y8HGy14O3}g1hs=R%zOh;-Di-CIRS! z2ggdXa*`#S4{r5MDWo!HZ-Z4CWBhbie=<1H+$d*y*M4xN~2pAf|?D5DSh zHX-`3j3ZSMU;!dXc;x64kWZ|!o&MUuL<{qDj$#Pu%r2x?`S?BAWXfg9d z`{gf!43SpQCYD#T$c-=@>&3BUk0wa(ksPi|o-cSP-@M6U<5*}|2Rh*;mjY0c z7%+j%L>lCYi_^7e=Yh|Z0k@G!e*0kToep8OBt+7|pNe4ThOhE(bL6P>?Vp7OysGzHA?Jan>v~B^h*?1RZe=ME;TWrPM^Rt2hCS z>YYaJA=>>bgSwxB8ByzoTP3oJnPB;>Hf6zutw<+;?fvK&Br5UXgjVT{Ul?h|198Ox z#rn%bn*SFvSk40wCBlD<0XFf@dG>(=!N^y@NUOWZY7fqiZp_S{=p<720&E1At5Mc( zpFKq)5B}iZy!)lLS`;-jkb?4bgJXt>e`pO|E)$OdO8Nu~eK)w*8xk!HSZ`Op0{j*z z|Bm{WI@Lt{77tto_L((UV89IXUwJ}fTJaNLtme|95?FgdB6Tl$6EU~&%(T|~f~{uu z;}Ciqdt{xfiGC^}`^cMbIQZ*Wtqo7Q%M$HHUJ4TEiK1(m)V*@B4btiO{)5mknnn8l zjV98PkO0$+AI=3*KPrpf6)|xPyI#P*0tc2igMcNZ)aG`G)%Kwg2cw=j z6V>iL(s+K9mGi9lr8(hb;(lv|3xkc==*ZJ=tqbpA4_@L9^il>`}-a+=~G&u4wG)=z0$#E&pBARpp&F9pgDcVy3+ z%#K6(5iB_`r+NhBe0parA<+3sWDy|(I(5@zZKyC-BQ&7`P_=<`nc$9PA}PA{=+*AL-%jG1vhAXl%zfW4664>e0!dHC==uP(U& ze!J$Z=X{wPM|!8zdpr{pp(%+k z47>K;UQLqB)JER?BjrX|uQ+XNZ7V*Pk;~(@R~qiJPJNkH9GOa@(>xFO&rQTfDOy}z zqTI;{S`JF~G3a4TxIYD)9XB2ilXXMhpsD`dzB2ZVn8yEA)pU2w$St3Ek@S|(p zt$ffG3+B&WI2d2mw!Fr3y7to9Ts<0v&FrlMA#A`KLvMp^a*v|TVlIR$715hIMiriz zobeJLa%RGxI8#}e^>Y8q^33u&=%@*ZlU%g6W7K$uhOSvBi*TNio_$X+3nRS2;q6*- zvn0t{OUtQS;zq8Ob%B`N#ULsT+$|i*VQUk4R#Wg|aHjV@V05WNiyXVzsbR_0H|k`A z#q@APsiNW5Q(kuA51KInC53fFbxU^sNB(V+lp&jnHr$<%6C#&=xlQ}Vvh;FDwhlA9 znrdCRxpByNWn6;9!yFbqQzL@ChcPiRCFLB)_1=j1oe9G?&y?7FZk@O?%Go;aD6zO& zg1THaweG?KJy}pdpQLQC!zkAEwpNNqfOyAUVj_xQSbHyWy1fuO-OL?Kpn4ZI5nu zn2M3%^!*6n-wC#pX*}m<2F;S8<&50F97>-+Zkm@nXUg8D5crZmcp0p<9C^^IhK3kl z?&_)Np5>pq&DZ?Q^Y~Nj15CF2d_kJJ14NW@6u6A0u#|9EJ;g9_jFWc2>(U@VXd)EI z;#oiCXXMrroYBxSud6ctGIJ>TC#a8n%@O}(m)qI) zr$~~a<#2(Fdh!h5Gs&>!#Tui0|73n?e5RQIS247FkKf;Vo2++hOU_9iual zj(Hap1S!S|aGQg|JipKRRNSox}m&#>b~pPJ{zAdyz6efzB7sY@e@+O&X-?KmhrIFMtK_Kh>* zKvRPQ#}t)jMO$wvDf_FaT3Y#A^zIo?Hr6|i3@HN&Z;C9{lUjkkm3VAH7mEplaODs8 z5-a=x#bCcfzUQf^RMb=`#SQ=M;7`jVbHIwRhBTzp{MbjVFcaTHbz z)OVd|;JE&Q2oezH-TbH|u;*bv%oIj1d#(BAT}Z{je0QppZ@c(Nz8c_W(wE6ZSd)z&pqE{6e*W_lAz`@vWM9qe(;6#7IP&d)Tdp^gPE~wwJNGHsu8EFS zl!i{VH5V4LFIm^nxDsGe9k3(}hPS^>BU25{h{b7~kbRN7J>;ZkJ3tiDkj-=LDM;D= z`;vWMG<}QH`E*Y1EQ(&tG6E|LsOlEdKAX%2})(xA3EHZ9$cO z`{{IL{J5TVRBb=uvFXBACWr4 zJ;Z*je*KE@GXIch<}z7jqqd}(C~`Kas6yNlfiI$AfVR4IK71g%Tjs1k30-lbm0D)ELzi%v~Xn2oU^NbAOL?x$qacv$@C^^;Fe>V z@?fR?viq;K^S~4ae2?IKMz!&x5RoF2Lgp>E-`ua*7W{?n_wUC)KUnN8XADTb3Gk|N zr;yF;SyeU&{#`jhXrFEQU%t}b5=Q~H`M}uSB4RxKX9bG z_#^_1xr&fq7|X<#T@lq(u7(!GvT{F;C8C+Fw44UTavoXNNr{e~BXJ7)E=C8W>WWU*R<15yOV_dBWRkY>v7QKd z=jCLOd&J$fTedk}shCIR@F}oYaAf$$Yroe?j(=WU8T?G;b$9CPEt!gQL~?Za{xp^I zj+RO<@`J7V)Iv+`gc zgI{PlBqK10A*uk`=d(SV4v2)nIl1_=GPkBA{G8hEl0A-Vz2Yddw}9TORATk${ST#{ z3uQ9+$=gzw>?uL!Tb5|nE9!G6IjRRhsLM7ucQll>sYpT?nVq1Re;B+sx+{(ZHoS;Pipyl(CeaVb0B=*Z$J zXKw);hw`RGl@T<5@y4G?zXB7H3pY0m@cm=U&(nllU=y2F)Aq&p9_?Y|U#3GOvkgrj zL37qP9)4bs^IsK&_$sn%N|a{gWWKDm1LcsP`WSXSiEo)~ zeHBI}EA3$Kjz49M{%{WRmk+`#ISSE}_6c0+`GAT&R8y1Y?fYOq_FcCF=SjQ#4exg4|xYQRCx2PfGOj)luy8H$+hxNx<7l+z|L2!&`{!H>VYx{ZNu0`RAP@8A$NT$YdMFA2y`IA^v_dMz! zd@!*xkh#NmoK#P{m-0$f6lWiRx99^`&(H63sIs`hQ*QDz=|mcKyPsaHUnu5ltMpEY zUQ5NVR@Y+c`{V`ECdt(wwf)0psNYB~PggpL5OIb?YRBUBQoA*ebxVYO!OWTm7894> z!emXtcC5$i+i9g-L5w$SOL2vl|DZ=vM3sCO^EAUd#|7Xi$x?r}UAXcHzPwCoE?xDc zZMifqUagI-wt2luo&452A=Y&la4=_oKcLml14U#xGl}U75!qkhm|Aw-!XoX@+|pjO z9xrQ`<(*5*s0Oi@L6d9q-6OqU_I-PR2|_ozw)qKq)8!2JB*l7)toE<8K7OxDLTG+w zH}&0HTrR2LOi`t}9rUiEKVTqO5ja7KXJ!d3;y8zUtYm)&N3|fc~fnq2G#t zNfD?!Z1m_Zr>JBqUQMd!DNy%YWX8D&IWm?xuc~jnyRkCH5h^2zF2y$fH8s4l8C2q@ z*If{AEk7g&)71DalZK)1VLUh>U7+&ASiRdh^rL!FzncX)PN}r8LxVW_A(&WPdar$bpxjnAeTmLgt$1&qy@} zUOOUun2EdUhc_R($*I@)BXBnwRFRY*JNi5t5;j*>1;;^hM0fj8a8$pPzD~IS15EH* zN|aakjKOyxgL5uT@n7bLyNSj%XeFYi06WX2V7D=2d&4Y>8W46@Dncz)`NWM&MaSJ_ zG&h;X@P>3g_eQ{q3BF1aRv(*i_9^_4dsDimZLsyba6pG0>mJ+t&4T2&IuzUJA|iL5 zHxSEoN6y**Y+cfXMYL4K4u4ozWmLgOr&j)VxWZH)tOA(e`l=|)fRghhwD;)erBmKq zRT0&_kkWF>@8|2}ci2J)fEIyNLQgF3bu|7a4=v1s6A+0*c8DDBG{4mnJair;@B)Uh zojyK1jPhN==?A5&wZ;Vp?5ACA&ZaTh#|eG$fDQl1bxH5=kY!7@=b+Xwq2JfOm|EdB z=f@2x`c3^WkEj@@NF8L_SFZA=+Gz)Qlw`9mJ>pF!Vbo^VC;jjPT+>ioqF?yy>OIg3 z<^51EGDBzkVB6nDtF1)STg)iF9UDSZzGHw0IX964-~|1iY?d%EYT*E6Xln`4q}!lm z;=ra92=56qk7{(tPJ9s3XX@r(-wukY1E|790W(7cM8FWE)$$7BpbV&ST!cJ^{s@wpInGr4xcS%MRM0LCuR z_9aT{H8<;BXIH{mfzzB^aSaeUG0ty4sn2+kqmR=sV8nW_hf9eO9{ss@+l{yiUKL5x z*6$Jd=Q38gFQ_AT@2%DNK`$w~0w|#4UqYLKB7pkS3WzKo|JZcB3(V5}eVM#~laA_Y zjXQkhDv@Xm&ux1;v2``nOJsMXnrZqTaf&bzVOT6#>ay?{HoZkFS-hfHy?~cK_2VM7 zi)&|Tszaa`eK$!G)K}3s^3^uI59=yIWdoyNR&C9_{lOo$0z`aD-7RBh$xtUVF}p~g zRt!t;E|`wJjE`Excp`lJG&>%16QI-7eQKpXylop1j6&3%CS` zRIiy}Da(C_`k1kQnt5yEe_~e&?m>>G4qsT$_1@TdxvZ&u1{VRguevy>LRmVsEN~Z6si9&1+3W5S~T?r6Q>8KN9X+Uk57pI)DBFV#K5GP;0 z)cPggd<{HwA~()L=JyGyKih%v#a5cMZe9DW8G&>^j<|;k(P#cjUf5O(LO!a@#e*3# z1ZC_y){xKTs7u)#-1;Tf2`Mm8<0Ax&>;F2nA9hr$n&d;Z7gd*|sl>DGto`*h z)vOTdQ;6dZUjo-x z(}rlOn5R?W)|L2`DPTvW44GU_!{_J-JZ{NT3;mNz809ZteS3KhZ$cBHr`Y3<8DxFv zM3rC5yXIv6L%~pA@qh~NAE;8}lmX|MCerfn96)NzBdNM0?z9{GaF$6B452BX4#0UX z?=Hd!u1C_$rk7e1l!Q5O;$?pL1OWnula=1E<~MO+VX-OZU)zYqA3c~J;Z3_oc(Y_( z6d2%&WKTDxN!iRMKlggjbi@sJPT$aoZq1E_Z?;>S3>f!8BP{cc5-stgeUX!7G?G?0 zft#{Rz$>86h<>*(Z(xc?DGy!H?;|L{pFF@Uv|p?{$N&sWL#Z4&pdGBYv$6-M9(Y_o zBTygb`r??weLKEyE3C7w3q^{}hjj=vMlS0zubBthY^PBmPxm5YB~UA*1@ZH=c#?3@ zXTEWMcF@VSBhW&STzCJ&&TzYeJzAGvuScPB9VO}FUe^^nPV#ez4$?qPjwAJsrR)(f zTwYSOh4Gqwd%6Su{q0^dRiO7~sX>hYgXhS2Ml^p(OQ0LJ`w>sJ{0U_)An#ZIlZyzFY}jpfToCVqDBF*L1L}XQZ>y{uGAaOJRWX70nFlV8qNS; z)cltWwZe6k0h?-i_CSI{2`kxB;CVva#Op{{+J!molEZuc+f27v=M@E}hPMN8a7$6R z;ED&pvwyg&Wj<5MXRQGfUqzGkW{K2(#j8uwO)$pEPrOq3OJfX8ird(9vL&ZHcuVSd z6FK~04X#oBD&-mrF;DfFQZc8h@?-8wjD3B_B=c1 z@Bti&;LqxMzTO2No;hM8?cUCb7^`$!+Pz0^HVjDC)HL{C%{B=dW`B%4nr+&Ivrq-! zpwyR>H`V0$ZkV(dA~Me^Oua96<-B}G{D(x@_T*KT6);QGdlpPC37;}E6t;<5rG80|v~HW;>@e3B@9W*$JRdArb$>9NG;x2S!)!`rZfyT5>bGEZ#flUIcl+{{ zn7L_UjF*|3!OJfW=~Apy3t_Tx&uB`>1(eshj|qbZ?+}v?Zb%_^1y}rbg9FAn2d1O| zT2NO4&^asIaI$%)2Er0xfEEaLB~%|m@pk`Pj3Y<*vcV%nWAzQNXAKlf*ilxl#Ljry z+kf*v>5Tq7>WqT(&JeLEC`L9XFAq3tOD|F#3`n z7-HuaHpp-%6~2+$R?r=NYss=S>d}N*_Blv}Fy&9Dj0pxyLngTKgg+9CJi_j$T34$; z)y{+Tr`8;VTkU-RwD3SV@qx(7baO{<8$qo{NjPzfSkqCz`UDQhehfN~<(+PDS>50m zKd)l>e3F#v%3=O3Sgj9+wGw#MVJ{2@T8#}HsEk;l<=cv^TnwP)gE5QF zsd{LKHHc3cAAHf*)o-slmA*t z9{2BBN8EHgdt-BIiW zFup9JFYQEQaj0wN{JgWRs=CNA9CeJf-+ywL!H?D#tvfbxa-%&o;eZBkqr-!WZ8VG| zC2R$WKQ(t2kwfq;#%l&t5VBD7H0--t4E5TC;!0$`bB_QMdZY3#072@MsICWl1{daV zeQV%yIQOz|S(#|zTf2&{FPx&~8UKcxS1<@*1!d5jS@uUjL@FB>w@XrG?J(r#(Y zqq;ltJpWZ)4@8#nVvI?_i@gxYNn^ZZ{7qu6(6A>>HRSrI>Tspf+x7p}HVsX!bVwxO zMQ5<{_oo89{sN0kdKTL9EDz2l&Z)n5qYoM#LTBVO}gaR=nCd7PS$P@rVFlckRJ6~aYC8OHjE8YkAG+Sn@M|ySXo0S zKoGw0zG&^~rD=P!GmOG~GfQ&qG9qT^iaH2?WG8Gx`tGa&2adz!b-PoMIy^mzCw63*#SoFj{E_I;h)gV zo+^lZ5tXz3cG)1RtB9u?Ns%*?^eJTIv_)jqz&X}j2K%ZeP<0^tv$|~q`p~b~(9lrL zS+o%AD(RGq0%E=XmiM;@J%zuaR!*4fp;lyQgO(SXw1V%Y<7H|k#UH-nLzkvM;u7y) z)AK!zTZm!j{gB*=WpmJ|mt{;MUUe=Q1pm`OFIFZ^2IZuB`0B-kLNr90116#wgY6~I zk)6&|+{~$S2tzp3E3jcLO?d3iYDD)0Qc9;rvUkFHKrIPeQfR0n6l|;U2}4WH+yd{C z7IneH7s~93;x<8*<6?Qo%vU4hhCGKMc42rNH+rZW)pI~EO;1#c`|zb=M^sw!r2P|j z3iob+HywOx;?+&q{(6!l2C$%hyx_%2Jv7@ZN{}~Q(1%|{d%PjJSF>wp$gh!n0t;p9 z0~Rbeczrfjm;Wx7E!diYIaK8b0?9ca>sK)`Z6cSP4^RXiNuC+v;Ru=cLklqx>xXlm z3Sj$O_;+uBZ|J1{cYdG=dG#db zKJb1}<`3$z+T#j=zp6g6g7j?tR}Sl$+sAUG%)`bXey|aCG8f|>#^7ry{ZrDJ2 z5`nE`lPT3A3_%~(d_y1yl(ZI-!!Q*x55?T6a7`Fpa#@MRFCvmlB1}FeS$R98HA`3~ zZzK;^mh`WeXI*l&!v}tDlh`eW?gH(CGdqq=C4Pk$#$(na(2%wnnDK-?uA%Qsi$Qkx zx}Z}eyBVD{p?s-2E>cx%j{j^>K{@a|5GxJMR4ZWpN0D?pml?`brjb80kd(x-lT+MxARuVLp+GEyu-Y(*g({eTaeRkQd_MnUpJ&mie=70 z!apQjbgGmZJUA7-{gofqH4)|a^N2*hks1GvhMn`Wek~@+J0!7zkch68XO|Gl!1IvjSQEZ28jLXe z{AcM_Tg3xl+kLIci)J*_`|md$=&c8zg~l!kDQM)#xEOyR%hyHcXyXh0DJS1z(tQKRzpotb z@Q6;((liJ8H1_)+Z^a^ZDsE=cjpl-_dkot`}iTQ zktDs2)ES?!uFc259^-0TVp&y`5qrIpH3cSZ8l3Q~Fv z5$hFbZ*fv0gS22y3b}9Y;NCL&?~ir~vUX}&?j3eJTIwOLz`dagv=+G}i*n2r?i9=9 zu~AHRN4hvWVTCkCZ&#XJHSwuWP>91r470lVOUZWyT-yp@ViX>`SiW3UUI-$-!Zux! z>IqiaOf$~{wul8x-LY2HgP#(3G(&<6jc0WHN2P$#@_~NXk}`O<)z{78^Yz^ zbceOI!oDikNpJWgbPY8N{iXuxwMO1sf@)HRDqB#@d-Wyd=v63E)XLtpRQ~%&x>(`| zS&x8+Y802)uCetYuDW z^yiwv{NxoQx&bYBzLao> zqEweH3)}h7;!5E+K4$#&Z=BWqL;no+eJM-p@FI7P2!D0Shvm3jS;6!xWxPn_nR6EX z3ol#osLqm+Rze(7hnzZfhGr>reW=7ZZ%?2x|8~OJo;&u$#2n8u+L>X*jzW@+-aFjS z-5@6B+}3W#b@Jk~JUYSCC~d>l4b3PGoKj?OFh@eR6-+`PS&bpfUH&vfysnxc-_PMF znegl9sXHtijPZ1ilae(V)LB1>v0Onh=QN+TKpZ{%9r}G#+%Ye;;MYKFz`KL@t_Ccf zWTHbo0HwoZvP37g2}q7Lr2HF&2w~f(ut`m@JprAl&=Xz!bq?+ zHJB^0w=4oXA(d1dy2`rRddOu$u_JOfE-KQ=Ih%vpYP-oe%3(mpnu`J3k+ zOnY9|C*^O#}X^2((Y4${*h2cFyE z00Q2Pl#jJs2Y&7m0~~(kq|AMw+?RR8C5436gjl)Dhj#NlNnswM;sKtmU4| z|IOEibt_GPS7bUW*WLiC+|DWxZ2c3{b;)gVha0#ZF~#i=Th?>o1jAS+huZ2zh)GDT zjcq_}dQh*nzpruDj?1Q|OKbHkaFBb=8MX_)JKq7Q^=88eM zOeE1?qdB|W9)IJ!B(V^y?y*W(+BwV1Jp*R4<^A_cJZeOjQeI^3N$Q;0ZJu4%c9V~3 z$YP!@cfw+L&_^!~`$vNHAMz7@o3S6Hy@Z?s${q!CB~CI>&O%F-xLTito~4bu5MP69 z-V2$kj9Mw1_#d0xc6bX(c2DjZ z#V5#o10rdZf0Gc?H8F~nSyy!K%CJcp?aqcEr<2Psf5wF>K?bZ<}5jhyy z$KH_d2~CD%^p$m}t@Fpt>jk2XDuo$Odl)~4^nMFRj05at8aQg5uu4$^5YhhO?OJ6gBa_1(w$vbUf9@Mn+Ph{m0$96S2lq5si~$QmXKPd#DP>f>BU zSuU(y7@1SdJNr;E#qs%j4Yv$osWBFHG$I{ZQ>mY85}8;J+P)6UG#6c;hmHV&gP>^G z<0Wqf{X&64bRz4@+D)1xeeNTiYm~z{l+qFEhdjfThlfK7199HhBktgH{H9EEh3jC* z7oI|OM!*VN=J*XDm^11$ z8lxn5Lhd}ilT~u{PsV3m3dTHgjU0$)?faQp&`Al?b6UIX+>c_P((}3(!w=Hms(A28 zZhL$+&U7>)@fpN&K8-0tiZiL3B3Z++L#$4xlFNjGpQi-v^3$+!@+k=63Hs8)RMez&_(E#E?~) zJAPX~XMh2}J%sP)T>N~7xz@RCO`<$?!wZOIn!b*d46aFpPZho6hzO|`9Fh#CMMI6g z4Lvyk`Aacv_62dYz~)t7aPp$#OyGytX8L)Sa|VWB5^(Fc1-xHc=4`vi#|rIkD!9X& z#SF4ILH#7uIUHfoLB0ld*Gn}JExaT32+3s1kZOBV>vm?VzGV<}bX|t};`^>YDu#E! zIhwnbd4D%i+S}jnM>#rf0p_dPt%XCA?q-Mg-jQGeRP;ltkAJQyw`@(kafD{!&Inv_ zrBWMqzjoe@5rS{w6Vi98%M;2_h(8!+zj%(-cHoP&&}0DO+!L#-BHeT4Fg!P+GH#as zTf2DRmH@{jk?V4(L5$$Id4y9?f_e_K&j9DrGZemy+#C~OzNZn&>ge2!h#>e3za64EYH-#k~q%%JF*8yJZ!HDoUs zO)EyCm@3bz(*hM6H&Czhh^B-KS)n^}k9g6!m5yas+9`p)^~Ti8#(z(z6UIYN1}YS5 zJ;a~%*5#J>Bvz1{e|#&y04A{Zg8;&jN>G!R{ZeuV28RDQ;c+8i9rt{a*Xy3Lu2Kc_ zi_H^Ip@vV<4Z`S3v?9L#lq6!A4ZLtL+)Z~~3-K6i=am;~#QZTHnAint-w3QK?$sJL z;_<`YW1j6P+Wq@dTQ%tfK}EyfQ6n|hUP1WXYsFvVTCiUIpztYl`{DZ{u5YC1-`C&Z z#eWJA;D0*)|E^m9{`&uuALYn(u+ybuicWmyv3a4MD{tw~PuC)Cd?MtWDHxX6g|d0p zprT)mNbC0Ar((!raYrbG3o7KG@2RX+C>GWeMY!_?HTj5dNVf^u{LFIkc(8ZyZ|G*i z47L`JXEz=R$($3;={~3J z^3uEchPRFT3|N9f(2RIt5W7DSo#!<5-t5QskD=0p4U|jR-R`aJ)?9Pvp~*DvWBzNF z6R#9`$&Ev#=xrC)&aHo-HM#+VtHlc=p#B#UK_b5o6Mh3&C)((rZFsG_(E0Y&m7bZR z7!(=n4Cp<)Zn~+OJfKwZmaTtu%Vs6sf5}9{#W}sF;3K)mDoXNwo8OKjnQld|?+YbJ z`q)ybyPxt5Mh+cUMTxR#m{ex&Flu`xnT`_w4y2bSxyYdp1^>C|`xvJ{)(dkiwI;&_ zFqo>iCE|y4g$4p-SR+GopMOsD0j~)qQf51BG^pzsb7&8;vNZeC4u4peQoh}^Fu=8` z|5-J#m3xjb*;+1N!t|N&thYjFF!j6JiNMb!i78PpQM55&iF0EYpDtD!iab!wTwUmd zA6G_>F!;1!n0e4$?jbr`j{+4%hot+gn4ZgX?oUbF^nr_bpO`5m^EjT4dZ57)KejmP z>=QqOA!TAhM$You5PS{lnm#WEK21uk5o2JE0q{1CkXUO1kVmaGOUWCj z1)aQm8Z~h4Q!>4(xD6B&&RyY(%loq9DK7hdPY%k?+#v02= z4j?awUNIXQYK1-L4aKzx2JbBMHro48_;je#;#v8HY4)&8$L z4fRCzJi9EJo%%c6>GK0o{af-)lJnfHL;hOuiOAT{mdyo&nmZ#mkV{8@xKD^N8n{fB zUG=c?ChXQ#%7U78u!*rpYnKLjLV6E+=LEWd`PM4UCnf!!#=f~I=Bk$kx{}LcoMw^e zu!s7yC0y~>ON&u{o{39hVqK=%^_NW2hp|F{n5FeXkU5#JqmnDEyC<|-tb|p!rE{?( z?IuGadkr`_6#0C`u(wEYcVy+$lh&)OUtIVed3*k37+xewG)yw{NXwI1W$8>atRHHY z-ndWNd|q?qEQk0pRp`A^JB3My3x;=)+62EzN%Wht1y}NFmgc~dBJ1Tds9zDGaLBI% zer@cu*0T!I7GKeg0~&2xU{d>)xm8_sQ4)A~DuAMx$*M`{xsTz1i}dVuE8=Y9p}}T~ z^}8l?(;4F`qLNK=uNaZh9w~9hE;^fjhC)#l)UpC#Va_y)do2*{U)GkmHVQ zi7>j2>ehpOX~(C??m$|4)Z5#xVSV!vUHQ;wE&LB~2BrS!crW05Q^J#Q>tEP-D-N4T8Iy)hQGd>|2Y9Qxl#)cuG*q2IIDi`A~%LYzTGK^FjwONwwHh@XrGW5VfpLJofZ32!dy8 zI2ZG5Ta4$CQhymkh5gY5gt*FBX;5*(lkJ_ z?U$=RB^Ptumg9Nd3ME!T>Nu=Z!4-F zJC-Fw;1LGjyC=`TW*XWW%yzrf`Ti#l=cIZMl~{4EUEIsTtjdu0w`-lslJ0r8I@JWU z89R%mL^AaR*7-{1Q*icX-V7D2b*^HBvmqf zo>+J-UV+!j`lYx#$AQvh?BKln2Iyg$1@a3!V02cmjl^4@VGeGzW zUfDPZ%}f_>QK462lAZCuVGo$EdP+Drr~Ztpqn_uK25XNi+meI%3tq;u9gN&TFbaI> z#PiL+Wu-_gif4Q!hQa>G!~FuU;*V@r%(IZ!Q{$Az%lF`X2*dg{4t>MEZBgE90(^of3^9 zPOo6kkG@&xLZUHSoSV*`##>~%0vAX})4uR0Se%U^&0&{=n9yt%+#2>N2!FJ849Z@J z3gnijIsDm3$HCYVh0*ZA`=%{;ZT&akY2|uyI8B^A+8n6~x4oo|cYIM}*&c!bA3jWX zaVoiZ2=g!6!FxgZe`wSKkIyNd`;!UKd zBxdi4Uo|smVkBGQmy5&}a+G~0U>DtapOxu9Q>A}XVHivRGri)SQpcW_y=W6EuC_oJ z-DfZO%?nbaiyBHgj8PKqnzEN_kBZ&OH2O*RJkwXl#!L{iiCwG6m%?>z@{l|Bx5FM_w-pO0~DG$S@-*wS~?c=z-7kg+XOh9UiM(#=mMCmkG3N zwBrNfmvpW6>-r1q>W~J_K;P)k8c&Hp9|d&)dFxywO%;U<$@Q<>&4QYC5_U@q@@}2d z7@GS_)Qv%(5P&k58NPN^SxdHjEH7-#ciQEgkyT^+bbGI*JE4iO>fos(>dU|B?AiHi zZly2PX9)sO$OjD9eQ}vHT+nG`w{43mf!^>ovBpA8@dN~8!|Xe`t%%b_=JpKZ1huIG zA&TktFbDbMd5-iiM6%nXHDLTthhNjjnUX)$g@vM?2imh6>ug(ffr;DwlOOMp2iGj2 z04e~WO04g-stBu}ZgZ+8Un1dlSHR{A7EWf8tHo3mfy9>be76#=9-TeXDoxrsesiOqI4urWcX4Tbh0$u%2$BlVnn6N>?G!m1c!;#;Ck? zRXN=bA&q{!j$&LghZ^Ndu`5>5KVAOV2H?@TGNS{%;iP-#Yzp#W7VZ@?Cn^1qHD})a zWjdN}tr?1DpK)c_3s%sH(aw)jbT2Roe6c)6>#jg$RDb`1D=d@cqhqg#7RT1v#Zlsk z%5FvP22bBi5mRB;)}73ij0*TkbK8PONOlq0_^1icFD$wXL8!1ettRWCH2V^>cYgUw znldygU+t4%8Gzdahpr86#)mYssCs<(*+h`oS)aI;7Wq@klZ;44JM{BL1~X}7T?T40 z-r$Pvkw@|=sp`68)Rh4qY^}nm;ga{vC8~9b&ZjJF=IbZSK@L#d-JvrwrbMAXV-H5- z1TX{jBW0#zBOeyrIN=_*cTZX9?xS1|8{^P4r@7Kx>r+?;_H{=G7y$xiWt-JMH7Ld~kl8D$*nv>Rz zR|3*@zTwxHF(X`s+}lA?wGON1n?Ly%kGm7ht0}efEA{OLx}7T&^DyF9_j&}0;wBbe zbG_rvnmE@31#oHg>=g3RfaKUV^ble!f%{dgR=QG_C10W_aV{6ZWDzqdpp||4T?KPJ zGwUKAxrwa_@kC%iw(k40v(e{XrsJhnM7b@Z~bvsj+O{OMS8eW>4g9eTTL@@wd557N69yb@O+0OPy4|pVDez9?=f_ zZhr#Zs(NhZrEsfE4feA--{bqp79R}cD=a(laTEmmnM}>|M9a=8CRFxwfYhp)bm*Gs zURcPOC*hnjk>|>{c&e(ce|r&=r){x@d9ZvOHA8e7MTJ9F^^85)>$$YqkD z&%9YUP$be705QjOEKEXm*7bEz4GRJn+Tt*cbUY7Q8v}p zi!=3v=cR@HO*9OPNAfRA^WuAa1o2{K7?wS`On&UalD~)%HAUAbcHS>h53Ur)%avd@ z%Pao=zRou6`A%RX_mr+syeXUPK#vSu=l$*Cvl=e>{u|7>xI{p2b}LfVWFG`w3udvH zud|w($bK<0kuDE|fHuC17fR6^1BZwr=`F#^aL8+pG#z?V$kJLwyNfVADfvM#B}a=M z^M-HmlCIgzND!E6O&h-n-wh98jR=1~=AZUU%&J6L-!Jj!!q=4JDu|JqZ|ty2puOg<-G`xcBJ zK2B6B1MT+$H;|U6EcgvuiT;3~OL|L8-|Bv{f+}v2Mt+NP0DwWY@{U2wU5H~QDk77#!KBJ?G>Fk? z{m35N&>UED@qoRx4&Mo(Ry)F=|iuurb64PLKp8Q7&!Jg~(vhtVA1sF7cPIwsB zRKI|k*X=xWbgft zn-@;$t|={oX8i9n_qR}pO*PY&XM(BvZ?FQ@tIJFB<|Z&&@OAHRH}hRFTP&WVI3bdc zov`r@t=A6`z4Ip~jP;=4dQ@}Eoi(D6A2KpVlcu`rt**7!Gv|DLyOlR>C$ekHU{zL?H(|=$WiEoSJ&4Wo zbWV|>vBUi|7-qy-?Ep>ax-ju;n8I?d1A;NH`~qN#wiHEq(aCXUvYt{ z$>U^^@6Rh~!7pw(O|rWyS`pQ|3oClSJQ0%4qCoyUcPY~WQ9Y|HDnvAV;WS8!hrVS8 zSW8iVD!4b{sfZH$DZZRMfu(YYMopaL$seRP8-pGb6f|%$NUjt23h;$~%S9G4^-T^? ze6IyonN94uhVi;YPOAkb-p>MvE>o&Ys}LkDY5sk)m+aqm1pagW@j172W*wAFpV&z+ zzFYqAMD2!(@BHzX*HQM=HDy#2 zV=70c%69a7g=L{LyIA3%b&s?Jr+#joxZNGDDjV}So3AVg8hcZO1NwaRUfs_8GYP86 zyYICW@?2bLcFou>a849$?LYxd>~D{*o&trPAZ)r&wzRktqg|yJdV5mS#KpQB35}TS z*Ex#oUNZ43I<^>wn|K-B@%GgYa(tV%N>KhAraUV@#W2zk{b-_Tngsh!Wb$}p)j|ik zw$B!0E5~Fjw1-~~;7N`GfGt@t{equQbzY6riuQVW=5skLcTI}l=xxrk9GpEd5*V`84(v6Ch$P-l zTJkU;Qa?Dn_A|fj%p|_Cd7IB29#v%{+A4mLDYe@t6h$rf{YF zJgY-Ofc4pFx_6xXp%7x`LEW@5rJaWHy^{#`(GSp7T=b~#z1it6q0BX%K8Lbg{LKK~ z6)Yz~IoBjvwXv#&s4}yJMfEUG`j@y|R!IPp5AkCR5(t;^alhqsv6R7pSS zHcb{yEt<*a+tpfyGkQ3w@oG9(PN0_(am2ihXbR%-2T+pfUgUHtMHk#}V_tN(EoD;_ zzB-2kiZtVaQM3lR`^7mXJ#H1$EG(#DId)P3SAk6ynA6n5vh8#H8GYO02+HqTtrm5g z{SSWCA5xxY*COhbTPf3(g4H+&3+>E*@G!IP6a2`n%KTSHRxN69o%I8|y^|XPN9K@% z2w}FnoXV>}lho`ex6AHlE~+nBqkk_bDC);=vdfu5;z>jsW?2WwL*QXy-tkNbW-ltb zKQl0EOo9>VkwB25BEGN5c9?zXCgitLr_p2v@wmIg4$1Ug%EX!or4Hs51<(42IC(_d zVc27e`DIZuq7qCV;-wD*z*zOifzufvec;=;dBHhL7Ef}6Vq6T%)&FQN!ud1i5Z|&R z)Z~IjRJsB@-E;1YG^3DBPYTkeTuEX3=+SAiTitjEa$h#J34JVm!e3NIc8!0p%Xg{d zY*sX|#)HG5GoF18;g;$|rM)hcBZ@7GjkdLE|us*B|G9ouP z6>8gcCrJE8?HC!L6@jcds;c+0bpsYj($8#_sl5IXxhVxj$B$;X6HCE+WZM;2q!sif z3>huq3Qomn&CCl&_GV-vO~ch*99X1o&Wv2G*DnUsKA|jZHUM zaF^5W-3!kJstMfmV|ga7libEiyE*w^7cBz2?=>cp5f?yk0+mE-eh*|OlZtDw9DCI`JNi&`xLA=LTP)@3*r{^A3eNs zpwe>4kGCHn&I{9ts&TgijbuyHC555P6RO#@{CLf)GyZVES?le>e|VT~)Ox)m%F*J4 za#+-LO~(nmcJRSR=AJ0L`^>nMBr(XR64a~qCvs@_2g{{kl!oos{BxEx^3lPh-7`wf z@+Xa!o))CYFE~wya-m!64Lpwd!k7GPskcq>Te{0QpF4|pWi`&cMz9UN)oodf%abad?6=mJ~337W^reamYsd?+}BX^gL_+6LL85^swZwit>-?Z7(B(qkomH zk=Y>&%P99KUvG*>FGNiB1Raa@uI6+=UhZ9F=7=?1rzBq5o%FO8iQ97bkBQP!GSl4V!dba5li4V!P?dl)OY2d#Y0%#wRz zqbtz|Kj!CDo(qH(n_KNA5m#haizT&De`9&OG>pi?)&3I9ngfzEKWV$>@sqJKkcvDa zI&BNzm~+A~Z}u@dqtphT&(=!14Y0R8+@rH!@arLL>kZv9&HB6p291F{`Qe>+N3n+% z4J9b9UR_sQ_>eJQ6lkn2DjOLSbP3ja?j!E(B0Le!UPJpk1}rpygAC51_^7=rzG{ie zpJF-}yPA&oy09E9uA)|kQBEHl5Uh%asYFQ_V@+DBDQIGt;HR!CFs5_ri>q(hLdR`F zmfRACjCmrY*E1V5XpPQzXbElQRbC<1swaibwIbST9Olz77j3DK>qU`-rK{%mYPIT| z-wGDS9sW+1VXpT;<-4FdkCHQ9U3Ast<8$D1d0<(ot$Iy0cXeEJNhr64Cy;Kxs#-zf zjOn&g+wyvC=h#WEfWPagJ{+{8a%U{6Im<_Z`RWPvAxgfsflyqs09oUJOOs!?Q~c)k zeV`-|IIEP+D5*Oq%vxB@oLo+CQ}joQQ~B4@_w>z7_WPXV^Hx_(oCuix;u4$@#*-#1 zi;vA4`o~=XsgAkA>gPJ&8tYz8PP!WTSaq&Uak)&}i9ck~EWR=evJZv+fKgSOG8lQ} zhFINv%-c#9xsewP>-p5Wfb(d(7KL(w<>7lpLHX7sxTA|i^%>i;G8~z>@gjX?R4U45Zd4S zg9%`^V(}Rq`!`eE*IZ^izc}{yiPE-x$euDorUHwIYZK^1p?O&sOUr%bZV-0$>5kve z|8MW@^RJMWX&leZ-)3~7cQZ^Q2)JM*Xn~Jg!5p3CLTumDo!A~wdeBBL{*o@o%KcY1 z{wy5Mt_?tky~KW(bN^dSAilHq7m1*GM6sLc-yVj{Glu2)XOMh zJgrDm&$UKK$bTfcz;kOO@*8q;i82#ometNfE{OwY7GEE&r`^k+w5l{X zZeF*uQUO;G_+tU6$RzMEseG_gy?6Y;O-Y2l=@bA5^ zCR~rt9_1wi4*NZo4>1VVu5;>M*&>-z2D@AwJ}PTTmHY_Iq}e)tia7cm?izN~`9_x` z9!K=E?~8OFay*+kLMi~O(ADlE7gjB5+(dje_#NGmG03w}`A++Qi{r~9$$f{*iPL#C zch>01lWHQP?bQ>+?E-7pb(W$2Glgu&)-76O*26;Zc#`h%y--v##H@dAzdE~arR`SCX#%Ft2#ia7zj&w_*o z7~7(D4r0>m|Dg$K2vC&Q?qh5dfluS4-aK`FfbKyo!;}_X(63%uO4C>GOaGjIc_YPn z1~>ok=0bectIkjKE>c)I$@&--#HaTzx&ubFj&E76il{SUV(lEcHcbHSF$XY>5DI>~ zJ3Cq)EI6ZnUceg>EBL0-EBK5#Z!6yMvm;3bBum}j@KhpwLwNV+l^K=f3D1w)LQ@q~ zb$bvt6|x_PZPNE;X+p;v6_m9hI-NKg!QUL>G{UDuJ<5y%YYuX!xaEn}$G*GT5Wz$f zVSm46iMyq)3;PdhcYbW!AE^;;mn~wGh8-BW5w8pvZiw6Cp2uD4eY-%3OiIDc{M+K1 zUpt}zGUbFZ^@tRl2{|G-?EW$)^TVG=I(ruKT4iymrrNhHQcqpUj_{s7Dxcq}+ za3I5?8IUX~e3|;pl>a12dFUIG{buQ|5v;7vbIb`dE5v?E9WYSOUj^C|D>}j)?oga) ztFLIE`xYmZeQkursgl9xE2iz%n>=QG*Jw@%uG@!?JJ#I&wc*AFT$RxA`taNN_9%Oy}x-uJ1*Wg)6)yX;Z+qw+Vul$-u zM@7mEeu!2F5jAH4{Zo^H=c|vq27W!1j^PS^=(h z)bX8$QO$PLvP8cQ_CzBmrx=zqxI^nLh;J!TPt-V%t-v>O+XBaqjcyaFvvaud-A+&hVeqf60P;xH^ z(tq}C)ArqQdW?e&c#JwYugGr>-8|rsL+d*z4*dHm62Yv5R=jVmuijI|#)JQ(%^;@t z(WR8mj_NK8-#Ye9>?LDAgS&3)eX9b?*LA>=&Xs+jNFOm}&&m^1+LB|cIkMSW*{4J@%#*Gz5s%6|4#~>)=2c$Ue)adZeT`7r9%# zTDU0r5+uj3sNVY7*9Uc%&U*d**1zhH_kMPgHoxOs1vRk<${7vJKqMM`jH{fxNb)4s zB0gYr&GLuuRkG#mmK*rx@a|Jygajg)gA^|YnRGC5qUw=XoapQvi=JZWN*rvT6~6Iyj3RE1SmKibB@Sfh)Aa z^BO_{GrPZ$x3nhqb|D@|$@jd0=R{|mr9{0DPvw?2+@+un2ia14yK8eJzoB(EqRem5 zNA+uOL?Y2TNvht2=WkpDLviq5k>jU^tBSf#ew;fQ9o#$O-jV@D<6W>A>8x;k7sI`v zGI)crLMLGI&RB@y3O6`yG1*Y$-Cwg#&xOzDjwzE8FH#Izq8ztx|TSy&5o z3Qn#|*<2(2ugXC}yuy@(Zg(WRH&Xx&;}y_Ll=ocLGaJ7~r#;^Ok5QsKkdPhscGq@a z=t59p4S!PPw|}{3`oIqgN5A_9gHd~NwNy-0a<^8@LGx`s?ZLrOgW=^UzkayrQMXMV zputcgO|RXzhv?OzHwBHT9$iz*-X7O9qjHxHuki10cr&nZMu)bccERYm5TQfgFYG9! zv;g`9V4AscQQV)EAs&cbN%o1L z4Q_RGKddy?7XSGrO!W&bdc|MH;z!VH>~?s-Eq~*YjAeG&7K~rU(eVIX<;{&RzrJ4{ z!QF2BYL=&?=Koc8uOqwL;edW^riPis&Ts@#O@XuMSBy&+zD?Ob>d!ldpdhdd%fn*_ zjow|nKjrrV)|wRb?kNPiIQBD@6^>m`$C9pBVu2X&Z)A-y3Dg=Ee0kQ41T11=5Pr-w zWtJ|90^c~Cpu%)Uph7>FV3cpYZH}VvWtEzieXb~F?B4>^+r6y$U;XpCBn@MM#~T5X zkjOhBFWp8dV7s5hZr8rIbf zU`F%?TRZDbf7ywZ{UaER-un{E67bat1F-)m$$nRTtah!={o6QJL`5nAXm;%dOivu6 zfD;4Cn&{pQL$o;d=25ouoSABa&UbURa>Drjahd22&hn}6BUKq2?}`gsgrT08q!*pV zfX0F#%I6yW&q74|V9r-9$A{GJ3+e;f_=JCFDBocAM#5Wh^KNLOGdqI|_PX}WGkQ4* zq(^e)3Lu#*!AS!8>eVY>l?4|vYz~?8bdms1+Z{vGAKy&(-l3V7Ke?<`PdhQQ@OF1iU{FK0vOKo@E zl*qz{r;PE7%J;yS9lSuui+)>uqYWwJugGehn zr9j7rEnijDeaWrgaS3-X+lghLkj^fVcSV5|RMxEyh=3$KvP2%~FTSHFF-se4mb;;jMC3y@oh$cfP%^lR2S%sWb)i!Ab3 znS)Pqo%-Fg??CP}g$C-0ttNC6Y~2V=6_itYx&=On&%Q`jwjjjykLN_BI zBPLuhL(hL$BZRa)`NwMxIm(EKxKn2Y(?^yI;Y0>^VNv6;+Xk1F6p3ck%Pdfa3#s(6 zDr^b&Qk!-7T#^DP(S_`(&+8Np8f&XzGSjUj>Xt+7Rjj0^hur$?xVA$Q_ay51EWlm8 z^yk&!n)W;1iroX;sEV!umd}$PzMR+WPWbn{9N~_1!a6 ztom0ltpxOk0y3j91yxzkr`OJ4c@8Z<*)&Y^Ce$7|?GtBh8@9`SzmNI*Z&`Oa0)jj! zQ%NL`pI{~F1{Hnoo2Q}V$lpwm$4w#)^SBd1eVP!a){Y$ZRN&Or{P>NXPGa~>$-3rc9!I6144dQ{F;8{ zhocN*5QK8Tpt?tYCycu`1V)!AN_3|Soh(Q1KXe2h7wC0 za7vHke3q$|G3;`d8~S?WnK@FkL!$c-Ec$iQXmEBPxh zzpyD2nhh-Y5lg^t8xygW(oS9dlIre()sJE(#-Lt6h?np}cE>c$4A`g|6?^z{lmZF4QAk!RSF&mi%Xc_mpMF8BB>B zdC1afaf>#kpdg}8MU*p%RSpj)f4lsJ*MPJAb{5Nr`nVlfcQJsGoX0kg$R5i=4KF9B=0Ehtvv0=x{ktAL3H01>p}_izi9*6u9n@v zof|!!3w}R=@2JWWLvl8VS5AvO3#=Tgqe!bRGt5|u7B|4`=Z8t4tKT{0fxvi6*Z7T| zOJr|k%`eN)eWsM2xT2OSbKd185@-cyDQpUy=;?SN#r^zuUH-N`hz&=&N}l5!b)2z-p5_is|E!LJ-toz z*ALiAjtxFw{==VO_o;AQ92uJJ>1CN3>&Mjl!lp_i4FwrqG|luaB2>&s5HLG_cLAhI zFIAPtE~b%7gzkfJCi(xE(H2-Ih~f5rEyA7+XypA@b8*!NXfBMjBnDV}Z~t*zSPXw| zFL5 z8_A#DEet9V;H~?s^qII-35n$QBWc2~4>{84)G^)GrP>>q1S|t1+4;F}Kwap-j7Ab) zngO&;+x|roHut-iV9z5<9UrNxDDPdZu#G==ZQ!j#>eDCeqG0Qr$yD({^m1CLVA(*L zqT%RP)`CEWmr&2jCwF$*j5Q-X+SbF|uOkisL)822 zj0zo_^j9M)hmT&inWgZh8YcSYjPM(Ki%-@UM_-8)%y}{oza*%oy*OypD3p*n5}*v* zHL{6^w&7idfX>2w4Zl0cl?8evyuBYs3$VPt6BZHOk+G;=kODP+3UR7=J59C=v}Uc zQDxU|{qd;A{CMexSbv7kF!s>E&TSXWK4e+~3U*|6%mn2=i(M9MCdF_>VOrtgeQzi8~H*ZSQHADs}0&*UxQh9?UYgJ^b8I%r`LvS8c>R-~2vy;M{!vN$Ft4hiuN z@L-zde_ygRORa4Qn%TtQi&>rBeF|7ID0w$XhwaK2nCP)U07)h5-jQsEg!wew~M!2oEpE)M52%s=|^Ko4ro0 z%Zm2lg~4QyHT#u8S&~R0<)Wj}Qv3p?foJ$~Yzc?KGJfIC;&y?J>0 z=#}8IF#8?GIyOTOKHf8Pq>+5!O#9ufH@G`qP-K#E5x&`?;x)OHF!M!3&FZfM==8GC z(jJIyD^(k0_Q~1Y?s<9Bg}g(8c4a23G#JjdDY<@<1>q>6OQZT=a_3rN8y}Wb{$$2E zL~&;>#Y5snf>?58_pGYhzXAN~EgpPaSp6lWEdPRPv-E#K^#JR$*)bGZO5Wzz6}qWhB(&jt z7;@UGF14bq#R+V$E@`nlYy#ScJi{=(I@BD@quZ>S&jgL|1Eh;36wAbE`6^;X*PgEU{9L&`s7PiPH_qR7 z&1=b&F~>z{+4{2$y+q;~Q?U1mimQhXvQHQ`n{oV@-T-WT?n9+&!IfRVhm!9f4e?nO zJy)P7&R)Qqyrm^@23*UT+C^T-*rMNQm8YffXPMX#^KDG9QRV|$A zi=l{iy~-Eds?^o-(~GMP?%X(?u1*xeyz^q=6};|l`drHHM6QD?dcyzJhvz3Ub2W#y{1$cje;&(Dw>;a#r z)K!zPvT|?H7^3v%?|i%+GXnvO*WtIlYV?aHw*7~5L~pWTuVkRDx~j9nUxeIHCP>Za z$tBvt-26u0o7b0y{9LqQpZ`6^LGxM z0t=*(UA~vANgHH*n{ZoJOy+|BIT%-=eKJ&{?5S?*+Zkax~j7|7Y z%NF^z!o%09MrzR3zF6>y?oU~&lM|+ho|?)9?;kBVp|woVx46iFtMe)aR4xH%z(X$1 zaUzO1tJx^uuuS+NryrDA99*N-mwcHn)!lA!rzhl-&YDPQ=!V&aiJv6uP3Uj{msa+Y zVV=PuEd=G9kyuy?2FlOjRL+%d@3$~O(N8Wqm?7G_2(~j!wYrTZ%8AL7HSQ}Cp8V%( zYK4nud@0(^5X|u$rNf)t`U)$-r0P;`=?V-0bX|%iKGD(T5u0Rf(6>sv6qe3mYHeoh zRx2_m>Uc`Pe}e^CKES4Ye*89`#diyzc9jv}|E}uglD#15RC^qYYfCgeeB^~kgj;)H zcA6bo3@s@Z#i77Ca?gbN@!xJ;g&3v>oZkQ2v+JGj4CS$k9KJN zQ_i3q^SzxE)XcmB{^IYxD6=JYULU?!?D=~0a7hG;XxJGUjg(n@zCJ8L3@z>(MjXi5 zRkWE2XlMo+nWPmpImx`ec{v>x{K}GYy%w_&D6c5u>|gI;J@uE56>FXBNK3a!wzo-2 zS`YY_+@b9`Kv2v!&$BsFi4Vv#0;p-H(q=_iN>Zf7q~`CTb((u$;d?)MGvRxuA~?V7 zD^;Qag3@DI3dF!X;*hIRuN6jBEAH$mzL5B{4d6}CM-lx2pgQWAJ;bl(ja5$D7(6jw zQTgt0+)<=tq27|S5%w?Gf(`E$DY$c<^IEcTo|~EM3zK44+p% z%4N6eM<&e?{q+ie*1BZ$dc}|B+MVj82d#q^30Viez2NL=d1T9s>-imO%b)Qw8CmQb z>f_2K_`SOV0|8g;&8xiTi;QnG-Y^ntmM*ippt#Mbzv_|mRm1<}C;^b1{Ga*B^ZSVL z0zB;RKY+jff4)$AIP|Ya6tef%BbweH`Rks})bP;Fg~ab|)9J%Dq5G?kzMcm_?SGCO z1-1F;0R|*CKa(=4!|I85`#q};^WW41Z&|$exb|3pg4X)3L~rW*-`N6{_$J7{K<;&>Q{02|$=kKUn=H z5v2K`A}jxC_nGfLQA<#Vk*{yxz$}U;sN2ZyKBDwOO7HKL`zmcL@uUR5w23v?OFa6q z4QeS=*AZWm!urCVE5pS*kJP~f0Q%RYMb@0#J5ecSqvXURX?n;>`a!T`4k*F}urWWj zz4P@iQ@{5e17f?jxCMY)N*ajxsuja-QQz13PBJsYkhG$W@t3&jNeE*9i+E860Kr{c zjys{|s$hS%jb$SM5IKLZqx3x~m?rN^O-+G?*54mF$WaU#oQ_b#y-|sTAI|sI(0&s>aZtIf4@A%uNnDhfDyz(Nz z_+K^gG4)D^KmMy}Gr}cjWw}7f-sy*qBsYRS79#DcN9;q%FLr~bAZ+S?8R;+- z{GzZ@v%UmENF|V&)2N)n?NA0oy=O>rTDiV5i>TcOi0V0yeM*KJ@;#8bNbbs>*AEOm z;H5b8YWb!PExcxz7zSGZMew3Z!&FDvng;(#@cK_!X!C4UdpXL5ryxH5(r?4Hw?Gah zC9lwGff;d!&jI0VbRBrw5x*dk+9CIVg9+5~Q#EUMTtx)+iy zSduV)5USM%Jb0Sts69YV0p0U6sA<7x8@=hR&pqmlA5cOE* zj{hj|AuaoYwz6t};5&2& zdC((3LRKd7d6z(`w&IF7!s5N>{BwyLY~u;ri?1&#TJjbukmJEI6+5v;bCaGrlW1FN zQ$fxPPef9MLhuwdyPH0pvlK4jk3N-(iUwC^gpFz&Y}}y4Ct^s|UTbN`x(vxrD!9=D^5JBO0fZ6SWOj5&GRN^yr~>d_8%!UFt0&{gKqym+@e zLqZ;JgsDB^3+wA#0xlq5Bkv%gjoG#8g8lAvSWIz#UE+kl2*Y*EQxvu6(F!~mdnx%j zXjgE+ddv}{f+*%#@wR*k{eq|s)-{3i)N$zWnv4gfV(+^VbLcv;lEr6uG_5@75Q7oD5}J z*Is|J^)z7qc59s&C&5`yTsFZd%c@Ijps#W^G${XNua*IyA|vz$g;Xap#iNDm4N3pY zL}NZLqsq>Av!_)ntgH0Nk<+MrXc2jAmA_SYlh-KQVq>dc&}SEMwJRM{ZusoB!@I2S>X7% zs8ZFs2)jmAc74xPdm$7fOddmk6GJi+82jLav8xEyn}YC0t4 z55Hooq>lPV0we8*rw=v(5>S52nf9FK2YU!GxT8y`x%bZ2*)9KMSxSX0Lh&6Gap!|gge z4uS801nmE{pJPS##jktLJHH$rXj;Y3eap*DiI-4D*0A?!>bu|X$rC#1TIQUKU0qq4 zy%_vO{bYyrd!jebav38po5p~s82;jU>D5lo#;A~uOZ)NiV@@g8cIVQqws|Ancn%iliu$G;yJ8fTDeEIbaM%)-AFNPPW|EYl`;k}Ss2!jvg1X}-$8nFZ z(IlqbaWrot0ZhizUd2me;GReUKL21f zdX}vgL_9dch|$wLs>AR=BOGz)SZ5&l+vcfx6?MAVagI9@w!U>W`hjSBn*?X+oy>E# zml8T1-!03#v+PsyJIedD@o2}(xtL>fQDmnb*vG+D)cb-gw;06D-9y8d>e6jE=thao{S(DqbrIMv zhoGc+Uns%2@V<}{i23oMczlzNMbJEkx~IJFid5t+1iYKT%R9w$eem0%&^gtT$q&tg zQKXY{8ov9zNx{{E;xsc>VOA6&k+pxl>LGV4t+$<=kze$6JZR!2)QCzKw^{fzPX`op zM>WGFwnFZ6MJxphTd4-}4(SlE0gO|NPl@M1|3Wm=Og2N&o%n%xQES1r0(ZI zo`Ow^q8+w9O_r@gVPeDb6Ka%dV%F`o+v(n``~`DGQXeWy@=t%$7Twb(I-@^CsQ+))Uv-{XuxhT7yZnf(LX|!^bmbIWYRw07yXdi zAzLk>CbW?N6uLUS`X9AiVRhi$UaKdTbz|4SR>i0?xpGTOL{5qWDS~;>n9KKR#zmfR zfF_n9Ax|5(lmkO0-Z^nDRVNIlhzz&{Ikt=tR|=vI*gK{oJNpl12Mr*n=u4a?Zj>;zF$YUV zL6UW72z96>8~hB2co|R6+-S!qj~YU-L~jK^zki~P=gR~15}e&eCqal6iB?Jz>s=PI zc|X24)MLF=J+)3|G3jNpY^&v1vAlH}e#G5;^hu>OZR4!Vn87ezva&tyZf6h(ylO&D zx|KXFNqYXiH2^HIoh)gvyUI^>N-5?=|3GSI`A`8G0=DbpM~di2>r{FW!K;q=Vp#?p5AK`_pD1^Fp|w{oTU&DyZY|PdrmSoGu9m zQqWHtSe1tF$fHT_q>ea})_Z;RHq_vXE;*B$2q?il#bjEb{PTLL_OxyvS~eH*`smLY z9dhci+@Hu#1-aR0r(Wqwj^R0HvvSZj-mc$yO< zPW!ZTAAwZ?!d;@D(Bp;L4UIw{P4Oke zKQ*Ty)9-!UI$od`{KuwEp123^`!!Uf@n}15KoQm6mm1}rPD0=C$ z8ofd#3oTOl>+z`Uupi7PmA`Hi<74xu_bUjz}0 zt8bV>Br=VRmT%x}@Z>pBVwUX4Qy*8+L>!QIHjZ1{O1spda&SUrQ%qXV#=&um$~iEB zT=Vl#I8bhtOA@t0PLM)1cK@1gB>XProL?caUMK0~j*EV0^DNHuXWP5one=IYdAzXH z+i0`rb(cDcS>AtULXo)?ias($QXC~BIrMlcITIro zkzP30`BtgKMj_u+?RSt`gz0@%riO>9JluWI;g!B*V>veC6alWc06BN1@;|31KT<8( z6I2iK*-47qZM}h^p6Pqf(SJ2RWbql%g-FVnGtioM@+{=vHRSI}7GuAe)qj2%Jf@4^ zyd)`lgRR=2xq?rq+p=ac>#0)cL!-Dr4{_6!U=w^&(ZxxCp3>>($>zBh3dZ`Vn_6t2 z=^OA~i(fZn2u${_+PAv^dVu~Y`w6acC}6(@tIWmc9s>A`k#L&&TwZ0r#0$YCg&T8ns_0?kGxzIza`Hx$y7Qg2~0i65G))E|-jx^b3PRCuGVz!mrzG zE%toKgh(&U_+N%gZbTIoO%g$hL^Tqz5!^2K-2G&+X6ooZp1ejA%&Y!rk4CiJT;G-VE z{=$S7mEFKytseH_y0~RCHkn8KW8W44VBQ|ln^jbkSw21BE;R-2(!TTi%;!l;ty~}> zc1o#tP3}+v0X{xo^j1c+BM;kCuD3tg2Gt8?cN0)RaDQ`IXe`mv1|GKd@zI6Nk#9WxnTsh}pV*~F zmW{&W3cx`QOsVcRHXn_JfoFPv6F2~4Vy=bt4kN$1;w5vN+g*1~hRi}_Y^f6E8vt9*3(#1ltwWLyH zXD6mqXN)AL&*T)omnPG*{P{%l_vUTX7ZGl`T!{UCPBx;vz-M}9*w@R-2t4u* zs;Uq^sm$y z&QDdD3Ld5(8Zq_9^u%k9XY|6$UK{0PD?Pf&y8@Xn7Gd_PQWs-C++jO%SMc+fS&y)y zdx4Tg?eJ^iUl(yVUorn1^Uejb1c_BXcqkj=oQRI(7=lr4GbdqNK`}TcHHQKypO~ol zCB_IPgTdeA?tQnkM!=_K?v#J~Xxc6r?wGhd(s%SrOlZSmMu%3nN+reE-k8b2LpW%B z*K7&B?)Ije{RfuTW_8uQ&vy7%h;3-|c-pcEjHSQ{C5GyI-Av&DF;asjyzTZi@1K+; z>-74^ODt;ZSdb%neW+=j>lk(IU7pA43&6sDwHke7bs{4i=V1FU)XrHNX%5^phY>}2 z?J}0ei|w|mNEFIZ@h7z)=T(Hqn?v94x7i$-F`KKj!a;U*#;TB@es-iUZDef?aFluW zc@_+z0aES^uB*`-&t{WJ@Q;9p1i<7Gnn$rzMn93mR82-JCKOs8FFK(-_@&X~VS1#o zh3(f6K@L}d6q_3f4r)sT~h5@H`S?s3yLx!<9TCVGeKX5aL{T@=c>soG$uq% zk3)(3X)N6NPq@sJMW^)`%kH&tyqjRs`e(`bj0{K8%6~$MMe5|#6kdU#TFiQwwa-DA zhO#5LS~N7gXlj1(QWJGjcG9fKN8M(sw&;6uCU@O?W1mOo)cgKkJWHBh^=1Nf``DwX z<=4{$T$I98mOw1WAUEs)L~&ABqObjB9YK~gD7K@exhSgQ>v9O@aGp4Xlb0oTN=5lu zdaSm6o!eB-w0C8jA0X+#WvjeXN7VJRglJ<7Vm#V2IffrLjOd zRH`ZrkZ}N;f}I-Sv#KC65Ppp1e+e#}2ovsWx?RpxlGbmkS@7|mJmF$^t+46~H)3S- zJ1qYcVVhCdfajJS)6KE&!X7f|vV03J0*y#@T}9{xlX9flQ8N%fd#;Z)zXbVSV*?yq z^`O@g%M|yM#MuQ9*lRw6sM}w{!gplMR=p48f&9Zg*S5qlsJci_Iae45O)pz+I6dMEEyU$_@LvJDDmPFB6?RB&^PUlx3+^X?6O$k|&7m(FVB4fD<=z%< zy)7jFUx#zO9}t>az~iZux1}}aNoKIQC|MZeNwtgQfYi;sv~gB`U=Qg*?#JwD>ph{X zGSv6P{S&iYDP8`LYJ_Xh+vA?uJ$8>WLRl?*Y^^MnCWW>1+RT zy1Z*_lM^^FV%(_K5gU&+K8H_CsD=*E3%>f*x*4kyI`gEpbRm56)n2DrXe%?=>#x=d z+*JohTG8o+B!~frfx52XvE&8l&awMRoLgtK_N`k}X!m2?X+4YK$W!fcr}d&jh2}L! zjvA+oyuX^6Mp)FZi&|=BJang41@nudgY3ElnMAR&})5j?v+f^@j*Dy<#e} z2*FOzn6YuVMq(<-8~k{2-zkY8B_(dfjh6Fs%WIG$eYJ7ppwmj}yR4@`W#@6K;pw=k z*+}cB?#+qv)(tvF!N|;GZDz5WhWzoDPyc~i_f-82x5ldz+mWi|si;2uycwgW*D`z{ z4(@Ic{9+tLo84bIA!ep)-sx6Bt0FScS8Ytjbg?BV7eIgNmb3icC!++31}LdHc>D!6 zRhcdAb&Em+dgZP#{KTY0+g=ya+>?ROo{GS&1eG6rD;^-xustz{i|8Qhh@+--#}D_| zp>+n9px3I#*vG%UkxF8z79-IwhR5)L1hC210JbeOQo-{}JI2P*_pO?Tr{3$%4w=m> zbbNf45c9gW>9a@7^Q0JaoGKV1W!2-EZ?q#CYP}1Q#RDh7W<~4#j0AgHbLqWlFUFS_ ztZ#f$ThbPtZi2N17nXVsh-~_<%=Y?=6x(Nt)PzUfW{nkETpx)(-R&1!Fp#nQG5}9M zh#`#0I8dzQYf(yZ3)b5^rDP+=NUNg3lUbDpFu#6JJW(ZDo-YqfiTB)iCZX>o!kTfE zd$gCF=lEG`DF|94%L=?F0?=+9R^KfrEl6Kqw~7L{tz;iCZ36#Q6E#6Jzo)aY`eLo< zGG;r}8*A*uIsA;hi&Y7+=D#&gyZHiE=*%OT*Mr_j6r9=$YP#UfLX`Ng z{r(8{RsG#tF+SaW4!~b1F1fP0_V~$?fmeV9EGME1eYiw_N^uE%YH;wnc*p%3HY&00%%7~+KFYHMl<7kKBc3c0P z0DuqNhU_)3^$YwZve1RASMos@v?xdi`xk$Ipc)~RvwI$BmH^D*+y#u1#~uU>09bM< zb^A$ojZ%Cqw(Pj}uB-smW#OZW)WFATI?HBXTM;i|plydZhA@B&Re!*r&SD0s^(mOg z1F~Vt&`8Oz>!Sg=8%{kO&k;ObRl0JhCp0s$R&AbTyLsp&S=B8`Z3 z?E>R%p>zg)LC2#tq~5MFgklqMlw<=+b)ao|K}q!K?Vz~;_Q(boapLsQm|$?olg{fB z^9Q&SL9oHNpFKqZT@;_wXn|iCacHF=kl%qfKQzfJuF1)=AEbSI?}=2iedC*UZ)Wq~ zA572YCE)vezpyyCbFkZAfNa+Oklfuly?zGw2dD`DR`rc|JGVB##7zjG7=5s14R;1V-dO^`vlda1ISu3+|o|K2U*f=mm8-D~!297S1r^cZkE&f8!R>FCMe z=rhSc2=RtrxaTwK#|JH3aTe#Q{VrpUg6lBA-(R4Xp9Ih3k00z+VE&Ly*p_hAWaDr{ z%_invhlBejgBdECs*U*#S#|wQE!3``++Ed2ip4-{7m-_ot7S>Sm?{;MZuSvE@>9%@lbue9Ec07wV9-m^nja*$j@-5CZJmlQ;+&plt4MeVe z@cpg;ZaYvyFAgldR{20BHnVn(l15q!)_YmN>kFFCt2hD;V<&A;kdIjFGO{-#J%7Kj z?9#_vJ#&M5;y&MrV|iMWIp>5$Uxbj1A&$_>+gAEXL;JlTVBTzLAmfAsDLJ2!WC^UZ z(R{`+@7d#SPreK^CB?oa>YD3$$+AU;0N%AY(I?w-@q!_qXSdNz>G|2%aog4fy_(z) z>OQ)k3X(E$kz9P%XaKC~7g*Tc$D*SGRxkrIufxtk()z)@%d4w)s#*c$MTBc8ASs@{OY{N7Ow4M$z_=zCcPe+2h+ z^yDK zEh8Wa#+p3@rRZ5e=xoeF6}oKyTd7+4Z}yQ7v{$xA#zQYih3oVk5)h$IEgg2ch)Aal zA4SZ$mZIPFf6I0X)wpz`kZ=5EjtLo7p1Nd!tyqdrzV=pg&7m&m+N@{HADAu zEsquk4G^7@XIm->?#$Eic;R0v@1i*|Um<4V+ao%U0CS?fO_Q^ivgV;A&D^#v)Jmgy z%O%s*5UY7x)$UuC_{S~mPqY*dUK;d+#7op(L^Dc4XcDi{b3>n<3C?t^P}hTs%wDu} z-r}jln@r>8;y%a;0U+ea_x_ch1g47-RC>Mt%=7!zc2Lp4M-EieiO*ybJ0V5b+DYWZ z7!s|9|7`nP$KiyeG|02%qZU|TfxA>S-!DM!0$IvYlEVBo63xF?}%$_~W1W4KP1&5`WLO#tRII!Eo18pq*f37IT{NQVJ zGd&40b*f8t%>AlohNRbCFL5X;VW|#8LuMKE)7b{%VX{AjisOVVMHiV%tH*=f{XW(~ z0RgVn+`(;7&7$Qjc4W<|O^~zmdA2r=Ha`VPx@`y6nv2fgcpwEf> z-px%wjWPI=a!;`sypgtKO0f{u^NGP5fX3LXwScw=*O?ZeM4$ch7V(2nN;9)s$hEt< zM94JrPK8D~@*PdyW9Ut^ejo7qAA|gK%c(K5fyp|G%u!Ou$wx({ISkOrUrkW2iY5=p z#D#o3X>CxgthOt0oS^(eyrc(BHZP*5aH=(KNc7Tr!tk%?5d;my@eGh&VWJ}fe$kYB z?s3#62XsyP(TJfIPY4o+ZB)?w4{-K7+_D-A93U~uMnl%@W3iZV2)@;Zg62dp;fSk_ zv#{u;3J8{w&_P!;kKL@-dYWH^pQi3s-X%C*ZJa-kcSLywq+@M9YW5yjJ&IEraw!Sl3fpTh9A(U$yac*HDk;1fi ze!_pA^vt7(|8IyqcViaN2>ySO`t!f=8EPUxteX$qU5B+LaJAOi;8ksU!I>bBKo(jz z{4Tx^xG-Abe^!E0Dlsj-Ve|B88EeQ0SIfn5idpL(DiYV9tM^Q+Mve8hOZt4&OuAx% zjT#jcde`N3Mw7cqzJv84PHSkh(hiom_}1B;mt)$q99J+dO@|s_YO7Q~su>qy)%$yC zo;hX)IDP4fv_d%t@u>nq<|1CNx*e!KB~=gZRKqg5d;W}GfF;-x(PIR_GaU5oH3$wb zT~IE7kfk&I&wH76v~vRCth&=!?VPeeGn?d0>dP)NQp$y*Vg-VLw6tpe_Xj`I5_-wa zs6u(1z@g**C&Vl|qvHsWhGu)nWp&WJr^;Q)`}=tbj-BZa1jWB;t$0i-%h{XibD1pq zEqNkTtH4y-0}}Mlj5k^QpG&Z>-DV^E+6Hi(42MrzUGvtWCbD8o?768}DQ^5T#lNY3 z2h2S*32zMsBUTGj%q}*CX#DR7B)o-q?g?sKA6R?jbFJMb=iFXahKhEWRy=-weIrZ8 z$OpeEH|*wlTV&Tdg@1mb6*X~4t^{J-X(e2x|9veGSWjTeeRBqcpEy7AOb>{i zDZ~m)UTW!P8Pxv0=6F*ygb@T9VEeFuT4I9a?@DKmaei0q~>7P#w)vV#>`1c;( z{%11&{|>wLzwiJ8$;Tx3b5ZA^Jtf?%PlcmmHmD5U>M2TrxM@Ut?e^#<0wkZyfchk6 z()dK=b6VFdVx?AL2xR3=dB@+4*udu908%S~wmfGxUk(`Y*Y?>pA6~z4K9J@`v47L` zf2hDNdHyG+C!Hwrs3@PYzz^8}>FeOI6K`^5O4MOvrwAJvQRw~z_9!l8eLg6AGhZK?^W2^~1d;pXa^YUS7NC1h<^%=77B46`Y1QiZAw5no(b>6iv+4jZIkTAXQ zmtI5x!M5#`FmYzS4bZP2b0rzTsoJqdo(v&~PKIJ(n9BR?Q)sC6Z()gAk-CE8B)2+d zb-=7n8LHE56K(`N{6+vK6#mXcsRy1~(^Q=pe5qN9vV_OCs z)YD;r2J`D5&p@>a0YLG&rb>Q$EK;_B>c58+k3EdMR?yCQyQM37*$H@>XFrSI8v%6G zM5{J+lI!GY)=&LH|Xc4Y9%|#R!Tbl}H6ImI1*yLP#-((k5 z_8+PMH2_cS*yHzR?5Tbm4ybaJAH_NYlXm6$yR5Z1fMhv$OB9aa9-v~I`NqlugIx^b zxJN$E-O9qeY{@YW7b{xf+mWT~iHCk)tSi^*Wo@4fybo0}-M6cdZb^0Ye(0HxeiJ~O96!~+hN1xnXLzOnD^w>=o6UDW(9YX> z?1I#j_G=03?&{5%M{d*~PdkGECnd>FXN(+X?$DA1FiY8hW9;-j{UVhz$J7_cM4FkU}V1oKtCViC6cpi_b z49Hp9s8jfX!J8}+rqDyL(TK{Hp_QuC+=%>MTKR)#!{ELN+v?e4-QibSqzW^K_uMY4 z_DwVC#g4RxW`|L-HW+(zN6L9-cL!t$=yalLSN=~)U8L{#`NtwNF14B#d74TwSt-3R zEh*sNCY^|xeuP0d1tJtrR5g0lc&v2%>s!eA`QVH7r@AYMgh2m8aK^5!b}_WhMzLx9 zgDSX6NKNt2v{9l7c+0zB2{f8z#e;7yUvxq1h(z)~nHwJe8V<%rwb$-z4aWW{8vl0+ zmbTZo-A}K;Nt0+t3qjk1_mY)1jOqR>&IPbKd9UKP~ znw`brgs}C<07j+Pc1V^h;&g?=kvDz4-{9xSgaD?NAzb6B&y6OTLUzHojGZr&mPvr> zRnX_!x_5A^+DLaUOun@u^C%d*E^C6#mPlP&KdAPXssrRY7~3E~>eS2Wefrw{#E$Jj zGlWuCM5oUa(&2el_c0DaQ9#94JawR$OrkaN6N! z_;oMhQbBcWws6d3-UU?e>Q*afSnd{@yI(1uYMl;aIq)iT%j40f@r{$$4hT|N9vF)$44e?xHfgNo*b~PzIVgU zi2>VA9Z;L|uRU^cp`dunc)DvN9(2T!K!W9Y#O0KBPMKxv8})EEy+{u=eEweRWwITX zyM?txDQe-Gf?4W7YexPiHBfuc(D0_z<3*eM!IWPIfp%KJ)F+){e_l|2&mKT>zNsq@ za?@T@1l#ld_x2(z4WyZlbNdPS(u(*duHQqnx<$hwMD`k453NqE;q8k$7Dyb=M?6`r z+z3+q2Y%7+XY`_~A+oGkAJy@;|5s7mOnlfjq!N^=OSJdekqVJe@_1!hD<0(so`00& z`vX>?>eSA@ic`Y8(J&MD{M6>70s^=$xs2H%h8HTS(m4Bz(P0F``BBXLi|)(!q<8gA zpEX2j;=A#8Fi26fIZ25#xe9G!wW)l2SVVb``8;olY|Eb5mi+Y>6I8x))ozaGt^pwH zH1UZkI2&Q>(k;v>gI=w%KIoQNSd@QTlL{{=x!N{lua^SNl1_>`ua15#@B z?a@}3?l@1eAIZL>l)amNa(d(cJjOU*=z4BxYecwBjk z<}}3?ohg{CLKiPRe(_A(J7cC?QvgSNa@>{sBa}nWEI8W1V}uKz$Uiw?k=l|qRs>BY z--byE#~k}`1#{xrN@6u_>YJ%leC)Gm0Od6rbPz~A(~7XW1&zmWJc*&}v<)Lk)!A?V zd9Lr@7kAn4#ruq5q67k3K&T4MInS&3?Gj=whjj zg_&cxtII=|GGK=*X?b{7z(>uI%Hc7F9S6m*2S1V+0hfEZJwAW6F)O2J8toQ$2ni66 z(C^;=^1-bo`{^7{+a+JOn6KVp)-Cxd{|ZK}dWzp*V)a;9Kdis)^t!j(S*pgR>8rIp z49J0tB>3lRZuiw6-$#V(PJdq3RT2yD?QW0Jq)zUp(Fw!o1(NWV7J6)$?PX| z&#tjHd(E*YLzjoGCd~>b#9(X=h|>ZL7=l&NqY+(d4Zd~0Lv&B zA{H&kH^pb_8pk4Ak>iDA@R|39B(u)cQq~@5TJm3nbw%GaP>fB^M%>b8m zaSJP0MP$sfiZrOD%Dk#u`k#=b1!ETfh9qFKnG94MI!|n7^2W35?y+ABIdEENGB%oR zwC4d)DTNE6OdWtJkhX6nm$B^riY$?8H;bAV46w8Q`-34cSIB!^t6WMu*RhBFmRl-j zMtj}AR$_84!~>nqf+({mS@g20wF=~O5N)>=fb6SZRi5z&g! znoJVX!Smo6eTRKg4CVYdWf?XjguucHys|`lR%#DK(unV%B!vH3A$3R^%CC&9N5wl> zVs0jAFqD{fZOh($!CK;#Jfbw726UzgvOS34%-XYdNRfhRi`(m|*+XJtYaLs&Fz-TK zSi~%T_PtuL`9VcT+{Pck(L|37v?fsBaE3l`UD{g=si4$?9h(|k2`boE9kS-oy(`6V zfV(RP$Ll?XFRrQ+4bfx6wwvxxPh77SWqHZ5xr8 zuvku=8I<=o%4hX1cV#LFrDX}heEn&20>FJtgnaq{OT>Rdfiu7sUkxkn6(n4+iEFQk zQgqhwWVjaYnp1r*>6|CCI(4XMZ}bhFH$_pzOnd1pcgi( zq2rxsQ=+g!cDCFmR88SdTS{BdgtIs8ypaUxdb?&r5wV1l#tndt(L68ZHt;C3TZNBm z+Ng&c0Ik^X^5cr+(YLD|ut=V-eUg+{i&>7(P|%$+_g!flIAXE?)bI4gc6Y3 z1*d)G$HO#o-I+w#mb#w%&h{GoG<--lPp0_Z_IY~kX*zBMw|r-S^(|?ZqMDQHTNWR+ zw5HVytwt~&w89vBqMA&xWDUJ^m^jg z<6i8P0i}e(`R^hPUV81m3?K9D91QFkoJ4OvK{*v|(E3*~DpJW7iIuiI0pu;Bxg4A4 zM$V2``tml1$BR+S*WGfd4|O;QRFiLhgCh3DwZAe-o`U&j#jREaXa<`wDMAPzlJSih zi9MU^Lo2zATg5 z53o+2Z4SqT{E<3Zi6FX3Ur>r&nrPt5MGpKyolZe!i6<>e`GG= z%{HEM6*`o)yIiTqe3QHUjmVb~lXh-jN(j}im5lf6tW@^U^*4e#exkt|>M$&3*|);B~(BosEY()u8*1?vSTg^gP$k-s?51w(*;WQ99b-XT0T z-Rbnx{-uqut60%I`?uRJp9k&Xyb>waLGG)T!l(ldMv$j6H$<`99ZT2NzHMjVcMlo# zFwcd2^y<(#C1{(X!rU2q1==lV(5q9`>nT4-qFtgX$5eamT{h77b-rj#FMPOCgj_0B z>T4I-7cFQXC_H3;X+0R~ST)6quWGV%9I`g%+A{%JFcl3nd2ZggW1v+Gnxhr^gfY=^ zzAO~RumMqVSX^C~S#tXR=vMw1ky+@S&Tl76h(5qNdqF=v;IMR9q%;78vU2Oc9`GR( zzMvUl=GNl*J{{0HR=1Sp+iJ8W`d=23f^8a=u9)(=#3So=UQ#Ma2Jhl8{S~KRoES$p zpe-hv*G!vh#nfGev*ua+{e*skVeIO*5STeu4B*t2A^fLfe&o(LlPUC)K` z7QoN@Z}NuGk^tVget9+JLo#7nL!j}}WrIHBI2 zQNiePbH^VA%87#INs+Jhb^ZA~kw|f=M6^4_XG3xNm;NE>K)fU)?+`lMX>zAJkUd>Q z+FKYLcZ}Y6GKV}k=3nt)1?=U?xnwJrxPx^xeJ|ERyLc5kO%!_G3~tuO;tYcN6ByEk zVAcHZ^mjcHn=zTn{ed*q#KLdt0I$HHsM*zc?SDc;?p=R4$dy{zZ1^k75O;G^4=-*8 zO{@pq*4?{JL(aaWH9)+%LeK8@9AQ4;h>-ylky^4g4yx}oLou;B@g>_7gw+4DDQ5*uY{3eTSES$k0Pwud|VrbfA0d%K{@tT2IAAB@m!79lu=X za^M$lFXYy`0;jgDzZcis?O5sK$dmUz#{9+(P9b^Q)#?F-xBmxII1(Bd{L1fo|JYa`s3VftO%3pd+SNTMeRW8 zf}x^krzd?)yb0m@GZHHd8KQpxNDV+*>#Jldbe2v=)N?v6w?YW~F^>TE0=T)zoeF^s zcmp5S`S4Zj3j?Zd7>;dw+Pak&eWyVLA^cfAiEPAzJrMdAtQP$c5((Nq$~V7!$c^$= z)hY<@jlud`9rWh4=Z77$@&32pf=(D~LQMz34B`6iXh02ikOBH$AE5vOh-XGit+FBP z#UsE967S(cCbxCLO1cm@#W+_ofgCmUsY>FxoTHxaR)qc)^XOfV(G{q}4iGWSN!@Cx}F}(?JH`4QZMkeyC29T_;@venw*`EKD|GV`xRM=)?S@WlrA4VDTLd%RJ zaZpUV7Li``?AvEcY&4@@NAfWf+|?D+@#ipGT$ylefYakDlxTQ_{40lWIL(ISgr(F} zp0m69#UOvs%EX`VvJc!L1i&zB0GdWA!OxAkp%`xg$H$Y1>%Vne=MBGW8@ALY_8D0$ zzUNr4remvr6rHB)y~?`8XV+2qAO?;X4qxWO>O0Adas z5Lar;6FmgP3EmVB3tN&3nerD(hbkz|;pACsd3kL=yu{&<^c?}PHd{b%2oal8TTJQ( zMfWgk#nG1TcpL}IdM#!dlG`SoVjzDZVEMYmg*k6EF`KmqhF^>!Sg160Mt z&t0nV^zWM{Gp7nbQ6jj43y~3k#W@7)*M7oLI}r9VdP+p~LYjh$G4IZEo7G`5@|Veq z=|q{$WGyw2$%)pVn*!jr+#{a4H$o`nS8j@k;XxUag#$^3>(|m=N`d&LoP3LMH;o*B zH%*6z={+DcKt}AM_pV_UhqK0A<4Nh~3j0=H1NKBT%r{WDv|U5EU%Ms9S#|E6V?lcj zf_$GYpmL^x-y^ePR={%Q0D2gl06+Bso2 zZLS}v4$nPViG_Wpc9qUwzla+sR^Q^(Q4T^iPg>k1vdhbU`qgpJ0#P^F64Xh82CUh0 zsK0|UsslTvYhm}36TGp6Q(TX0pL*C=Ogy|-=tML@51Cm_kcbkkB)^S5%zS~z%4yJG zQXZZzF*w?m8&MEAY1EifE$7ftf;SU!!8GS+(S{A_ap0E-N!wu1&6mT(kd(Op%4U9H ztv8vyH%f+h~r_D&^31kX->EIR@BAM7Nn@CgM&&m{(@mpu(gx*1x z>i00T-PuZJYxsU~>1gjdESH2D8IQ?sYtIgRFnTb(EmwacZ*i>S`LT4Hj^ zU5%kNey!!>{ZxcJcGCk?`2ik-;mQxcrr?ssBrWVwct}Tdroq}qJ~;Lb({cv7s`=ul0t-;9I;fSUgl764%Yswp)VcAl z*V*c!K|$FtdDB%|dd3@^(b~-1E9pabiNzx~7{$@Iu>*P;UrM>ae2Ye$NS4Bx!Dvjb z9(NwA*;@Qb;%%2#*Mf2?J0jXA!h7i_V!u^H1Dl%Xh^1)pwISfrZP&JwaOankkvnh1 z9z=a*UFHi@ngfNbU1>e);bc_V_KeXQ;Ouh7os%V)96ERWKuq2I&1}JySQ=!w;h7AD z6VZ&FjoUqer11MW)-O6%Pg}JMUh>!oeCjT$tAz))l~Wjto2SS;EImt0Z{2!rQ=<;m ztJdEFyb9#%z9*z_228;UG{Q7OdE4hmr)tVzsXv#44VZ|+b`3Kcf_TK}u8855Z(X09 zPJil_i<3ao_qz!}k_~j*BNxPSjN@3Nj*@6p%<|IPMlG-X-SNmY{YFd0y1`}P1%VKR zpW(avFPF6bd2mhc!$5FL$7CTU4>KGP9C!m= z>}_m>EDefcOlQqpS>1dA@xB(Yw7UAg0xy2MWa6(b{p9@3P6wN90c{-VVAh@fHJ);| z)Y#&m8iI^xo%6gr^EXuncf0|WX86#LcVJKz@rKpw)KdHsPrPnV(os)kI}Fb#-f8Z8 zo3@Obs$b8>E7_SgPfHcxJ(szZGT-@0e!l!lNav`_uA}`e-~jSCapUvEdN9>=%OyI+ zOthZOd2_j&Ni^!+3dbx<%N;f?BiyU)*ET?CFFYVvsr9JqNCXkYVaQ5=Xnp3K5ra~M2w%S>iyanNKSYHQ%ST%@`!p_d?s)~~M^2t-ew8pNV@IwP-tYiT z#XDv4>X) zN964qC!{jP!kbfyv{cYHD?`CmXO@GI7j4a1Gdg5$Wd^iv^R|QhQJQ*SX@<^QWX$+= zS5(&q0SkBo?RDIP8{%0f%+31;y#yJhTl`oJIyNhp$Ly)+RpAMM8c3UXv)#!xjYikj z>Eo$&Y5^+xIV6V`-;D|G0m9tD!4KTD{`!qg6T-FgTCCDA>#6hwx6R8jPj*Z=*jp*2t1+aQ^yTI3QH3@v=C=f&k@2E z21_l2C4P064=fq2uctslhXM=~rwP;(Z&_{$`-;f+RqY;YCnK$T{*!if>KN;(K+vO? zYq?~lFPKdK3?y)1TqTr(mXcsOs?tY>X}4#17($U&lE%$bdPqyV1j9%>)<7~i!Z+Th z`}>gz^7mN;J3Y6&h!4ls9pp>e#j%0&pz+q*8otKtb5)Dn8H(pLf(mr{xHh`zRiXpD z_4D8B4_?KJFcWboxND+NPGsdnQw$=${jA5e=hLUS`%z1z5W^XMJM%^e#gS{b{o%!+ z;J%V8bt2zF!sV<$wAM8RnF$jJlIwMiqoD)4iX&u&34}%USN!%Px?%kt-Z7}I2}{to z+?xhey5{*=1f?k$NhcuEoe15*t=jaVJsvOKR>U*ApUzJ12Th_2kRB^aw3a>6v8ZdB zF1U1cv#7y(wzE&u1w@rChg|)&e?M|7ePJa0i_3$POx%*B(Gxe>-(mTb8Fy=QpaKZ2 zf^n8hLF>OEe)r%KT1+ZjVUHTogTC4CD@@NvZp1%)?dScAVKe%nTlk5>iNR5t3eYTyZW3+5F%u*7M5HU^HUpdrgjBRUHrX(4Xr zQR4KnOEjxAH7dp$%9xBCQ;xR_ih^F^o?7E)54)<#-)+m>TkV31k9b zFGvJ929ABI*RmjqT&M{}B6q~<(^-LXke*C6(b@!(lrl*-%Ci2Z` z>+YF*9Bdrp{MFL|xT=0|lf+w`_6D6tGaHS#+PXHA242gm0*5QidQnXai$0TZS5EIA z&9BXB|4O@HCvW~G@=FgVwytImmyRR#?ZSFk&T`ic)=3$KSiK>4FSl0^3$5`Q*!q_? zBB=A_OW}?|3ptcTny0lR7lMqO+8t|~h7V_^`7d?QF<0P+){`?QndSM!;%+siJp=sy zFEdNMSN0{az%eG?z9ah*tidnyA9sGQvvCC$6hc3CuY`1}ev5@%3Y}A!H7xze_8wCa z`13G{@B$Vd=$PX}iL6IIHY;hU!s>-S}v@8*G)qG)o(i#}sPqi3Mw0}ywxX~OM5&I(t zt~7yMb!2VhYFf$TM94Ph{5WuPFY#Bbx^-vNihzRH%+x~xlly5!Z$(^@dWo#t4yuLi za&%X6=c;Tdl_Ph)8P=rN!@?KYp0=t_S&b&hEiGHo?wF`L{ki+*8I=b(GgoNO1RzJ; z>X$Ylk~Doro7^l-c5pd3aju?J8QPHxeJUH#{bPX4f_A>3pq0Fmp|)x6g30B zHpsK`)Tht3>h&5y*51b_kr{H3DrzfY`FA{M-KV1mlEdtG@Ds^4-|%jWi|@&4=m@Q} zkw*z{`K!fmep;$>$|_FxI1>VmeP!LM$jZq?>)sTUJju(az~{!;&@-IdW#H% zOc>HlMV7Xi8ext+R4$!LjZT)mya2U-ZB(&;4j%_WWdD(MCH@KrOx^NV3wjx zByS`{`za~8eDuWxrf_c1oDKH8U&#ip?=~Y~$CFNM-;HF(I@WumER{-^SGfWrZ|xq? zs%ekp7rfm<2vpiKF4VtGB>fRt>YXuh%=PZq2@CK)zHbYUc;;Dtc?2wK2A=7-#dIDP z6N#0IU|kAh79RX+82Jn|(XPEN2xlqiFf1Z*&QVZ*#9Syprkl{&fySI}>*6Qv{lz@- z7x`-FH`7&ae(uG3z<^2z>>3^!0!ojvA$EbHOW{=UQxxIX^?Ev=$*z$verj!r9AK5U zULry1t1+*bF){SI9(WyR?@`$_?Bh<(Tq*p{Adk0$u?Rvl)u@;Kdz51RgH##xR!mE{ zFyns<;|c5>zMz}AAPE{5=}b6m^0SXzcp&ZTa#P%1GdjQq>Y+wqiW%HGWc}S3^aVnj zsNyaCp=>{%GH_1giO|at8>aB{xcxKxz_XoiN-a{+Qjdiiisq2VA!Snl8n;_+&g`@hPNGXGW<$Wm{#<_zat7I6}thar{*2F21w!L`Qm!yox#J zNV{^py+$S5{#plDX$D+$UUwo+rNtex>{WE?WiL2yZS>e-)9r!|mIMCGH+0D(QzzO{9?TAER&utDB&E%EBMZw{^6--Dr=}pqT9{Vi zTKp01+``u*qS72^#0dhw-esM$VLs+e$J$DQ8*9ca0@j_-r3<0C!i{)t2e0yEO%N6Z zOQlA0Dne)se|S=swO%p($V93>OB>SFM^0(EzkMaoi;Bz?1qhOF{BmEp&c`^CYn`UG zd9JW_|2nf9*M!hu&nUcT$!qDyu~wtSK);saF6G$v5n93CH?QpfBg z5cXJZHCXR_`bh#8-b16)vq)am@?l@QGPS)%rax^<b1BN?8GMLV>O<%r)&%iHiQ8^2-<~6JXWOoU61!Lmg}z)6-69E_jLn%z zST!fUAE}-Jk}R6<5zD?o0bSTuC-Y&4oe|6AF0VK3U>yCHiR8g)TtH{8psLmV!o4ja z^6NoMK2;VHWO^(lBBF}NzUdLCZm!YgXuU|%XNiU`u+tZ9cyfVj|9?nq%}&AMWMICyQG<^#T~q@ z@1|!dPO|wd;z(ty#6u2(WpAIgFVw+`ycu_SUS|8$=F@0K;okE@w=5(M#W;zUwvSu7 zibYxUN4>|q7T#B+5!R(ZL1lS5r}f*=4Xy@m^7n8Db!RN{9>j8i!B{zL^Oe66?yqzs z%+Qy|43Q7&#Hk2%ej0=5N?(`kLp-iuM)0I%+goJY?4T!fMz0YvCngPzPDik>F7u3D zQd-pAfH&BPN}ZhA&EdXTS%g6;L^e0ZleFT2pyA(WD5YM{BF=Edb@_sV5$YwN<%v+*U}D8Lo3$OniJk~YAu(v%q8xLGrJ?!J%Wvqbgyga(j1 zHICzA@dwD8`cD)!%Sl$4CTxU9(c)O>T176tNJHRB4~agPiE};7rpLM=Zo&7|5PR`j zDb^Lb zf)kds&NZY+wj!YgcwkixIiDwX!%pXTRgMJsIB@4Hi3`|E26EWKF&$2NMKv^j&@;SN zlx=%h69UiZ5tEw1jsN5|dxLzHazw`{e6KTWLxESa3JMug!V#MzG7|p4Ml6SYtz_|w zQFn2n?blW7((g5Cu{z&4V&~L1Ay(`Rjn(Zq3Hj<*RBaj>zG|HV(iigcW!w=N-^;1S zF9xzi1AdJ@Wt5CHq#0fUc-j%W;FT;L9L1z9diA2qNgk0e-#l0#H~+SwXRVUJ^vxMQkjX4@6Ps=I zG3{5@>~Lh)?)kIs6IjlbuUnjTgKtHC19jhZs8Em9`ilK&-Homm#9%z^=|t-w$Xz@6 z;IXBC;oV)X(5fvcbTJhl?_6wo{WD5+>GFB}q<2E{-F5{YZf-ct z3@(v}l{~*^j%3Uq;eCnTzpyVMDBqq8(U@Zp5^kL$nKHornJI#Q`%#2j=K>m0BK#$l*cj4R`%%}A-EW42@&_Mfz z_X6kkYbb_O3=#T$_5Hux?9ttqJWW3$AG-}bVeq6v8qq`!V(}L3n z&R5{_+cIxA#6?iUvq_f`OKI9;HP8nn9FEI!NmBPCt4C%hr%J3zK6&9Nx}Pnx_gpTsJeR*e6W@-) zMR3{1@?+Y1MaL|aHaL1K6xf^pcEI;ra3thk8f(smny_Cnxe~159yM+#PX79aSE47s zN#%1}59%lwK<^j%m?o?ov-RFa>H|v!-LUduo%-uE(}1I!>RNT2<1&UeeLv6Zci!$J zc<1lOsBqgnGLg7Ny`Lzq;`SE#8M7YG$^lEIX<$f=@@@iO*e*9VCeF<)AODX+F^TtK z)e}UKID_1#C89KgsJnUS*N-Cd)Q(02H5Ldv_QYtI7sVkq&Zn0rg`;7@C7phw1M;i^ z8dsMoYA!~l;`St^j_D#z)n9-E-Zy${ogP0aAWJO#VZX6_{#H>L(Hi@D@k=Cy?n%1m2bti5i7$-pHQx#oRLFotFoZ&Ct2+#eNwFs)|?yL>op zPna#RWEUi^SC~hanWn^XU)y$SKK?UKl zC9Fr6rL%Flz>~bpwYe0N6hJbe8BOC+lJ1e#>(5p%Ln?mNxALZUgA1ps#C=0@xIBgu z>bm5JuU-w^2#-h@-5v(Yl1%1>C^gPWCO#Kfo_YMw!4FNet;g)XfS))2G=fV(*XN&a zW?XYtJK&o4%C2{gG~Tq8kF)+zDKM0kk^kSAd&{t@+VF$~|Yk)rQ|6Tjo`{O>=;wv0X7-QV`xUcK{o#*uvpPXaEoERyluPcSIVyWT zhjewZB=I&*dr@54f}BaaHm@=B^DcEnZduBZjdXjW#fnwjj)(6T^jVS5(@et>doEgh zhHMVg9$`k`KZ3Xp2extrzkGSP_L2mgj0m{6I#Y1(Qp^ZiRUX4vE6-KE7!8?V;U2H@MO`eltLx=fPD) zW_&wA#f!uU0!LSXSU%60CRig8!J)EKp+ zc*0m1gT%hx3Xn51U}YUZ(+Wae+-}Kiqjm8Nx#@koCY$>zYOzfl8wSVzG64OlvqWq! za(xy2-nxToodCpeod=iJ&Ad7##n^{#`ef}=b=Qb}^QYB^?IBUOCvT+J1II-nmIn!M ze$>#kad1?Qzba|-g>h6bR%Mj5O9*fVb+1a$HReeiK$P1jFq5vzI zvHCncaH4}M?}E>?nNZ9bT%TAi?91}rFO{bY)RP_NNmc>MKNGufu8r-)U}nFo7xLGh z*LY-sFcubM$D-Pc`*N5saSDB!b4rMO5J?<{LFxyJzR(6>U6xRP`BUN!AB`z_>pq2-bvZKBl`Sy-l3ys>( z#@(Q~aZ5r^pgD*v7Vf0m`1FgC|Mzu^CIz5Jai$mLXWC<8=%-GvEH7IG{JQwgjATsF z74S2>er4`}&S+(6DHpx?JpfO7`SIB8K~gx@nTaUe&hn@!WQtTc>|r__zs5m&H~o2I zIRpw%SNJp6dGyUe6=|_#(|!5|giBsO_d~@Qjel}J&O{DaYx~R95)PF44lQ9z-wd8( z)}^3HfObj!2RmdnZJG$8f#P0)o9z(CR1dlY z4YR50auigtDDMz92+$HVnEYPSC<<8DFpolih2j?bH8!s^w!(V5SqY3+{7ttx>Ymug zpCmK&kLKc~=AF!S+}{GZ7BjKR^5FF@Cre(3U2eqsjoQv|{hv6Wu05?V9%;aLI)$KnZQ4z@oOxM=3`m)gzh zkqv1NGte$awYIUi+$}$siI9lXS?tC6jpQn6&WWk(jA%h?O7P{4$BLES1^Gn?{Q`*y zJ$bQY#BHf!@XGWPQL=1!M5_^4I(Ylq1a1|nL3ZwP%4#F^9y>dpW79Xw>oCQ%%*NP{5=3{h- zYRHc|^RnD0sw4VQBk6_Y;mb$UYld)lQN_Z`#d|p};*51Sx>xLDXYAwHLwkk#wz4jt zX1MS$x-hlja**(dbprXv&|knb?Ysid9a|m-Md1BetV!bYb6I<~Ay_Z%-Tj_S0Fgjf zf{wfvq3SK;vm7R0EK5IS;Q0(S1(JeM59gZ$;m^p^Cy6~?QSOu9GK;wfrIYvck&sQB zz(Z_K?UyN-_VGqWbXcm0*Sm?Ijg+D*spH2SBb2>ut!Y|+PG*db5M0ZWHwP2Z#0%iwMwRJZ^In3>F?Fxodq8=uB~Dm0_7gl zM&_IaPQM~&vRay{|8Q`Ii#eN^#SHo!Xe~^!=+)sD9iy~)b+@Lp+Mm0JD&ZNZO=whU zmZMLuUe#+w7EL4yy{(#5o>(I%o0w}?fTdV7H0TkMO)&6%J3EXaDiHh%R9f#~fd_|o zt~E#c6Cw3AP5wnY@Tv^>LwgMcWkWvxYbcWx1FXlKV<89R%-f18#@`)1WPHly;DqKC z@Oi>-8O_3~4wGe&{yV2J z{ET!??!=mX5jk&DiHExi!t*u#Sn9AyvESuuB)7zOMYdt5N;5K5!;NbsVOoTh`Jzf5^WUxEPh8E&+N1Mt@Rx-3(uO{_#Hv`T+@X|EPlasimM`EP<$4B9*0pfG zV=2B-i5f@nXp5qS^g$thJ~5*@>w*g+eKtRAju{eh;{1@FJ=AEB?Y3Lvw1F@R{(~C@o_>#v{i?Pl;?{Lj&Ekw`MKN+Y2FY9hmuriiq z6>r?gq8$5A&-@R1{Q{L_p3HCT6nkWvF=Lu5IG?AuzL+CVV^SN63nU^l zAS%wX3;S84C%#+@g|nyB4v)9X{K}e!qMbb9c!GU(1aMX$Krf^b@pkIV=p{ziSki9M^Ced>rKHcgaGWvsUI- z3D{U)PJEaKH&AE=ra>}|v0Amm6l1?DXOe+sG@hU2k{`bNUSFJ6v4Vm~;ZiGqH8#$0 z@@eE8o(5Db0-;$8pG^Hl{MxITrfaM9?Lq6BBk_?xiGRE1@4Pb+a>MjHLMJAUv<3-w zG2s$C{OAK!R$LdGYIIzCgy!5=7{{M)vJVI4sZDe*teOkN*qKgQ#vE7VrB_6ru@{R% zcp8rKnU-HIoCJ;*%)k7xVdZ))z25q>?pP&Nd}PE>iqg|>>w{To+&n@Pcd@8&^JnJm z@t>Z$nzlY)S@i{ja0XcS_GCoZVt=ZH?zVR)({45E5ZE)-{Em96qn~+=%hdifg5yChFh%F#T)Nc&BvugTip#+b5k}uh+>%@)981P z^?JXUCcczrNvZ6ixeSQLZ>-_p8JMWzXbFBXsa;EJ8H!ECSLEyxTebqV&U-B%I}WoQ zQ&fOn&RQ{M*?uoa_hZ}~-o?&~1huCV#i?&^;oc^}Y2W#VnL7{R<7WJQNj=_LZ3A#M zq0+>wP{FI;B8@u$ue1JUue|+CE_&gT4F|ZzG3}U>Pw$_R$Yc!>N|%NkAv7-0<8?7d zs9et`yfHxeCv<6YyP{izb%Uy!QiD*At zWg0a45F)Q)q>{)q`fitPqRT1xyFS76BDdDw4SKYHF*3?FdMZ%obxx`5cCF^Ft78;k z+IBOzS}py0@Z4yPZ(I}Upj1Q}0ZUHL zJ@PmMj#_&+8;;0Nk?x2Zk?{BBAcE(lvdN^XGY5A%ODuRnuz{|yf%}&ZOTR_?z=NOw z&6H#43;dcg{JCE+ViK$~mXh^BF#g4z4g3Y5foBe%^xdNZ=M*dOLzBusD!_O8Iwer7 z;)0|<#M&9J3+W>JEPTimz2i_&$kUJ?4WVAofv-i;NRUB)!WF5S()@yNI#Cii&ohWC zb={knyC#VwEKx92id9%eCm8jh#-|b8F)_eZjPfG+5|hXAx0kHlyWI7~z58ckkQa*a zq=(XsQJH+0W2Cc^m2SnXUD|KPUh#6w6QDO}7t_ZaE^OQf-O{_(Wy*77;D-2rVq}^_ zjjma>UAh^z;QNjtW%~*C`Nb7O(J{t=idykxMrpSf?GmKmwi$GIBDm7&Lk`EEeX5kQ zeAma&=_1fF?bI5l-jw&0gm7HL+O2jVO_M2R#9@v|0bl#~x97!#cNd89g0(nRJ(;=q z_jd&LB`E=>JT%k4zY$HeUur2p;t%mqISic4+WBF0>jAn&zg&j{Z9>6Y@}H`SJN>^_ zI^N4seoOC!6kiR#a7qF|N(H9iN={fsghDE7Rtk6c&8ZPHn(=|Cv5c^5+t}JoEvYVdI z^C=u5JQ`Bn(SfJxkz@*2KE`kQ{bV77OFcWxj>{xT<@+yNiGpmu3P7JY1G>(ATJa<} zb}DY>57O$5k&e@aO}7xlt_1+?ohfmek=)jq&bTgcK?vW2Bsus>yX2~dOSD3`bUZ4f z_&V*FEIrKCGGkOVtfzYC&`7o`{oH#2MPcxu#~tXeWcKKR&q<1leOlMX3VOXgog*yM zoC(h#F$H{5Ho+oJ-(KGzRiypALP zG-f}!Uk9E9rH*Mum$^na(^f^Zh^CXYU^`c;Qj=UaFW0Ip6ojkiiZvcVuP)Q&wnS5z zi{G0Y5#GRAyRQ^rMdnGw0`aQuG*8lJuO4!w9!sdU@WGDm4r z0&g$zBew>@AXhmhW1%fP2;7LZ&bn6X7xES7TyZjRFJIUvkI+0xrE!e#VbOTOTNKqcoWf6FP&H)0^heE5Qk*Ntj${I4r}gbF-v?Q74Se zhCays<{N|McrKRHCm3nhAq%-_ko~{W-R;k?> zliIMFOoQ&k*!|H}>T^-e%-ZvOP~54$hE+zdkr4C(jphLQ-q4kI;P(czH802_pyQn1 zpMyAv@Rw(e1F7&JSb5Lg>w39ni?PcHrCG|DgjjNrUns2um0~-~Cgeh9A*_=gGd0=E zhu`24P}#oRzNY6lpcpW9m6vBAx)|Ol&)kT`w{mO@9d+04lUwk~IN^^Sdh_qLo|K>B$))3tY$120zIe&d( z7-ZYZ>;oR34Ht?*^{HtVADrZL^O(=#=h6v2lZ!gNi7P4$7g}?aEf6qG^2sztgK3F& z{iKNvOYpHDA3>fi3Rot2Lr$y_2@+3)#mg;e4wlj_(LMC7Ier~HA&I6mGQr!Bv87jE zJqh1A8Zp^p9joPxU_5;f90*t$0U||Ra18afk9jfBi1DXNl9`;bDqa!B$OR_1mKxdk zHFq!+sU!yIS8pZM1N?UwONG%ky~PwOx=(q z3(t@P3jbF<1=mdh5$=PL_oLhRB)^i6pRRb}VW2KEPzZ};ir$$!AvrIFsC3iY)hK3A zNBBJ$DJEY1{Zq~)w5H`^U>GiA<9ut9M^L~#9nsyjHU5c4I$RA`EQGK1PCU1-Kxpz- zT7`%zJ=73;)SXtIAU?zzV<9pEF4rywvQqYzXT*T0eP} z&LQfeFYc}_@DW23LitYm#C6zpB5^IhvXUZm<4{vDDF3+T@TmV4UuaycHqb5dZY`jN z8{FFVY!i&cl6SdjZkui*_FqP6kUt_iS8WQ{ECd`jMm~Ut$U3v^0||DK&>s|4_pF}H z$nu^51JkFfN=n=7^c5nB&Z2MyVGb|Pp`I?(BQVT1M^9kh&tgMs5AW5Zm4uG|Z+O82 zm;3fV?h2ssCH(*2HB6p} zXup_}DI{x@s>SOuu)WV2MX>z4*+IRJ|H5Px=}|x?_6XOScmFY27;%kI38GFd%4FsS z*pfH!c_T@-xp>AIr$4QZZu&i1&)T(}3vTsXyKg36${!p#&Jx-cN>f0Ys9>`?<8|cVhymh2}uuO?+;%_c);@)*Xs;^#4^@yt08=cnzcG@ zH-}u@pXHxv3E_otlj&y@-*k{m)-%-{TIZ>0jO&f*x5z95(Gcj2r}% z$c7#;a@Z&6dP~L-)!83Fqv@IbMExPFu14d3z^`G%n0hOjD{>O?^H!rdq=-vOyV$#b zw19~rME5TO9NYt2w{npyGrO$VExcrPyw2}(bepO_{_A+5NJqB-AcuiAP?7BO;!A)& zc_r73Zl84D?j+)!-J~7fHlZy|4=Z-PLw0g0Nzcf7GH!rK{pD?H;rh#&x?+%SJ3bCO zlT+OhEr9#A6meeE!J%q;C7NQa?3WBp>x-y)34BUMFmTmy!v>_EhU$ZvNwIdt6b zDZ0h*NSO7$CMjl;Z=<$G^)(%|hPwR2yUhU3ZkXI`w1`Dn`ys>EW%bW|_}56sAeyGZ zPh9amoUfrdc$_)JkYm_USj*@=^^9q-F5NYJ@YrA)B4pJ zvnzBqx~I>mE8^(DI^70yrtUrnsgYbiN8mzrVZq{iF1gGf)KCQXD`0tV1&{(q=<@lY zog#C-;il<4lXf#QZ~gvqmd(KxmEN5^*Y?_>bdHZlSM8KuxoXQ7=!!DKOOX?&e?J||W`3q4rhS*;4;~o+s)}Loo+6mSnril94 znEqggb8Vah$zWKcl;*gqx+c6JZ>eLODxfb}2%{aALm^ksgQ5wa<$%d6=1dfgDo0V| z?7|D<$cM`)2{gCtM2)sD87&B}wO-)n9-INuCXWrr9@|HBdH@!2I5Hn(-UAy|2{KO# zclqiRXU_C~sVU1wi=@`y-Rqk?CZx&qe^SKbl>yAD|IRzPT9Be-m!&Qs-T$Qffh~TH z#$V~GT}1u!$q(5F^MI4aDcpY8r1xO2D-u7Bd}9^GI^(SGn^{ke4>>?u3_DkX$sfxE z`6Rh7NHaqvR-YR9+cJq=BbVZNv^WZr4~5F*A2eo_6b;0amdN986zP3vxfS8QQPUJ( zQv`fxZ?WVI;|l*?%AeKa*u{;fiwO}lV}y=KGz*h4B))r1Vx=!w{?$KB`{mdL!-w}> zsONrK2EpYY>h5ybfpzF$^c3FdPW0w9K&CYuUw;>6BQRtB$x4_{9;WD7z3%Hz00=7| zpwS>48~9cV_dglp#v1OB4{LaAmJV&I7f9@Ow`?W@rDvT;Fh=Y~JOCe_F&mY#jQQ=Q z4UWaSGR=YE(bLN3ABzxpAoTU`KNgtz4LY3ayJ_k}O@BOpb<(vmFjs!+~;LDULfS-qRb=TW`cjTjj z%zP!Q8*z2}aj=aeg6_wcKlzWq2X1MUJ-!@d{P}Pdn=o^@T<9_%)xsX2zhTWy+vX|< zI6CanlSoxquukf#B#&!{)N0=m9B&p3ZzYdB`%ZO1tF@Uw?ft8oTz@02R|)KoAbp-9 zUID_fDS_MqKH}C+dmPX z!0C~J@ehd)7v!4M-@3yWjx#}iXk?;YD@PL`A&BTh4DnDs5Da&_eJ~fT>WYRlu6y*+ zIhNS}Iv^{qRzz_1-g0aiJL9l`lKW3y6F`I=txkHbbLm-A8#*xLrdVb~OHg&D_ewPq zj?@D|jk*XZ#&aTi8_wIXJlHJ^7XN5npds>7aBMJKsP_yQ!e4ty@CHz555t)bzU0f$me+=Wp)hI&`&l@sjE)0=%G@9pcETl} zRcGY%gUJBR1Mmx&u>4)K@;3hurhdu$N?&m-_!<~e-hR}an`(|-u>jzQCPKC#9Q8p} ztL$;#I{%E#&3!*w3OfY}AVi~zJS*#ZnB#&$)~ZIiAJk(@>`==`ln=-+0U7}2#+5pK z{LTVQM(L7YwK&x0hO4rR9^L4Ix(i>0lrREftYjX0W!FUm;eZz7!77r%{B+w0F~-& z_nH!>!mRkF!-Ah9Cm*}wQ>CB4p3lP(NlMjaQ>*rH`wSUMRaZ4= z&1%TnXkQOzgc|4(@NdYb)uv&Ht!OLa9{6^wDUeagoHVCywC}J)?gCy3w$FR9(T9$u6u ziH`m+LSh(tt*@n4?efdCLL04C^CU8 za0zw^@}svQVuO{0Xr8uD4g9sZm)sgUU5oqMKpdn8h$83T0P~FQmj{6eubT#=Nyh0DDaqK~neUKZ}dF^({JT)CP9`3;YOU#4kX%a(ftdbd=X~s#8z% zP~u?C5^3f!oks`gugL!~d^{4)UI@j3(Z3n|T?aD8sgM$?a+%oZLZd|c!WKHS!Z3%2 ztzQ_10b!^6&v;7ieXTvy)n-K?hI66B6EN$BHL4lLuOwU?OA;IQUXy(K|gTu%n#toR@V(}>cat?Z?= z3YnfP&-Jf5h0B1S92y_?!T?Ov3x|l?>g_O_aBAby0lm3fF4fuajI}Xi)xIv>x1{)I z5QGCW8il%Ed$rww^7+=f{AkxjS6Kw2n5$)!vHWm^b5m8&=z({~r|-e)Yt}0kk$B?f zo;}y^wW$_uHoef^{Dw$x@Ros^b>YxjIJBpv^4C@3;h9fX3ys-`UDv7rZf(&%dLDn0RM`Q`|)AlO6uNZ9)&pn@l^bpiZ z1tB@rjDR(;)*6l4|F(3&Gg-dnMJA^r4i`#Rp56^XQ@e`P&7gZrkYwYwhdcK#gkL#TS~k}FOTweygQhxc{->o}({7$;BY?5Bn87E0>a zXS*3ciH%;r7D_tX|E!iJGY`u{e52(vfZD->`A z&s9#<9E3~mDQB8)ncdR5M(2^9e{IBEBx*p=h;WfMiYmHk@)Xx_e$_L3hf#9uJW`0H zLM&A}{V7BirkLus`qCuEgIE1J86g=ba<55Z&q*SMJ1prSqg{20czelwhH#8Q))IY9P?n#-tqID5@XdaM3O@&>+sAbH~un2iIFw>i6fP4jVy zjBLVl;cDkFRzN!&U=FfdhFbjJVHaoS3-GgK!ijsy015SE`1efExfim$cbr7k&%Dp@ zNqgMqdb9)c$71fqzVGBQtAk;H*V>=sQ1Jma4!=&yqIDSSDu92j)?Rr+cPt8T8&a#| zrXFcb8-HT|+te-)q*crO{eh}wxDRvFdU5^I?%^>~&pE-&Fr>I~r*5!w)K8F&6yB`A zSaLn*%MZ}3zSe&Syc!?Mjau!o}= zn_x$Q_|{L8Ot^|9t<;XE0zVpY%HMhK=d zVtt6tD6-`H3N^G7h)PaxZiKtyqb$77QKF@~5HEhCr+~5DADl!!IxQ}@WX6PHX1Plt zzJ2)>m55ZG-nh4GGa)@Onf^Ye+rUUXBMl3H6OI@;Bvv*}=>t3fSk&Zp;q_Vb;=J79 zWvwUO#@i~3fE2DTK;!hbxy2)u2w2AKZ~!{-c;m$%GAM5M98SM?$4p6T)Bl%kmMt`7 ztBMYI4&IDC`$$&2^~Ww=%au42TjD>CGY$oLJe$bxN7gRj)JKuGB=0Vr8%4tzGamqG zz#a!VSZOCr8A?gcTZGxJ?4N*bZ_RvatwW!yC*Mda_x4RBZuKi~qRq_*|2KBgprTJW zfDBV`O0Zu%c(rjFl+AyylYYAm`dY#EE=$_(vAMP_{Mc~OX=p|cYdIp56*$+(*#7Ep z{ZbxWP1mEJqbUn0qoE!v`0hsYRAi%QmWtE6sA~H__U1B;I?9?}a7l(Ko!c{0FwmKb zf>b!!%gz@|Lpbc~MQYi(S&HWh)PdZDbMElGa-NthcWLTe+T+WNR$!xnVs)c#VAp=< z)C*XXHFJs{zO;LSGq6JU9bYXJ3dJ(9arB^=b>qAdVrcf)qxy(CZ9*x#$( zdU~a8mFA5gGI5CW3(MirPdXudYbGE21I&DoAjty?>L; z@u9q^d{N{LOw*=3#{u3WBDLId$1`yru{FO#yS||RsG=I^a0J`6arFB)Flc8;w|(DX z_=wY_{D}zH83@&&bIhXc7t@o4a1@Dw8CqTY9x(Y@!;cMUzFIe`u z2@+v_x!B3R+xe&mbz5sLUJK>-24e|`BFYU zz`&LYuhp-gC+XR$T0Xqc*9AVqU&oeSBlt6sh@9$RN$z&3Gw_9X4QUMV;`) zxqk?J5f&{A&c=K3wB0na8v@R6zdiCVc|CUUBUL_8H9oZM6NDLmN&V^|?Yva72=+N0 zi{>XgaoqG%a#Yw*m6in>tpJnHnM8Ox2G`q}{@WxEFqNCt&MqL=%>pWD#vQ)6c|0y(2G`h}U>lm0}M)E=-)3YG5+CKQ^v@68qQ+ zm+q;kaF2rbYV_!`=YM?7Ui8(t5Y`>N*jP|ArGK9>hi?TGDd+>0ai5o1t$^i0a8bcF3_5`f z>vbyHPy!P7w_UssGMo&S;`yWp)_mt@yc}$K8MIQC*lbCIBN`mWf%$-|&_c0ytCE#0 z(uf-h^7kKRNnn~;J^x-H3Hs8Q4Y7uAKF2>4v&3E756y~_OZBd4E;sVm3ayzGcRc5= z2xaFd;Kn>0NGITtj`ojCtiwXtG9TymOk@H>L|2zmi@3j#ht;(hfU}L&ARK3RW5gL! zq2=5)n!d<{TsLiBZpfrywCGsnzuGuOep6h0Iv7-njre3I7g6e8dcno^62^|4P#{_H zD)Cl8f*@W3hoDkXh@4F@jt8d1ioq@B1C_&pRDrKWA30iimCwGxU+EKtK~fFP}0n`ahBApyS`ps({YxnxU?a{T}`!U7toryT+_7L zj`Jic1H5)<-G_=gERnf|@W;(&qEHn~6MD9|IXlVvykeIR!XaZqX{1J!iL+_Sh^~2K z3815wh_jv7H)?1wH}pwGMOYdh`Xp7uCa;%_jkASCSIe9Yhv%{-r0r1f5Py0D z8`J)m$Gk%~r>?#BDh->rBDh#^sx}&m#6Va{aB&uU}dlQ4BtT8IIvv z3VHN9pSl{plw>=^aPzKf6HFa;;Oa*40H$tsyd(RiP7V2xl|xXf7y@JdkffA>roy5* zDvo-KmbjV(Co2|on-AB1fT*-f9l!JqTeHO%a>yC>uG-e84OS`+Y z)AIp&*sIZv)cOHZUCS-@O=xTS`4|p!G}jMUf+J4h53?!?Zg_WS~tBU$*Q9&%jn}jRxlq@(9p?>-VDO(d4{kQ$jgxh zJjBC|3!L_76sPZqr&Yhx^tZx3REQ|#FbPn=*qygOY?L+o^_RU^82zH2h`BJTt;E{v z{)3wx8@hBWd)nyqxAilS;`Rn13^5D{A^vg~=3bRov^})psSu-==0@mp8q+z*x^GjilH?PVg~(l@^UH`X zA3Y6SHX{m88|Od;jwX^=)xYT{lEyZve+FmOhUgQ=C9%9qrn^@#z}aL#Aa_1<$m$A+ zUd&HM0=fLklG-!=>|3KAF3RaE49)t-mC4g@ke#@#eD_?aZAe5Ly>$OG3eK;t2y<}s z`q2+K<6RAsU$nDwP9!1a+b&ve77Z(6=tV2$F(hQ97>Ky6vuAMo7|xy@e^Aqt&%)yE z*Ixh&yQ?&H9&H+y2np9i{+FAb-%pkeo`CqnSAm2{avMwbB;G=i^VMM~sn{GyelCi} z=C1J%Z@@bNlxGO%*F!Ehjk)7OQ3n~&fL^FaFl773iP1w>EZ{H55R5fqhK`)U|Qmk8q~`WpckJG;-kvZqXD3jD4f0+MH$TY+KlsLia^Xh+L6=#K z%l>4S8WspKSxSbExhDi|av^ALVRyh_i%cPCJLr+^J4wG_@`vB_z`dT)%{_5ZB>jEp zwoUGpk*DYujeL$>wD$Z3P~@Rd+e{ZafoVR{hJe`S-ElMTXhfQ2vk~>3=~&oJV7};B z*Xs)@@)0$JQAtdDY0&L9JnF9D4Ev;?1y(X{8=vIlFfak6g+hH4U?opl=jC59(PYGQ z%6XjVP;qYL;xpedTk@>d}DIyTa*YN_n!+#KPa}|`PX4)NHY&Q zpHtEZL$w-U8N^0{`W-bX<4DyCyukAD6mxXPK8+%8CDkql9K8J3Pl^S%6+XlXrgl4^ z`Y>_%=Xf7bg?fhLtR9jl7He$b)^Aa zLHCrjJv01)-RNcovXAjh>5fBwB1SXyMFGF^sHT}&4g)yDu@P=ajx82k+P_cfxziEmpkhlleOBhN2<}9 z`9qdGQdr>RgfcEOhXXwNT<*KcOhba*nedU3+f5?3=!Rf3yfM7Y9?-}+6?=9Qr4{~=L;z!%P)=1XnOA@au6dm- z8^@k2W~h4%BI3l)4y@Fh| zhh~FVG=Kq7!l(=>Pz?6K)nnG&waFp0lUg#g5C_fDxwiGa2um-Qyz{r8U7B+38LVbQ zg7t@WC$|l#xYCpG^*Dpdi#&w+YMx}TM{G9S;dHKez{Vb$zUJmWWeu9UT@Rq)lg1@} znn*I5qn+RxUJ7|FWUlbngN*XUG`o4l*avC>hze{oJk8#&n)D6*kw%G*4MD>E>W!;Z z4Gs6m+MZINsi^2kn3s*T(cbJ~G7SI=eryXPJjX2zrH_1ya^SlpUO;5** zboA-4npV#>qfcW$s>?`08gx4>$V^;4eFvQrlP zzH+|?wKO7a6SyrB9zns2>O2j`jjq*e2_L>f+ey|_qw`2tfRDUdEb=Y5$@jj_uBnG- zM7K|~$z`j8xU7gf340}cyar>840$tkY8bKdpOg)rnfy0HheuG>Q$4E$q7>zUiJcD%_3`WfM39$!&kKLoL!km>dojxI z;|6qer7deJWSI>8|0Z+cu4z0bbNP*HH9Wsjz+_##Ac>w(u8fTlh5Y;-Z*AXh#O z-4hKcj0k}`veyNt~POCKT4HF$uD(}o)p##NaQsrdxAN=nkGJP+MR8Zg$7N0 zy9~HkACe__GqZGR!q*g?Sq7cpBG-&Ax5qAKbSa#w+78g%cn=iseVEWu=;7>`0-A%W zGw@mK6YchtKwOK%{~lSDW`F`84R2KIg4q<<9Y=N1crT&?Nzpwkj7}EEEq}rE>j59! zSLYD=#=ia@q-U~Lbp@%Cp9)1FUZ!}LbX(n26l2v7SY46*&?ZL?ACs$W;6y2Ua39pu znSX2m2V;d3 x^Tq&uG7HfQxToh6zpI=3-D*F^?TEiNO<^rL_-3Gya<2V0bG7pBJ zVZM@0l_Vw4pTOF@rc~Tkk;Hl_(>kw)?jF3>VPzgAVcT3}xkIV>Cz48g?Syb$qPVSU z>)_}v2R3ofq7Dq3e%K3);D7%OoVwO!-}mXh!RC=Z&fLxOH(f^J+RkPc^h?l8GqYD8 z$Cm_x%kL3n3=%~6)e(fg-4o8KU}Q)q*9R`m%=~Qi(11?3fN5FqP;Nry_dLG%GO z&x`*iOtnMxDSF`k{Ik)euT%n(1pt{RlG_h2$0Z!cj&hgo>-HZ2%wt|V$s}6x}bK}OQ;iw4c00cS>!4- zjlLilA4HX~RCD;BaH%6wXh7{5OMJibDH{K%838i@a4L&hrydpJP4(1Vgl+@=ufMra z&kghi5F@s@c}nUjRNAEDxDo51$ou%A2iOh9NO(V&W+=XjBUx1y;E<9UTCY1%nrw#! z9-QHH;_T;)?I`q&b-K)Hy#8K98+EHg%$*Galpad*$~HyX$?k_;12ftSI5pMWdxv52 z4(S0vBk{s}Eh#{E?ONVt9{+ey^U%+D2z0Xf9(09I`c=ACmp7xV|JAW;qIiMi-HM1) zZTIyZCzHL~MezA8Up8Nj)$p8m_PTN`<3@WEw~uVSX7s39fGH}3|Kk11R19D_xE^tK zqpKdZStlSeIR}z<=)qkPPkHWG!IrAS9ybKCRUs#kBViU&Z2QhLf{G>L+QkR?mkxuW z9D5@>IRMUn?pyixOv{}jO#IOLo#KM`a&Ti0sON5iqh%j`;`8Mchi{VY+8#p~4HZ60yOY*vz8PtpL!Wo&Jmcul73{R5 zH-?M56lq>JY#E@mHU^b1g6D;6#7Za}Va5PJxktev(dCQ!_kg;=d+rEh~77y?gAy2GC;Z=(>QV z6g^QzMzhrUTF|?!6s{;NC5GuP>%SydvT?fPu2s2|R2n%8l3>()ucJUj%^^59*d0T1 z1@c;vj^5^$*&%;s&ZeL%T#`Gb9==QZE5T#NnI0|(FjR1#7KLj?i%L>~;BMHjbs@y) zO*am-;N=@lg&2cBsZ`od8JxlFmvV{bwcnbC@ug-NW5fsR=_c{hWZ5s4xqd|^E9D_R zxue}YC=}PbMCK-AUd>u2ma^h9$ z9{@!)G<|1xMFdce6__d=9w&f+O(Xgjv$@Zgm*FGQ@wWD_~8O;VtH5CLmkWh217h(3GLK+K?tTg!CFR>%GLi>VDEC zQyE;BR&w(cM)~<|)L(tN-MQaOJk6aU1!9q!l}R;lDP-JdGT!@a*_}*VOe@j0fc@GN ztf)dqM(^EyWIA>oIO*c~Ez5@ObnReyR(z1M*Cv-(X03kcs^2H@r3?naH4iMUa$Qpq zr1j1a<9vDNJig@}+b6C$oyGQzV@_GU7KVUi0Q^7KS9ex=-vAWwU4)ZJQ#53wF{EvV z&|22*QMyn}GPCT<0!_r}jCPptso;fh*ZC5)-rz7KgiU!$YjJ2ZOw&q&(k7Z{?z3Jsqrq@CQTT z1VF;&xydDF$g#5(?!Mf>qV45HA`plcqCMA4S_Jr4CZDO@J0OMI`lJaQ^HUwU?`yWF z)nN_^65d_q?x0P6$>oa)+(1hdt>tSJe~9%m00+u+&T@wQv<%Zb*a8N=DydMX5aXMO zTy~PccYkr2GqAFFzOfzNF8{gREU^d=5wx6w$9ueE3*`w!J+0!NLkN_yfDR9cNe!fnw-ImweR{}+;e~KC4)u> zg2kOYcTR?@p@DqL=NCZF`iDrBY1jIDA1dv@Gz_-6uoZGo?~|Gac?0GJ;Yf(`5!q$` z!0(}#x>b8MS>|ETA5z-P!q;~f&Z>jW>F&&2Z8P+#gKTYAp6XC3421`6_)VAM^R$yL zJvXyk5+Gy>Rb`*LM%Ya`5gP7{Y33>dk*mNM1k$zNC%XJJ0n;etFwptD`zoI-D$N`I zH_nRHo>{d9io15IdD$$@B+N^M?l%QG3AD@;A`;ipcz4P>p!aEKN^u+1TA#9bVte1B zy67{0k~Xyw?HQ6mAw36#a)<9(f(#Y7{GTJRTJsC*?8uJU-KYOA+RiepsF!w4AlOY9(DaEMN1Ts25K`UXac&;C*IPvP zkV@`0P^SMA%@lmLbBOF{|EJjb7|RXzL&1K(i5gIV#bw|01W&%%R#I)7-zJG^bf`Xy zl-O$m^nJ{^4Tki!&NqP}�Z!u}Q&!*pq~N?5b{6Nfewb)~WkqmQen^_~P-&0@AV| zsg3%+(>b<1QO8!W#KSDv12NuFDFY?yU8IpTf95_Hn%dY-Rnve>xvumB>l?+LJ#*zg zA*PuH9%t|jDP%g(a`|Mi^_Ein@#`W7jO_Bx4inZge&uuFEdE~z`ee1P_GQ3S-J@ng zx1Q?1#^^tOp~cmh0i$xdVS=IXaL6}sn=}#_qd|CVRwAcXP5UiIS~s43M+}_!oYeXC zaKsw{xhWW;TQ}A|wuRM+&F5qaC2On=52r5e`6vZpXXpCOrmaH!mS>Zgdv?70Z9y8sA3|P;SB=s(?Y*)zl(o- zcJq1w%k#~E99^43RBw;`4&qTMq(5Xap=GGYyEKdKqtRgK_I1IFvhT+EI!xg?cY8Po zv6GvMM6(9K+~s1V5IZ(EYdwl`+A?%AfzSaP$8;aN0yBf#lMmYm1wx_OdWOMq zMQ5Bj`~w}HCwya>7ft6CWgnlM>+!pUfQrw*u>+eF3PO;j%r(vJVxf8c2ba)B?2S2i zl7YUN-Ppm%pXkl0O4@-2L!su~toP%ByO&JYM!NG0yf{`r0-w?YJI*$JGq?98873+m z%bFeYp`?b_HRZS&?ZdBN-D5Ke6+7sqS7dHCBF(vkY=v|u)@b_ylf&$%!VT2D;r!LD z$=>w|H}!q%b8$3()XEEMAU6*rDsBZ6-3+1a9Tjlm2tgX-?6Mb(#r!U#JFLj`m^^%Y zbpfzkRbw%GXC`~rtGAJY0Mg)UbHAwQX&8LUFVJ{(k0G(~E-6K!>YAPifkz>V2>K%1HY@wCRaGP zLx!aAsmJu4!lZ~Cvy}FAzT1xRGfPF=pZv09`B)t;UZ+{>6TuB6(6C?eD9ID)X&m2? zO7w=Yw<(UjQ4qKvFiusC4}}1-nLs)s)K`CUDy4T7uvvQP0b9b zFntB3R`9VM*`5TuSP8kOs!|BDJx40b$~_^{{!P`ad$tOV{^W;7u$OI%TJx@mkSY|U zJGryW$L0^((oN&N(z$`Ar^K)$1sY>b#zsD|6^D9u;)q>ltSB)t@1u3x3-bnZ4#G79JegGY`?PS1*S5mVbJ@L`B0dJy{E5;{ zkkJinyx^QFw8#`-F>Z9&@NOo0o`}MzhAnoP_EJ|fixYl`)09l~00Am0+=a!1i90c; zBqH<2+jW{Ek4J01{06{~?kB5SLWCBw))zw-!K3UYl{6wuUB$uhC=+U~T`Ny~kTU<0 z@57X{?GNayu`jUvJG?Biy6<;rdwWp=XG}pFi-*|berLWN--+m-GQQ?s<`E%k@^h7E zhLQ8Uv`T%S#-oVeb=jI_g(X|Mp8n$_vn2i(Nv^yZ@Z<16V#c!9bm$Aag^Jj9K zGBrkfyssi7{r7+j{N2?2xqD(x$t%ZU{R9xQ_o_pI51Z~}+fryUOpLVXV+3NEf0(V@hX4}dqs!m}IEw;izUIrx8=qR^qEQay~4V8ctsXLW|QlCr(!o;QtR- zQ0s-hV_K-skPr~=hOg6E4Cn>2@tLT1Corp1XAQQl&@+>F6w?lCelwBa8bAW+WKH&} z@Bv8$Zq9Il9^z{`n{5?`?TvE9`h*UjRN^WFUo)}Vsac;{#obnK<7?~R#S5_PL758X zfEZs`oH0pR5c}#eoxX>j(e-0aR$1I**M1J|(QUaG9Amm$Vx;fbG=5YzRog0KP5Pg4 zDE1f81Kr8qn>k^*DB?Cw-KW@%ZthbTECW&L=g!V`_C|UxjS1Sg8$| zIc>XagV*QmEdCC68o%9W_Zs&vZMmtgDZ1_#`FO@ zFWj?1QJhLIE_9x?t>xwohjm!pJx?96Puy#eD!^U8UfSmk-T%gIY-Y3iRb}sW_VD|L z0ht=zqPCUay%LfqFt&F!k=W(bRR7!>##TkOdpb+i*wr$d?I72p$QG`#(B7^fWwQaa z%T`zZu;f(^NnM1)4_@kZq@XZ7^>h(8BBGgCVT!9yftJUpDtzQ&-CEdmDW!Ipk&GBH zNsn&{ogBZ$U{dGJR{8Eq$?gG+5@)ouXO5~g+I{+Mj6sq!oc#xvz2~zrYApwey~d1z zmNKxyMuj)|#APV&TBOFh%XpvYX}n{xl5BM`;!O9BU23rKmYoRs^nGyh9=jvnXk1M2 zK{B?E<{kLWr03|z4qj`ecf`2~MK9_qE%8s&ZXFT|+~gbn{NZ0%_(kECl-hl}bsx@F zc>RW)#iu@>hYLVV8k2+uX&E(i6MylgX58U3ZA?};rvn>k?P(Jo|Il-lum4<9>~&<1 z-*@Z;3q66Cnk}Cp#wr=#C;D(vCxuT+DT(}LfN#xq>5LuY7&x>}c%<=_!i_djJ+lf zJH}GXBg@thKxy9!vXq`SanD(iTlR+Y$DZj;GxsceF9sQ$-T*F$&2R5*E7pqmDRX`9 zP=hRYY}y#z*RFl!0D1q_|s#yvpQLuU9#3P zS#c(X3etsC(YyH%PiJ#C9X_fl$Qvw5wq-S${BYY}pBm>Hze*~>E>_MUKT6)V0KwH! zMrzY?QvTc}|8nS;9a)LX;O8~&8)-;$+zz^Eka)R-PCCj?TJU; z@WqmCgn~~-zfTgrSPw37qh46OHUDimo3YT7ZcwVxf%k4_uP2DdJ%5y;OiYDmzlx8O zc8og+(ctc}ZlgdKRB^Bb%%ZnXY1tSdk!xfaz*kAr8}wVQIk#5O>Tyl0SRu1cJ2_vMiL+$ z3;>v=;AT@FLTTbK!h#4Ma!x5ypV|;w@mcN+q7tWmtwZ~ z2cx}NV}U<%k*|iI%4#)ODTAC;aX_FskZ&3tD;~D!VlVsCa;&HrxtpYp_SZ6uwuZa> zGjyXBXM--zJWZxbU|j{{U7(dulS*2lGM?vLU}*h-8`9)7%<-gWkfBy?0eO}&z+S`> zt^@t)8g;J~CtB2_qq;%$$t_6$-L;Ui#ESxQ^E4VGc;#uThTcX#um^(#C3*`Xe!2qg zT6j8%n-1C2+?#BXHsT(Ij4!J_%}#qd4Ghg#wmM;bI>fxa4>J@$2g2$%IjL zO(ZW;WoCaiH7F`YIe|0Ff*!5$HfmT9xa~&G3yet_Qnj=%_|41jrR~T@Qui0HZ{tMM zERsZkC}Zke+#ij#rZ9cNv-wMt$U@Y`OW7VI$7SKlY>Hj)G47}wVm5~?M>?ym%luc& zF6K9O9c@G85h=d_R@6b(rgePZFiw+){yIYV*B@7LAy=35xMigSw|GT~_UEZ@iS*?B z1PIJ_IlGD%VSfp-QFYd&ntVFA*<}1jZJxbVD3hIojwl;5-_%ZdX^9e`k#qUFc$H91 zYS}32-us+DkJw$2og-LI=aaSTO&IjJE|bb^_bw!gbh$aDg1VMBQ;WKG>y(A$g32YR zEiKTpity;VSuIWk1Q9{JE`LmFzcHE56EH-PQ>j zUQE-Us8P_)^%<4@Zx2-gpL@u+dD}W(&KW=b@*fGQD0tkcrx* zE32k!fwqkyS-sgYx1)@Ie15FtCA?72Lq$7TYp+~GCwfE}VH4=R^u2N%MdGOOAK zWLk8EiGo(X@TzI&IQrkP<-F=}O`9^zM0G^%tq%-+V<+mC^3b z3o2Rw$s9_F{$R~JuIRDc(v8Ha{L!^T%8INGlaCs~KaCCQ|3f!x4!!58=U`}m(9}$k zUsL`dsLcEsDvE)0^y_Bm>F#rU<|}{eoNRDqTI&P2#LF+2JGsRTYtWd4CXt!Hletk| zqKqX0*z++zUpZw+fh`?3MY@TQ&ETJqS`jX0ji{K_-MzTrDZaNC0=!HdAWoP+EA8{rPxBLhybH(@&%jji(IU z#R{R|%r-!8Re=iAq6r+{<>m+oPFxhbC5HB>mf%jq?y$L1q_bIptiR#ywTkDpJfCk4 zGO9{C*_H12-PyPLhvKU{xXaEwq^w4qo4q>;U%mmiq#hHI8bqo#966)nFPG(jfuhnU zT^8f#v7)}2>~rZ5Z^wf{@b6h5RZ(qM(KT6Hk64VNW|hmM+Go}QuM8zSgOLy&3u9M+ z4;9Os_!Ax-`5ft|P{<5d)wOpFtO@Y`@;!($SaIYFocCi7SUX;oalY6mE_IZe=<(Wq zEk4mUZ*e=KmOILqUpcr!&M7l?MaGN2T9l~3F^k7*ny{=m_%hP^#RDhp6G}j4AW-O6 zadQR*-W`&(yr`7ruciCRKZh#1x<0WB`-iN0C2k!Z1m?nPMH zG|19?u#T#Xw%iB3CxokT0Hd?&)18Qv${0EKn8Dg_4?6(#LRl*dn1&Ft-0dqwkLSCL z`!+U7eVBZDKZPfeUgBA=<%27|%AKY$Up5{N8HKwyB>MV(`rkaTuFzu=4p%Gc;p2wx zIiumR*SevgF3GlR*iZ6oe0*%VsBaYx`1q15_njN*zK{SPH0d?{lTMvK7v#Act33uS z$XzFJ_|`o;|Gl-n-;2UawtLpb9Y7SveO#n#D$sr=;Vq4Ljb&Cf-NEoGiEE-hfe~)tYVaI@G3ggELLV>We{FvAQ=~PrT^yGSQWnyJ@Y(WTVOxq3_ zmP>+}4)D>ifSgL12<`>U^X*xCvi;<#TblkGi-6V!L-9+ z^D4M5cSCzG*4_@z;fSSxnj90|w8zLtF(SIg$dAYtW;faJauD# z`zRybK~HKD#dP7cwh4>|A+I|#PO@$tE{R(sp|r&xf-5@vaxZEvz*Ltv#$rZNv=Qk7 z`fO|s^-~$YJ?kfqB?x|=&$;C(Q5|3QIb*Il%+px+lvZj)ZUmxv`m7Zapn4|=-(a4y z{qD(ho2wvKg|S99;spN(-xhEba!XNrv`C)2e|BPL{C<^fXOX&cUS!^kE*283z_OIC zK}UW6gm+Qh4t#=1m5Zj}5tY3Up&7-`-WL{VhF+U&Bea?efCf5DiHtMW-Z9H{8h=OU z)wO61s@%NAVcMy}9{d>P*aDT)CU7G;5AJuDb#E2A&ITg7l0UZrbrp(dcs^~oH5$lL z>#!#U&+A2tbm_h8P16sAa@V^Lkf*#OeEUCPzfb3Iasn??TC~UQSRKZP2&O0Eowkj? zc+BDEH1BCe1oG*nkP0mZxun(98r5k>5lwmlymW0j_tqLpi_Ij{?T$bPy{}cu#jWr5 z$F+@&(}z6pUS!~HM-RidgG- zPNVD{TImRc`VilV54G1NxOY(i&n71H<(4ocW}E#wbkk?d^i)x}%W(1_Gr z)jxRf6Yn%jPc=vI)k5B@W>P3nm8f|jfqma8)MS*N`h>rcGrE`%OG;gUD~ZNnju(n^ z{8(dJ4PGvEYt*f#C$wohgbAsX#YT&Li!BFqJW*`4i}P=6(ktdYo@Ks;CC%m>0Zdw6 z-?v2q(&f#fXVhsV30z_tsiGR6+FobS`+P`I$T_)EQ1VlF+feEH4vzngHSly^W!qMk?pm(XD&wGp5 zy^#rR%|+W=hs8TiUbCbQ(r%P5N#rrjuP{l2HVR(R9tP*@1hWUpG( zicexJBEXVUi11iA=e{w=NVrC80^fb0;{1t7R;Vh4v)EqIFLU`-7S!#_D zkQH7g4D(V!V2c_@@Zg4jBW?45c(cgy&KZ|a7hyN~Gz!NFMGeuFpDeqVlgc*Jz%zkJ z{)(~8LV4%T&g(9i3~3s8tM~cRc0CJjGR>^N2Io&~IS)-v=c*d2sy@e%>i8 zn;xSBN;vz!M1h>=&c-@A3m-O>)TGkM{m)2=dF}DxGtgcJf^qJ5c0h4v2!DM;N^7vs}ln*)Bi_snl<0y_2Jk~m#xZ=uW?QAdjdbq zE`jkHF1)%x52mln&O2sdIu-EKa~Dm0avz5*pZc`F+W}|OY3QlVB~2N>U$Mv_Vbw4l zE1H5UT7-D616;bv=Oq52xtI0tzZWv`24q?MAmGU$`WmVQj8`h%j%I+VICFO8TY={V z@obvI0=$0Hp|fVpi#T51a1;z)+{R1BAGG*Z2~4NxgLJND$z65|_zb2|odn<}X&uvl zb$ga1keW~J2UUS*{IRyi74=@dxk?3|!Zh*AVZ!CU7^M67e%GN1 zsccpvpK^GJJFv=IguWLpcmE^F3z+~I1H~QiK|%H@&r8nRDEWHRY9tpX!x?v+%ia@? zRAiPyUjNHrI}$+oSoR!OqP63Y7a25CoHZf!_l`e+4F5Nk4aJ-YQpeP3b@ei(zi@cQ z3w6Tpvtx8A{)?Lh-xZ5Vfq=lu6N7nhQCdV~u9 zPBOj!L(w5tfm;{x6oT)fqKRuo%IU>_oP0kP^a2=*yNu5Lgylq3Vbi_qHomHGXLTtV}BPwIm_c$bR*<8E&_3tI%D3}An!S0v1s<6aY z_>=ySXG2brhu!2~YoB027Sz$xp^qBL`)OQzbqz~42x{xj{f|<@pfg3306DD+qFje@ z}dR%ZLrK)t{JN5)y~_!Gb z3=p`b6FrwcjJ^-tNte)PV13eN3h!YT8Z`0wpYiqruRZ)2smp<;tllCaqkC)?mmG@9 zwKG3Rzq`QIr$!EZoF;|Z6&ec=`<2>?iZ?vksi2-q+}(3#1M7wFyb^Q0DF!vOUAM=3 z^jv(?r$;9%TT8f3-glg*XMjYB=SyudY$>+js4t!W^nB5;&l#Dsv`2B$4pl8U!-&EZ zeWV26we1?cA6H9J(dQ9f6WT)FYsIjREK-zpO zs&j(nCM=?ThqR+*UxWsoR{~4Ko0F_dBTIx#*{5aiOg~HYkOu(5p$vjK56Lp_n1^+L zd}{8YimUUAlz@Jp0`dn2()}*&Fv1Wanro>fWqiSKc-!KU$txNRfNQ6)$q`RC7_eu@IxFD2Qk2w*HGnSDLA4BL-ndVMQcgCS_lZ1LEjogCV%N z!~D(Fc-lL@Nh^wS>_L8`CS2YA`Anf6 zO4P8D_rQKW8DkEAN)N`I)0+1zeveblrXx^0(6X@~yxT}WKrq}HEWBYXJ-GSeepCsJ%8$`mJJJ%R9qM!S}i=X9v3N(*jy zE|V*3>5i6AGA5E%Oxq<^ysk2?FfXOaI!&a??d3l}N6NOHqol_8+X;I2pT%>y%P7_} z?{_ryzElIylv7%A|7(L*!)4Djh{u3vUUTMJ!`(W-_dHU+tFuNd8utSt#<(2I?&TYc z0d|y=}@L%r54Tge?7&K<+%$g^uXe4cz#*^f`}2r40QkK0o z`?TyTIVVqw@+If-yp7VXDmqnJv`?BCdW*8Q<~X42JBa^&6ysFr@yeu(Fxl3BV~o3< z;PV0cTP~<)34Jl8Hd=pOLU*$=H%hgG2!^%FfiwQ%F|+Phb+gtQ!1=FQt6o#8Uth~M zrqW8e)sXP~ZyEUJP47T&e*<312S$zh0`}xfo@&j`8mDzF6-kQusoY5+J?hp$RHsAD z_rV6U#5qr={W{mQ1Lbe(;ohTaXWSW471KvJ&JIK)cgZM8wA z4H3nX^|RA$wHA)wYa`sZ{m*Hn!x?;VoGJF86kZvwz5>i{z(3}bB``=z{!>@`B{akJ zD>XBV{UO`HL<{J6n^RHej%b_5XD9>qM*ykrmVu~GMoCo%&B&QM>}9Fowr|7&FbUIC z_R|EoTXAu@!!G!(?OM!+9!tF2k>UiF^ok6y(-9!rLiI*(PR%uk+SM=9_c7UqSlW9V z5nU-#d7b8`gC9?~V;f@jy)1!>qHRUApJ|tAHY}za9&Z^u=!l5toEeAPsW|F)r;YJL z9*65G*(hqZ0}zw~>5lK=tT~4U9e=HTdLbUnvKp!TIOuOx3Ix!au645i0rWzTj+gl) zFcwce4bDnZ*@afLaG{!1C#7;( zsrfvPk{InZ1#*SC0&^x7rUVr{!tTydS0`@>VP3sf8t@Q}kDa_MxHoOmd(L%p5Sn~K zHys!C?OGu{#UPU%Iph3U5^2{~_DeVJp;uAy_({ZOJtjH`F66dde%;Z-<)8$=o0~^Gt+npajP{uyOXP)`Kq&gxNZEsP2WXOP733Aq_HMgDXrg;y;KuQBf zy(@t=4FF-?8vJ7ldOWcD_Opy@YfbymoDK)NjD>>m(M${n7sT`XrA`G7fRO-S^DB}= zf|rqC8wH)Qo8id;%xch-e>CY!+yDq@OWA`}{73+zpi}dU{(&*`DC&RCGr5!9`Rvzc zM?E0i@*ZsrS2b&Iy<8?z6Al$q_3Lb8w$s}y%|q=UPk@kd(f{7?8?_0ZpNXhaT=Aw( zDb?$|i$_}!{IOqFcrAoXWy^UT&^Yow$EC;JJ_2+c~>tf1_$6bHwsBZCVi{zF) zi-e$$snU9s2!1TCv+VDS_eD((FWj>gm4St12~)~FZHXfanL4+UhcL~Ra{^V&f!A!Q0p!xdeEdqom}^UwB!uT2&W8>0 zi1m7ATm*P9eUa9r57@okiFfyd2^#KkHT&wa2RH)rpPf-Qm$h|GK)om!^ul;Zigrd1 zx!PU~UgPu|Ti6wfv<~|-uQl%DoV_~qK?Q7o@Meskm;6-V`_Ue|xz=C;3MDI^PJ4qV8XDM2kr1TdvUyy>0F+UioXi84m0F2lwyi zUIw?mC=%ynUzR@@(F(gCn!Tk$k~w(J$Ty^;6b5J=o=L$T+u>ZpcdW*UXxaUoqfFiKaX0 z84aVbgV9P~UBR=%#Q?Zbjg{+RQ=enr8gA~$DZ=jlyT}7(1Cu4T+Q9Mr-he{;qY05n zGU@H>HiX$j=`%C!pCyw|o7NunqSa>QT(Wv`vSoDmXtaMmXv&F7wd;yC!TPSK(CIq# z@|~>myxJD}rCiXQ=K-G814J35+w7Zs^o+-rH)KWvUYQX&*W-;FgDM)ycBKlK6E-H% ze03`_EMbi=H@?zhUG1u>rx5!e;Db%a&CF`sJyv?A979LKnHTlMw3Kes36LG3kq^;A^P2#FGtVZ>$w(Aa11=*P5h)aS*5l^iP?;mumV3WT z%mPNxTB;=kELjwY?dTr!jG6jVE7id1o$5qpf9sRf5&I!c_NHhKpCt&vF`1?*9-aJe zwGuKx4%lR#=5xlBk?rv-Wze%#QF&~q9lO%u;MK%gV|PGYmw_EA zyFc(Gfc?aKpbt=$HQASk<)IC_2j(JpOTf7g&Tk6$Z35ntb``?75XxUuM|q6+oVBAn_QV35IvWGZu}TwnM4=Z;i;$X07C4SM__ z+7U0swj&QOFtCIlW<~(fQsZ0m;?kFF0y)06r~X90VZQYibV02*I5y%X>vRU^chumq z#GK#nUT|PdKlXyi1Kt|6vx%l`_eTHNgB^n@R}2d1bjm&c6tQbg>$NGvS-c8_!WpxK z+s#wna4_`XUF)|I3g;R%Z=*4b9~Kt`+G`{CKmA02)DTb5@IzMwXyMcnlqn^w;_3k{ z0-(KO0n5qBLw8OfJ`^oKCV~Tl@y*JnKvmnCPw;GLrlYpk<+Y=AhXg&~lk=-y9fOwx z2SPQKxT>X3G5&Adeeh&kM`vyjaz$$`Aqa6%v}}$p)%y#5{q6xpciW~W-|Cor1#W`v7d{55dBZ(qYJNKIdOk(U1l?qy2d-gA4= zV)k?H(Vx^cGVu@k_Hy*DMK^~t#1-Fbu$bdK%_#o;t>}l}lJ-ECFv4Ic+vp{gfwO$$ zf)CcngB(4S8q_F*DQpc2&k%CRvoQkCf*TVvctl-K+_G$*vJiZLl4Qs`>aD3y zI7@V2iTp*P05?BpjWV)hx(b75-sGc8Yy=keM5O0j|IvbCEPG+1n|qLQOxiq z(z$GH!~=#yX2J|NoyRLYOyVURX;T|1jQ)9Ac7Xdne4MiBN2`FGzK<~libYS4HCZCP z!qN`V>(I_%)>U~xdv@S3@PBbO74q;)lfGiM`jq<9)JD$sQ+K;*?_&IkF=P4Su+a=% zuRB~`CpRM^n{sCOYppX265-IGrVf{^kCx$n|6RTR>)(6F3*5Wv=4KayH9L{XM@^IE zbIE~3s++{I%_f-9RzUsXiOey1aA<_%V)QQJT&ey3gLbW}VuBjlZ0^T(NQ|D=t5#E? z!!z2LpXcMhTM}Y?)OzJX*AmHtvU;Y$uG1iGlk@q{B?9m(Eow;)l>3^x$$^#@wspX6 z;DrlE%o6MzDy%|@>ZGM))BvLIDYn1qWd|t7!fHUxXerQq_fa`s_HoQ#S^Gk3-eP=J z?C(3J4OSUPH~X3Q7?b~=NWdD}WgJec(}?F#9Je1VDP7K)TG%W5t1$ zccDYen!HQy%>RS5qCUZqZx}#{{qtI!y@Ekg*uo1%e0zE@X0sKz^kM=N_vNk7*cxN! z(1Vu_2k6TwJe>265WE&HIo(VWt$C-f5?$&!;9t0-n~xjinqO=$ILgw1O`!-kU2knp zNe&4p#+%D=&4q|Qwt(O(!`V6y9MOUekqPz z6!wu-j+!+dnI*duJdFG)q$UE8z6DVQD;OX=`D5_&j+(&flE(bv$?d{@c26~8LB@T@ z>^%yR^l>)p4a)hqwNyvzTaD5){@dH!fx3d?Dg!qS&LHQ%YZHZ5fkC$P3WGsCmB0pL zHCNAa$raqZilyj##X*&9IV7zP?}ik8$AuuFfX0m&15I zIzqe%rm;{!yP^_)na?o>NO`-Z`^#JX@ri3ESTxU~CLqh7e|fJZU-IGMK_i|9fapL- z;0S6@lB$swTUh2?`?or#WuIP=MjGo80;IN!Cz-yp;dl3wC@Qy2^*lfl@&W$b)}F|H zx==mRJ*H)!4_mc&vY}w5u4IyZy4d%9c6IH+_-j>vas}k_teum#?D9oF%u$4DofhOw zwn)AG)9ku?OzpA{_2zt_MvzL!ZDF&+xFo%{%7)gcfr=$_M&Ak8xi#3OkFdI~HNd3L zxxVM+sqhIvyC2GKU+&GSqeqMxS2?K3Yw{htoRA)#+32dj=HZ!DfEb!fSdL<=k7UF$ zV^5GitjCd7z;5n$@Nm<8mr7}OKaip3v%|GZ@uQbC&eMV7ilTY`mW4o=COh!%tZv9_ zmx!#t@7nlpB+WDNq9JxR3IRu>q!uhoxz~W&BD^wQw)j6SH~Zy)K4qlzR#fEF{`xD) zY4Xt)GsdxS?^8OD&Cjm@o<4AUUf#J~yp>nK14&qb>EUMD$Y#+JbFZLOw7}%-}#dg5M zy~M#U-sb+9x2Q(R6Qw(N*Q{qnpDz*Uo3TELDi*&!R30q(A6@=wS?w7T<(~$IDY{@w zEk?V3%I_lCmlGW3NaW#U^e)>RMu~f-40N!ys741PGPnmNvA2b|yXZU2{tsO8f%#2E z>ynjiF{Kq1XbyrGQP()7^?+qjn}n6PdH><7Wo=i?pdHKNd9SkTzC-M`Qqy4yBsaX_ ztT>%A-0wl!mWkIwgV{k3o%7T1-|g`N&8q_V$}}++*L@#_=bm#RXqyLzR8bZ|8(pMY zGofbjh;S6F-qz~`R)WUE1!dcEI091z{ktm4Z&&<@#y3g|VTy~ltTAPm#RGMy(cJta+_T#{yky`Tg_XD&}P z@)5x2jgqFm@P?3jvk5LbJ}!psAO! zYl4L>A0*m;kTP1w5EOcE&GBOtK+-UabFt8m-`Z_GIQnO2+@g+d$D6?Y>e1w)fu%R;(?;Qs)i>;2 z4dmmO6U%1~D#7HIgM-wUWJ{p4astU>>P@8LzY7OWRahsx@teKQ6+3yWywDRhzb@p8 zeVj;3ZzGFh?%uy4i^VzkpX|W;9ND8!uKGV@CyJyRx8de=9Eo1eIe~lP5S-7yeVU^e zC+XQAw9l%2+;z-iq=Q1lgr3Kh$EESB_+0$OK@*S}vfmR#2W+ETBMM%)w=w|*Qqk}f z`?JjB&8y1Zo!bcF2}Ikh9CggQQyQxa*1L3zseH`W{Q{^P)v98bqnRe+BIr?{%ELj_aNx|R$@#V`J$-gqmbPUx!mk+b@? zDCyWewl$}(jt-wZARYb$+18*y^ZwP6(-@tjT`nQC#*HpZh0-4jYPPV=b33yEGH&WB zbmiIqj6UbT8M?r}Qky4nX{hbp@&0l{+BB@p1bbEtGF~ztvD4ra*xK2hQ9*Cs6xpHT zJ{{$<1)s6lu;BNA{<+L~ntafds%5Aq!EtSC9_OOw5IkHbz=!=bN13{PkTnR;^jFxu zE4r@ROVBtTTS>>|gultp2K{m{{X;O5Xij;VjZyPgPPXh@m+Esqm3{PBbD9@bUr;uV z+PCYkdKK&hxX4_M)PawZXlHXz7~N&`BPmbL;tjS^UGu66F%DFP9A=C!dl=Eo4nIkR@qXW)GR|scoE_hz|;#7G6N4Zz`f@r zf~|-SmiL~#&pQ_!hRfFC=RMa%kpLYky+8(7Y;FHNfplEe3EvmUalw>A6P$YP2gOL+ zyZ4^1W3IL^0tVSqIAJv5xK5VW;2%H9flcRe*=z%q!fUOrcnZJM_FQ|P zY|F@&X92{j+?55Q9($<=e*imAQ)5Qnpxd{H7Pk>cuj4kgngv^~&!{OJ10Bu7@Q zs}u3F*ZsAUp8y$IK)AAs6A>aSRPOFG-^Z#Ya!^Ypm(AA`i|;)uMSM?h2^>kfcO2|k zI3Jyuke)Va*=-BHZ79V zzaKZGTIo)9K+lM~eEB`_ts{Y}j)-YbyxQsH-5F!*<;3xJnc>^!I{8a4vE$Jhoa40(%S+j`Sqwtwka*>?Ew2gd-O-=wWMNJm`!A&wr}X z&R*=hD zpeZEv95%z7?o_S8KQJhdEfQ*9+q)<+KD4dZ7R0t6qr&x5y?kFIz#oQl`4&~Rw~*h` zzlUW3j1#}931bvVRi&2bZ&!EUfRg*q=*8`~W;-(MvjrfLNx~=@!Fls&-32|5{l)_q z`3E$i=AY+9yANnu8)y`@NRALFvyNh&;9Zs24~)d72Y~`JSo(^-D)U@g65j6#+-uso)sLgT;XLq4pJPhJ8fjI-17L zHcwH;nEbhrai%RipCIlO4aNq@F}LI$>Q%(nljJmfgzRrq%NZ%Rklr?IGA8=1VM?mX z02g844h890t5d}L{^hCbzKGPClr{`3@`nOgf?3Z5Ed|ZJwt_8_`}Y#y^db1s_a+!R z<5=H_iL?R!w&94%R?x*b77GAJx$f`_5qPi;>Le5mr^1dYC6<+O;sMV5%pA{i8n19!b{rm<-6u#q)l-o?)Nvh=~LMu(j{+ zdVXK$dM_Y1AAa9{jeLslvx@xkl$yZR@6U(w$cz> ziw1G#(x9daTY5$Fqr7TWhEDknyL!XMr5`z>sJ~L?2%<~k@COND0Ktv5uJ`UUp{wV!d`M06V4w|%i=B$h_2 z`iMAR7-QfTn>gX1;f8?ldxilmkwLJot*QRt2n&pBrO38wgNrL>%E5&S$$fR-y?T8 z1Fi>LoJxQ$s%q#+x>G8tNZ*xXZz;+<);0aA>J}P zwZwgY*FhjdCh~o5gkwQOF<$K#l)=|HJJLg&H&2Hx(&ws7RXmw9els&-KS#ZJ*%Rf> z9Gy!QU1vNMA>C)qu#-nrc4`x5nVmoY4m3rWn;^r0aWn?K{!kklzZbY7FduMkSZPSU zFHf)U1Ah5|NXIQNxm0RL`{#EnQ#S1_JI2v>TWrQJzOEM6XyXCtC$jYkL;c!~G{(3Hlh3zryj<14>ctucw|t-alb{b@v@<5$+{I_++kKJ33^)f`*^ zi}jwGw@ZJLKQ8f;(PWuLT>Js3tMpy0(I)m4sWq6XR5YFUnU{tJpQg31Leh}nvxR`! zk-!up_b;~Q-^+uKzKR$NBYr>jBqe31q$w>EaW+O|;KTZ{>2p;T)7U>DVmpQ9+WBfr z=z=-<=Lw8%0bBxs?zB;J+n^1{XE=M?he}W6uu-F+|C+xGz zj4$gy`Y~JdTg$S6!D{M@X!35}Bdt<_m4+LOyFag&xc!Jk2#Xergw_T|H&OBk(@d0I z9EfHaCH{&5LsWr`*f%2_?r62q9;;tTCNyG>SKp#4UjohoC(uPgNihwiDt7jNzgU1>G*;_F z!qyp0k&Rg4_5G4>TH$_N_9Ar44G}bA(fp|e@dG30XvUiIv#8F=Yjz$jOQpQoz%V>& zkpK7YU75lf66Ab6LbPX?x}uI6*%!B#!!wucHkah((&6K~y-_r4Q#K5Lw;nqf!0C7h zJqaYkf`a-GiW_j{y78u3VviS9h3p=kuOqP~VXj0#f`~;{%t37K7L19d4L(VnJ8UK? zbFYL~2pXaO5DRfj6^fKS#6D^v4Reh|QYCr#Gj0606C(zfLH2&cEM2RrS_-HUC90FR zNyll6Q-`j+ZP*dTG-@&q_~);7{dc4M#oveNq4B`*|MG*}MA2s*J-)9Fdw}rB%fG`O zps9=t2=(5>rVxJ&Kg(_2En*7NoqVG9>TUT~4210V`|OkU9U6L1Qe>WcEI-AtI0iO^ zwv9IorQ#oPwz5aJI>TF?sArfL#lF5KhBpv^qm%_0<}}bUH!}3t|EkCJ;2s)TBPv6| zg4!&+!1(3V$Y|yw5U9#e_2eL{*j?-5rcgnOoc6IO3%iI-sfb`urgrCsOM2wU99{g7kTj zBw6-#ywb8}kY$eIBgDiH29-Csz#N9CbV9T;Q|q%IXHWFM*SlBs4#IAL7aiQ%(?Dg; zTcTX;LDuK1$Qs;!Hp;bmVebAF&*O3C3h5Zx9sgO@d9}weX<7b1s#f^b#lA$F47~63 zw&RTGo@YrG0F9qgZz=~`c3#v?-LJb}Yye(PgC}L(duCghc8XUuVgF97v90as#~7XP zd?a+j>=*;z)uE)O%oC`{4ZfMBu{skpS}jdfFnfHL(N|H+o?X`YDp(cieV4|=$Y&OrFD=rH@`e)FdrWRL>U;U_G@3TP=y8lE{MvE7nOeV%I zv9#M1JS_OgZ`gCQ$pQTgSVyi%&92wgjy9QqsMl`uzF;mzwOTepK`w32FrgmZkL>Dp zVhaqC>s8%-7VKbR560N28dcu^=ONPsEoFu2D(PC|N$P$&7&hS)5G;uHaJm|G>%4qUWS`w|ll~TAs86{4#+#ZbAD-0Oy3K}EfWO*C z3ryUvU-#y*NEzYD8H|N`P}=4DZEnQbT+7$mq|xqvO?uYkDNp6}7M(&4zh|`Od%6y ziXYLgzfN#^AZ0pBd~WfF4e?=5rdVk#Rio7Cv=s#!+Cw!vKR*gjI&Y2^UOTw*wu>&V z){Onn8krMma|Wpa6sPhebJJf04rb`;3}d@5c~B>pUcT+7a(s5G=RfZ9bm55mvgXW! zEO$Vh`>ixCP8X8V2mLd?SiM9>v@|;XqoPRO1Ul_-l*5nrtt9_hjAu4g8??Y|n|B|; z)~+k9Y-NdTTI+QQzm{&5?Q+V+B^FoC7SB{vp6kj4k-RXj*xZ30_Q+el>XoU-BewZ| z>&^YI3`(?mxYQ7sNn8)K(3D-iwXtGB$UgF2IfhPBbo3_}ZD#%-?%p!4sgewq6HY=tN!h-GXX19 z@h3~WH82$9j^bI4=(zUo^O5ROD?M;s;VzGz>?w5XpOhLPRV5oN*XDCp{0^I&dyb;m z0@-*{&?ZP4Z0>L+(`HH1hqB2I+we~HIZ;F-mzzRFZsoGvQD=CwpZ+=R0ZN%xok=GG z!EEaL&7|0ixmx7%)aAhLj`ke{c^P6U$eIiv-9&EV;agfx&#q_5P=2LS4OE&l+=?*M zdqi1xzgC5WM)D{v?fkirEZ0~0rP5wr;3y6l08e5oUr2t5 z3LvGN@r*YAbC}wQFyL1puGOfE7cS%o7NpSOjva`In^D5mM4XnsG(__ALB51Qz*(FT z=4Hp4HWSu4kA?A?iN!A3J$T|xXH&er{l<~9!B&X zV$(u*J8?}yW0qo|&+!9%8<{e@w{DPI{dOimC^vTD4-W)n!rX5QX81CI1o`2+j zqpS+S)#o_s6NUQX1=Nor+-5|4uRW4b*$?@itUYf1`}0N$*2W;H>}TL>;_nbq_iKt- zV&4lGa_%&uriEK9>=v@PmZB@akZUQ9M2oC9QfPR_drJbE-%*P~1@Lq|}R6GDwu}RNW$TIil!1U?=oKR&#uxi(PR6*&mcw=i)i0{zLlH zf0H|$a;^Ve{_z!vaHhz@qEYR{Q586RnktM2U`LeR;5&Qd?JdBx^`FmlBEfHE>vo>mW8R zz}jZ=*wx<$dnX734ZeRr<@IX7wBlqr;^jTOGEKOYYV^-Arc#X}Vi9nX7?GHo>6 zhU`|wLO>-=X}X#8Z*Cc?R|}lJtU*4GL5&JrW$jtFDPomHT-GNR>P`fj52RqzZ zdwiP&zUDs{1d8}iKIWYIeaT1awu3m^urZ|}`}Bu=fKsu=BH|&9W_jH8t-&EsW zsTp8n@xp7`a@R?F?7*jJ{FE{9V~^aQ&qSxaO*%Gh^LVwzpBGSr{l%@{QKGO3 z$xyj7+_qGEZ)^%~JeQZhpYtzvJHqaz9RpjN;BoVwir>n_jCz5a9TlilEQ}Iwcb)c_ zgXqeANXrwMyKF-zh3wo0L`GA^GX5HQ*-jwu~JzUlL-qkTGvmFoa#|3hj9-QzDmL5%Z03VP1*TBeLPYGuTDedD}#4aw8VccG=L65OSr*VdS8^x&8C(_+eS`O72kzg)4k6tM(L9D@Cf`>yMeJt- z9OW0XX6z)of3mEe2zKV--Uh|=JM+$`7*sz{!et}MVWmkXw4+vZ4!SY@eKl?5&}XLM zAIml2?72JEh(xn9x6S!fXV!*AxPn&wTu(Rk`-eIMoG!l@r8jki?6)Kpu})&KL~E$6 z`nP!97Mx1YfV3L)Yc1s?KC|(MS9$C}5v8xj6LR9$k<*nG(r!)JjFeeG*Fc4!Bmb!} zk=_j#`*}yyNHFg1`NG(uGjdlRY0R8UEaGxrs!w=JdW+JFiqgtk?3p+uYey6}g{*@2 zx6uidom+~hfg< zBsTiY8M^hwdDqB+>rf1DxETRq)dhq77~Xs3dJdGCSHCD~31wi}A=z8nk)^h7n11_| z!SG9yRB1+p)XaG3BwkW7wY#pXbzI)*f5P3kWc&``GPcoP`n~obc>IdCgu$hA*0ft~ zMf2;x7?8m5*mvDhfaYU1KDMAnF%j~@q}VB8yZPs2)FVr?lfS!aZ)Ip-b}hn=bnr5a#!e_MT`nb%Nr_A;b2`p!h@EK6?TegO!fT%tm)xK=OJ)2_P_^(X%v?2} z%|6M%khFYS=gmKKr*ew+?F%Sn3OG!$EORwYOrMIb^cRf-1H5Do>hA;Lm*S#n7c_Lu|Q-&!;D@HeI?fuQ9kq|Rx|0RTB(lY&z z*albNXaz#R1j=^`uO)NpJ~Rz#`t>m;j`{{WTZs~ui|f&fUX|5JgSK8#uQrVamv0rW z4$~&bF@V?#JpI|PP!DJ~+rOUh44 z2qx=LR7k_7V+LKcgJHDf#*`I&3ad&BA5Mr<9DYpt2h^7yBbicg`G-7tVrn4$PE?g0 z%MbS>NrfR?+u>F$G#~G?34R5t+q5S_Dj4!NS@bAF@2)dzGhcaSqgZY-Z|nw=rCv}z zWKRjUZ}{bXjXD!(fxfcq-|K}=c#^_rAryF5-oLs_0sc7fsm!V0PD0i*zq&NQHV3Zuc92z`N{?MlGbGn?(If^W=W7v5{u0d89yX86{;TMg2^wRBAsJbw3q zUk+pauJa0sKAfI<=JPRnZbSR$i%0_XLBKTM0}aDc9EI^}&8rY8e{DMExoip*_n$mz zTn5-o{BXMXLY3&)t)1hAJWdfC0xe1dixgf(MTv{IJTM59;( z+tLfOp8B?*8MqaB=1ttBonN3*=RV$K&J%n@4DIl7!fSEVnZ~TdV7Ui2>sb1pUa4_X z3=ja`K-uQ4g$-1Xlo@eMDYIb5rDLR*o!M_UHqqh4g$=WI;6UIS);@?fib}VT8>}Os z`@GOzqV>lo(zpQq^~vk}qrr(rmyL{Ex?eVuQFoZ1GSIkI{n-L!9+Vps58H>SUQh@d zJ8;b}buc>TCz=+wUOpY*jM1m9#+WezCGA-nQYuG;)nLrLgkQA7>Zp9OaNyee6`v*U zal+|H!4K4kI;IEe0bR_R9_aD|?l96x+l9*&7?R$7SD`z`(xQ$S2-jf1VJl(M zLJG~zr1HMxs!d2>f{d`p8D14WJ=cFgr`>u^enlRTnb~`NpZtOu|8rp*?ptrcg8)lt zQvTVG#J{O>?er#_&aVi9%$EmFn6K5>b2*Nuy4GS;HFJ&GY+O4lL8GdmFxAa$?G2At z@BE8O6wrswa3TUnf0BO30uY-YYAI`=_*S2B*ww*p;T1F4mp*hc%7q-;HRgi_JyGZV zon!f4L^zHI|9qXYcp^%FDaZQB7p;lmc`!nZ@0jU98wbNZS}%5x4BRCcyHm%V>6fq)tfgWva=w;p8& z-F3TFgh_BJJhYohNqKk`Kq7J%yo?6P7Sj){E>IjRWQQ+eT?_(U*En4$M4CEiZ4ZLK zkS~N=j(eP#$uTWpZ9V_>2R%`}{NT1JX6%O<#I$|_x^#W>rIN5if9T3B`$dM_aJY8D z8{?7U#6ow0PlpnIzu#dEa5ZCujClLk@T{A;5wO12L#_<%S8s!{{*&f{T248)Qzr8( zv1Zy^a_60ysl)DD|~O?;|z&+*c)!vVtZ$H(#lJ zMfpiu9b2fT=X4)-Y@+CnC*ENK3Ct4Kh=_V z6gHToJEi?W-6rtoK_?dk#s9f3eeRC*S&d%&ETk6e=%n!r1DQHe(F4=`S9^}%=~(Ln z-O4=L7oV*py&$P$R(I6l@9sn5dzkA6xSk6Oh@fxjlq@$SwsZWu3|6Q36*L?%pFM@q z-wL8LtqhcZihtIjE+LZ>XcaUFUtrF_=bd5NrIVW{^SeNIw>fjxpO$NH_71hu`R=O( zYaz{`y}U;B<#NyEyFnDJHNMlgGNt^hs68>21*#<%bX7O1&3Dr5C^MXc+65~tVb*}E z7s^yXV^IAfOljO$HPSsaEo_~YdA?VuP z<`VOOD>a+rAn2?eH)spHLV5%7;vcxv+Q0DP#V0Rri@M|YG-N{-bsq=R3)bSQ?;p{^ zErEQM;*l3RztAPv?|h^`l2(|s+R2o*f8=lo7t_-W?>Q_*q_t4Fq?+?7`PN@l!*(h( zY@F)-gf{Z+2TGqz_3Iz{jf60*=6PmxGe?_t?4JYGpY-K%DvB4Ax^+L~chDTZE!T-S z>B=~5R_~<$heUt3KRFO=^%umvs9XogS+=FnPpC88Pu8dwhqdck^k@jc$qRvC>i|Ca zZ^4QKH{TGRPt0m`AA9K7YsPe2*LNQ62F?+Gv-!FPx@~_`{>6!|h$=fvw#djby-|#d1>8l%w4xU&pQ*n7taLPpNROAzXt3DE!fs_R z$|d(&bwJCFty`5T0q@^@&Y{epk#_XG#w{U0-EPo}^F=n*KMm|JT8Q6bEM7CHFhY!; z1MXhzSi>pPozx^4RQtBVc(;l@)Z=Z!?qIH1sm!3ltFtXxJ3p~AzT+WrN*J>1Kc8VSK} zipV7;LZ1g8RJk^#`LxY>U(MrIys#~s6QkxhvaxhAW{hxR6u>j6@6u5Q5nO7ayZU0- z-7;p5U#K8I2Y;kwbh}qfO&@fQPKXZ z=nehn>@_=LbW2@NQ05uBcsd3hlRM%?y*A1avlsxP+>bPT(5c7LMV-m|Xi*Knw2gT_ zYU1Je*fMW;m3ak*y}K9wQzhpR*&zb6=@F1w4D2>Y&Jm1CVe9+39Y(6fDD-%E=20s( zZdlQWL1P(+Cp0ubxl;pIp3x7a-ENP>m9-4_z4~-5{eg0RlP*oI(zGIhxd6dR2er;# z1?03e(2PFecNX9(NgbS;sP=tEvk|aqjl;hw3bLZ*>Dzix+pM*d8T+Gh zInli~8d{OqX&-eEjxZvKwP>U@xD?*`z!JnZNXzjdPT~nSKZ;bFs7?n$)QEo8-S8QC zMVMH}#?6{Wc9N-xx(#0eZNZjCw25_>nM7D^x{V8Zv3Kn;L$f7MuUE7a`h0F?(<+P@ z10a`b6=Wq)!4c!T&!ta&icJ>B8^1+6V$3A=c>w^jHER>)&j+bL74Z2w@b1O=*G2HU zm61}@ms1Ar2{|3SrTm4cYT;f@QWu@wIB@bmXB_iT5U^1C4CQd5JExAu=?=!}jtP*~zR=nyv3Cn2(llF!2(> zb){~=(5M3|Mjh=&G#K}rW6-jo2Jtm+!N`^9rsERNN%NU^xvgqtZ-cyDbK`OltA~`e@^P75?bx54Bd$9=sWOo*8@jMLYNU34asa?fmIx${ZE0Fk4Rc?F*G#0VmQ9+Hij`}fxf)i^ zXKWEWj~+*pdU$gbtiAufYsAd$AxAiS>kIZRPNK{kPP~rMsF36+ z;g}DxC)e~B9{3H7Cv^1;TD!i)ybo_GmS9wiZ)Npi8t5ZKMrAPSv~WSmY-D>bSwsJ6 zk&{ujZ66+Xew!(+D^peQ$~WI`b(%vvA*9sIN3jx8`qPW__V*--rXzx$=tOwUEZfE_ zJ^MuvV`2<#C1aOv;W{VoysBDTS8|DkYgH;$_6e@J*O`hA!d7I*Ec>Ug(GYQtp=;qL zJOk%JbXMQH;uJpPJ(udpr>(YGQsBL{q~Q%TSQXxFF%lKmqB=LR>;CQjB~Rz4nDH)` zT~b4O38U$Ix8QN@opF#Zy6y)IkhR7b|CPR+X{d&qcjX`$xjtzOh>WFr!O)JPbeLhFR>Q)YqE42Nb1sES@AqVk8*bs#h}+zyq45) za*X=tbR}-^eFf@HvmNhZe8uw1!ZL9h5mN!jAue+J{tD`$7xpb^$P3K0_1kE7m<`46 zZX?Bj>Do~}x4t+W?KMSTFLRG|8~as39B8@r5T>9C1ERMONjU8{2mG*h4(TN0zlU%_ zQl`Dl_G``jxgA`3t18*2^&nqKqXY{8*6HY;t|yW&rvw|>O4RHTH`_)PY4`{I7g3d0 z(O5}(ie92^b1XQzt}XaLGF@IrGHj6C^f-nEsTKOHds6Rl^XN%mqEm_)P^475~dy)Dr(hw~XlOqMtLR_-b5 zWrrtME@jwq!_k+7xEag-D!nCt)B~Pi-I>(lX>lzhXI74_hLf<1T9R1hr|7^roW)OM z!By4Dt+vaOj@o96vl)NUM3&`ea^@m)v3(pR(h@u~_up|me6z{QBpWG$Y)QNp$Ln+-ryp3%Nbea?jX7wUfxV8`VsuCCtjm)X{! z+V^qd)=~*~wqO00!px1)z-Hx=k4lD#pR3(*Cf0o}{eorbZeRW|R_2F>jrFuB$$P@V zObg^}ctRgvkPS_BuW1~1O1yg&K6KSGw-{6VR{0s$Y8i`I*i;fLD)L`WhUxJa1%eCf z;P}(K#~orP8bU2bVY9Wr!f-B8XkN;nz9a2LUiFEm*NZdDW-1Etwx{WL_}d>1dH{rx zPr>haK-`DHj<3$fr}MtSw`kwTo3Ki~BMm>szb}>gpnZ)wqDu+h2s~Lu2=u2WZuEN9 z8r=r@a2e@v}>(hSb=)Uh*H* z_Z$p(+Sz~~#vU3fSJQ_~e;i}?!P2i5VS&10{}L_gPyf%Em7J(I=Lqc;zZ!YZ<$V0i z1yKzb9We_9IwBq<9f}ialbHp-rc63^Xr7kt2>u?uPSRm33UU9CwPZv z6K9+?0H%PaSF_m1iG^0~3Q%B|&&S81ZpO<{uqoT3H{oHy66;q2h!&LJo+s-+U6B; zKPS0o%i_>?6mUaH{-5W%6qQ=;0CkYN-0s$kNbiWd1NV%6Ng-skrQdf)z)i=)R;zvO&CGShG1U6n6xngtIf-F z)wMN<2_T|kXZ+jxe1v~tpMsSBGVFSQ1n~S+yA`?Lin6OkGNYR44zsl{#Ap03^q;)` z{-gbU;M|+r6W&Iu7@IsXt95TA)$A^I}5D4GGnWa0a?@C8LScg;wu|Ezqogu z^UW^=5+6k3sSTeAX_KEzSml0tfzbAS%6ePew|RQ@T6>VVBLC$d*tof2j4Q2-lzrTT zW>pJrbNa_RPNFTu4Ooef1p*{q`vL**? zc;vQY47sEO#@KaKh{Wt~p9cFp2b-7JI^pAZz^&xN7k%lw09_3DeICohN&=N+?TGt& zOix~G!+^`c4IBkwEOGM(^Ts@FD=QE*XX4E2;;O!NXHeO{x*z-_l~5P=0dpCmX9Upl z-g2CxRc*4__8^l#0^^*Sm!@m3lQygNHWVuNq>To)0H}+skb-6CyYJw%bi-26-`Z$h z)3x?Hc%d0B_%NN=i^BCqzTaTw{|fAi4>4=2SvB80hGekBK8a>|q5Y{6$8S4azs_1szY*T&}v>)Hv7FL=H1h#1Ltv1e-1ICH&yNr$2#teku-m3Xp@vLjF>T@Y~w zo1Ay?=iX8_X8HMkc6l@(-j!m}VpZu6OGFM~<*jn=<1~o@v?shIu4=iHFYe?;%x}$f z4sYMz-j}mZ-;>m5grE{2Q(ckT>Rh30N5_5F?D4h}{;XaDza*~6Nl$W3k2P_BZC}1G z(A<9e{K&2Xy!B~qpqFL!2=cZpPP8;)9{%{!^XEO z`x2{{oraSi}WvmSO6zU!wczg=^OT`&U1t=uT$#8*Bf}t zfwqH{>*Jfu?(qwP#iPn@sO?O`sKfYD03}(uc}pTJas~(?we4H4byD{N+-0AJ(_?k+ zDWWrija~3Vu|7(-&|7Z;EFwTQ=2Wk$k+t8q+_E-?&cXC^=Dw2zd+>XS?q&O8LR(x! z^}X=he#zL=sL#Ux2L|unm+pba=!k1;Y3U^CxHsSURN<1^zM&KRyxh0Bn#AkuQ!LAJ zO!gnFKibVjHIO2EbUJi>fZBc~Gy0yPiIqRn$|ix8^A=e3I~-QhMr4QxR{K$rl;3Gn zd&k-jcZ-L-DxCb}&?a_szd|doL zqe&LCf0^!RHS=KPAi#Aklor~SrnoVKCqBIoU9ODDRcN;D@up0^Ma?9;))B~{8ak)! zCfws^VnAMBR=lTFsB_M7J)fI6!Ok$ugRvA%S~(AoNV#LjEc}?6eJDT_GvY9 zcMaR&udn6`=p0id-fjAvSPgf=5r>d&kuIEX9hR^|xW#WyrtjbzoQABV<&3bxh&GYO zdV+kiXeZUSMGIGz9jnP5;`?0W;bPQ4`O9Q+ZVcT{Ep^<#n3Z_S477d+1ISY{A$MGI z@}1So2q<{W_^Qp7?RlGT8y%GZMI!?2+Bd6V6rNZgyB}q}e0siC{VFFO%SgIm z1noKzA}Jl2u~zp|=T2>;P1|_UCYu?M!w00qw*do!plcYHW)#i(O2qg5no`~s7ku3Y zV@Q=EtJ@a*i5Ed`!N$!g%qenn4Mf;+mW_O6>WdeBfv{fxdYKyH6Mh3%PfQEMLqOqv za|}NY^jxjOJ2s~JpLePuhv;(^;Mgnzaq=%ep)4C}f~hi1el@8M`g6b%1Xc4j&QH*ngOUY5{f^N z4^j1bwoGE8XbK<*7!p&+4&y`}z5`WD;oPIi+wErh6d73!-YqWXI6{h^YV;K{rKLLE zbu>R2)M|(wz8TP&znihMEVu5><-dlmsT`efNCwsqSVxjM;OuQ>rlTFOua%;4wK^2(`wWm2(D`fj(-xm4^w&yvz zKG`ALaWEF}sVwLnEyopZz=wEmcS;e3;EH*PF0LTGxtcxtEPPKGpZ()x4#}~@i$(3? zQN%E|KQO*rI)3z8;0<_N4Si;tRcKYjQH$?6nXsWUDzO-~JtbeTJMFXw`%F=pVg?`Au-?SYIr?4lA^bfOK@1t^WCj-S9_>p z*lT0mSh<#vgHlL109|wRnFLL~?IV8@k5evyo3iI}h_XK7v0 z7v$H3l(={d`?&eaSvZz5n+7#m<3Ummz##Ihw|BO@QW=AleP6rN1C>;EYl*>Nw@(OTsHDxOUfH8XtNP#(?*N#dq3s!zLk#_ z8knDe_M@NdI=_ev_ka5EoqriJw!9K1&IlK*7-+(cv5F(zI4uo-?1Zzpm5+6HM+u<| zD#B$LETKZD9b53RPw3`)!`K{&bRv|e`!kVl#+U{Z8r`1|kk6c)vZXh-a>YuwZyevA zM<657JfpYE!%G^Tz2Jn1Y%WO)q z!noqd_L|1j^qx1PbVG2ByK3w8lnFTFiQ2eA7ut``o0%9uHEb<}Pw;^M&F_ucCAR7H zzW-K2asCC+`iQ>lC>)t)2569wii4Ejr090b_B(7dGjrMoWN#w~(CdCVvwOO6m z!PY)I!1yt>*TLIU%T$7eX{x4p-v3Qd=!J{d0Gyk%F(`G z)j2<`J8$oaIPdNAuE#PoR>g08$tzWw-V!Aab}YQ|Xk3^QS=99WicPU{%@d4=RP7$S zmZEGLhfq~*FQL6CX%t#=VE6S?WnG6BW$w9!FO=9q*HgBfS_SDnnvkY!53C9lN>OfZ zrK=)46L2F;5vr^{{fjGPt)gLKcI08q#)GC+@$H>DXWA{?j2)Jdn?U11^0OMnlg%JS z#@t$3{214K457Me_Hm53b1W@)^Sz{RBbzP>ndK63R0LdY>vs+1b8XVpa;UCT@f1pNnWaB2 zFT%Q#PCNA$g4WW0>skne7HaYRxX>(ky%ZiG#&_>-9`v=(;YYJTsF`auRNTFce2;@( z$?k0=2PTogvf-2{gj&y_k&vlIT1YnA%T(;ox}P|6LU|U9Yq*VEYn1AvZ{ZOkm-GLGHDb-_Dj@R-mTV(GUTb8!Cmc57Fy`i^o%W$*Ej z>ud<1fbcwOVnQoMgAsn6C7+wh4Qqx>KS$WEyq}N_<^-Z3Wzd4|Zk~4NuXDey$Yok2 z;JQzHuJpmqe{QYsFg9!8B$7%X=mA6|1o4eu_xOQDb)|o<&ll{`U8;3A{q6|Sofw(1 z8)kPjp5kuyY6B ztC8(pQ(t#m&9BW5(gNLE(q8XXNdruAeq}&E(B0brIu%ApQN!etgoYBvFH|(f3Aq&1 z2$BNJ;?q>1Z%%-<@%)Qcpk z$x&PDKCj%d6-z&TfdS#Nehn1dGPY5%+8e053f!|NHe~Qf=A`*jv zVG_nMv^RK<-rxD4+73YjUSkvP0DGo2fgRim>1sa7y^gX;c*o91!J=;!o=3E=KS=|@pAOc1-Cj;`Zt8{Uy2;abr!3VcupKDZX@d4vKqfFG z&PGY)+oOI^@g1l$9-=~=Pv}d(!``}EI<_89tN&#MS8C5T^@C>Pi^Kp8R~ic{0+xv- z&E~{#de^S2lIb??x*u(;CYUqvkfM97-%zlQ&rZHHw$JA}L#xfZst6y_6msE$WVLhv zE#mek0EUQ?utu!-=kFL!gye++3~X48&94T-;`w;qf{+R2E&w$hW{zF&)@{aWcgVpi z{1*IX{Ftf*`C%9K3wam#lSu{zmItl*Q(D`;z{ zJyRTeX_IZm8t9#AKb?2VieO^3uXLs0mO!D3LR?vkUU%JhzDIRG(GcyV#ye9UIn4o~ zcaV=pX!T|O$?@?kQYVge&0h4`dL_2|4EXmF68+k;(-#W8;YUDE#)hbuD}X{@5X8cC zz&YF_Ij;uI;_T;!PkGqWJ&kc=J)d^hZ7uxhhO`2>M~B21r7u|F9JufD3c37!l z{^gBhsg@jt6ZgiKSAy1j*U=z&*AMt4KtrVQx%Tg#GcA9l{`4L~Vhh!;)y!C6-lIjI zEKrrwK-J_kIyUPcR65!&_lZRsCGWCS^k|0D3GZWx%drdco`V)RS~H%WSUk&0qT3u`C6HtYY|lt>;AB4G^`Z zlj`RV-s3PPD)0)rsv-%+WTWe)_a*Cq4>$xO_UpemC*zWzd*WHP_{i(|;csJm5kf=L z)DiV_(8crvZww~3ZZ3mjJprhqkg_T} zLSdG$M0>J=wT^2$1fy^bAjrN=T==S*mq3_&vhyOzp6UMfXDvJ}O`1|_0#Bkc8-XxK zc90V08fY4L)PdHM_@mu*4?rnJ%kX=AOgTS6d4#~`!+=emY;k1aGqOD9D($ZdiPQ+* zyJa!8nqdbGZmly#N&Q(--<`tvc1FGeNeoKu4qQJJuHO zmI|64Z#9A(T@n87bm{Y+A`1fvUKN(X7 zAZ`V~rcL_1*|B5*;OluZ1ki^5?RCUFUPtc{ZtD1ani zvn4d3znLv(VqPZsNW8z|Oq!kySj2z&W8PG9gqPD6PiTOyiJW2t2D8|4sQ`djx~3|T zJlIU9yoBln=O+RP#fia7+l{#ptCuF#&ra-+1)pBrVPbS&9R2K(v1}&rGC`*(M@86% ztgSDT8wK%gQ@ynZoL-US1{h-kRZ;?~xem)@wp8 zuN7TbhHhQ;gor>e>I%wxso-+%t336bM+nhy>y3^Ief~sun<|d7{)83VSEI6&Z$ilp z?=y3k$qjTTP);-L529qKSF{R%F}O!CHdnTo^s=0e|M`?uo4X z)AKst-qj(-zGh)?@%?P9!8VRPBt-PNed8}8RMb3!;{!IGTc-B&kDiPiFCxKh8f91b z2Vy%*KdNFWE=SuTfFdbQD>QI5=DVs)zc2p~4+XQlLe@*r8snok%dE^WSs$`c$l-j) z`0KlfHlNpTkqdF2!!V^3FZy+a`QnlJc-dQT2tmS4SJw}3LxN{U!TjW|i(*)- zmWjD4H*WI>fX0Qao5+CXn`Leo$XT(T2PqVoUGeTqUSgOU&uJ#ah(h z?*~*X+|uRD&pw#IcyK+xF&Xu_5~@wGF?wl~MfUZ4oTJFCW%7pl2alelTDqRfRS1Tu zx{#SOGlYy^!b|2Z;8(`|Y-yahq(@(O{&aX>JK)me+*i@{s&0Lv4JlR8gxv7C(1O+a zO5KRY^MxOQtmr!Y= z>UT4VEUIGO=(w#zvmEabqqYwVx}e|aH4dXKrH@N1Y8!3^>c+014?PunXtY)0Iq|!9 z-7Xe$ruu|ER?2-FLoTg#tq=`4O={+w@Vl#nmP|Ir%+bzW!E;~7XOiMi(J_2wTN|W+ zl#We0FlFl54ETYr?v3K8)VD=n8n|r;|F}6_INDbJ%5*C4)CtBBG(9q{quOWsQo z7{i#pd3oY`-m5eCEb(lYJnC9Xs<@T^vh$0R%S`?X`!GyKvg7QuiPc)lyA&RNqpG8F zy}l1-Xf=zG>S|;m4DhVx;sc6}1x{37DGjEQ;1bkH6xZq?5s{V3L>vJcPOT|Ryu0_1 zQ)I&bHW^^)?Ur*o_1{fqW*e;DsKyvhIuflD6!}M40CWsP6R(zOlR+^6AGFgL-g=uw z%t8BQoUZ+~zHY=EGN@McvY?gNe6BOkST8S6#-lMOsm3xxp0rUQXQhS zl|A=4yjfRvK}|@zhGrPfpSoeK`T?=rrr!W^gFC{Ed~hk-NZ_+e_#kvnGhY&s&9W4B zIY^JS2IHJMdW?ea_#f4K`*d|1=igC>TX(_+~Womb@q2I=x!Z z1pW2YCz^yg3f^)^?wjU&+vFikowJg_-*S!t_3T_m+BOSx4RCY@##ryZ@YkE_-HHP;%HP7RAHmOk*|sNb7~`%WCq3Q zEKR-ozimE>+pab3p?G9k==M*zeN`)J@Irgo#TFonHq?)DqR99Bu4?KkIS^(iw{@Q# zDe=ui;PqbP$PG(wel=ObFsy2)b48{0RV225$6D|Pi@C*JgpQ9;kms@VM}4uFF$gdH za;Z>Y{sua8>20^<$HgYjzJd6m<$#(kwRrU-(ftgUVb}xjwX4)S%3~ z6g5E{8oZBvgZ%B@N2cRvH&3sbzw5a5%@fnAv0s^@Q|2vqeZzQW@zvv62Txupk^$8is!-BV4X8BCjr|DtcG!{E(I z+I8o^yLSkHV%^rqL?Rep^PK;?E)lT(*m!ff=yYEfJ)zYMlW9r zqwH(_)!VZ-=avebuiV4$Ifz!Zi^+9LVfk+{573YJ7skf{>FmQRb!r_nSMOzidYJC! z+tJ^@0TVCazW0fMiGmj>M+NJgNF9l&kCK@Q;@XVR(64x`{8y-!^{-MfI7#VsNrTwj z=6`=3RepnLFHYp}TVm)-`dIIWMKeYI`Z`@jA<@BG-P?=F;mxOp%+tSpik;3dMwOe& z1>366z};L`jMwBv!=7_Kv40FM#&3FoJO3Gg_!?cT@^1&+8$fNtni}H#^5rwEOW#J2 ziRlj=x*B(SWAHH5m^VIu%wmsQ_2>7`R-=89AQ8J@o=5!KBu&IZ3HsOIt7s&-=!<_v z&G&LZqtgpoVi6FvgK@v>&p?mLfvr+J;xy8lRT58vDjPnemp%3AOaE%~ME-aF4* z%7g#l+tVz-OhkFNXAe`kMde<|I~7=K4&WziMkds}{(gf};Qd(`;F}m1N{0;IWOA~8 z;7Ft1!)nP?-gR|*yNrT0eXo9jc~HCf?!_Jg-M?4}|&w*Vo3%DJe*`ZLX*$h`wQ=1fMH8K5XloJ<0m z4KjZ2_u&mHKS*2KIj#FFmi^ifXGN{dr@i$?ORZctq@cxq6y|w+0CY;I6Oms3tt9$( zEN5?KAn1+0ZS3+`OPXy{zs!;QTY^G;Fnz)2-&uZaGR+G8mR_#5QH#o*xkUAdtAC-5 zVddPhzB&1ci35Bkd~!d6siTM^Q=waIh7F|FFF}vK{5a!CoPbNxhPlA0cQ~QxJ1sZS zwG`~ufHRn|$js_5B0xg?sM?x*^w-M4umn7UPbB`+f+S1l;@{^A5P#);HCT54D#_GTK~2r{D1R}pTqyHGD0~Wv|0k&^C+ZhkALI@ zHv*zTB(zql>liCBz$gaM>8c`lcOk&2+(I#oXVu|QsozA$F+fD^aB6)f>a8hRcxFK_M)>a6jyv7OLnqaIA z4Aw3tb;mH4-)0lb8f7@j#~IWCu}y2;^n-LEn0DJGK<73@w7WfHjj7}eI+fAd#)CxM z;A__yuMC*Rr7CFdgc|#`s(iG{<^{dgTSwj6{$Z}Gtkqa`1`AvDYerXVjzX;!HU9KD z0q4leoCM&MVfuEu9&uz?R{Y!Gn42*nwI^iq|LX0n0-|c$_HR*<1`(x0P>}ADR1uI8 zY3UegkdSU^Ns$gkK#&}yhn9w+5f~ZNwX!TY)2=h^u-zqQ2<%&_8I*L9x9 z@jLEpdq_VyIML*T2748@C?NMr&o7nO%StC*A>p^LgW2i@M*!5?bQ~dm?4p9ToRWB< z-EKpOS#cI8;Pq?TVWBq8JBX%zcG>X+D>x$5nDiqLY17Vk!Fs*D7NfZrzL{s=KFX&tY2a&?c$XjcztO478YOZ{W82r%3#o$DZuZ-r>z+ok=f$W~+fe7I5q_h= zd^pAo_|jK4@lo2AmtzlsVMhET`~t1FHW>XH--fYdS#_Wv?%KR6mjQD)8-8)c8(z6j z#tVGMW^r#)VBWdgNT|~PmO9ocx4{B0#!=MhIF{9$8u~|?FYk8u9NXP;Kpr|s4rmaHR*>DB&-8$60c^(mtG?IWy=KjU6jswmJl?{MBt z3NE{zWO#vFlQ7d>NdVi!&UzF2Za~%BB=g*vtti1fJJBe7HM<*ppyn2eT}krP*$Mv}l=oj?14>=f%XzPLx( zFHlXpHR^z7cFE$j{nJHf{39*?!)jcFurY6I#g*QDqGn0=Nf;VXmqya6$WFO z$?iXPWCfScVy`r`X{MRrGzw1sI9oJar70`o<42UVuG^c9OP`hzy8P2YW88NoySSh+b6anj-T>ip$G#KCrQ zeJ^n3_ezoYqG5}v53ltD;}fyC6SV}8;S?5AaNr6&Ulr(*F+nUar@eQr$dg8mXKqKB z4?c4}xI@gPZR|bizE8#f%HQ(jONn2e!t0&s@eD|FGy&!85 zs}%c`a;T5`jQD^|=yfRHRPlHkTy6S&u*~MuiM?KpS9~-dY^-@8*dtMx{wMFa4z(~e z-i`Z$se#@YHMY|&#!F=cE_dEv7CFM~(VWy`K(@D5fY*=(pfRN~`i%~_ml3r14(HDe z1R#VHAK1Ygv1^Gm%%h+_1Q0ZvL0e}!uebUENDbcbYF$2$-|#7Y9JqYRBakP+X}{6* zl%8r{8r3!$Rcp^#b>GS_d$^j|xWlaad>G`qE-CJ?>>>$zU!RY2jE;l1$1dJ6%u-qV z^Jcrki8M801T_q?lZ?{UN<{fE6qnF-4nqi?%&bnEnRN|-REGU%U|RUdxp1e@izM^P zTVGVm6Y!nmlEjF=E+@Z+!K&Z3LNzONkk#1t7XW@sH*qp(Z5tEW7m>(6oSg}p`4c}% z5qXJ$o?j%-Hb2yrDW8^-eqj62N7<|VOBY-&!hBbp!-P-Kzy^c>}U8)TDHn)M;nAcORM5TDTT$N2S1MEI&VIWPGdNwjNlUM7}*m`)@Q~Y zxU~yl)NUB_1ZpmdnG<%%l6<{9v*u4$T6w30K9syAD;s@?5y;VEzozcX>UQQcg;!OS z)T!B?CCnu*aIfNhpCsyhT>eOrOHUJbTVkfm9|rRaemaq)YV^pNNOVP4ySC9q)+g)R ziYJMqrKR_JOO_*qa<-xL%E8W=9OVGih@U$A406SE!}pDdFt@DbDj>UT+L6f- zqn633Q%gLbzrO5PX7{-i2%S@&zRrf0q2yAOH4|nVARJcaRI4_dx)I^l+*i)`C^;3; zwN>mj{f4-`lqof{|Ht$XBGE-{RC6@4aW6w=*MLO&S#}wDSF++#i!1|%*xZ8}Hp58) z)-kuEmi0zE&yU_o=Mb571t%p~-kaE#KiRqxUrm>Xx-!(KdZSz4v-5Yg z_XeIrXY2)XkaeS*VwET8r?6Qc zM7c_qnBgDU9)(u8yymM(``OrovtI-`tHtzK43dEuz^qdu8;}QZMlZc=4DAv$ z@gf364-Bxkbv}Bj@YUoc#l^w2smqyUJqraxXg~z&$*FKt!5j15I7j&!n`V?yK?;M7 z8e{%i4gC%sIq$NUwqsHH_i+b4%zDYsg?kVcx>?eJo1=w3 zXD@eqor`@mc%c1Z_Lt462tpC|a^GCf-5GXRKLaj`=z9n%Efr9Y?%5^5aHGUS$AIRIR}X6N7_^pG}qz;2rKubL4h3{0@wRyyikIP*08grgYJn+!$vb zB{7qH!5sY!WxW)5hHfOCv`pC4F65D}J!tSJ!i&OZ5E=FRHUuj{)2HfAdzS=M zN8Mck007e7mH{wqc___Wrtko(Q3tN#SRQ`K3min`BsECIlKl`fP_@3#sgNkyvNivg zp3tGG@~p<}`G!@Agv`R2!}l3u_$mP%;Ys^3JJ=#c>oFC_;^RfQfm^E}8H@$_8D`ED zg%IURj9Z(n2WHm`<+(f(bUK{96zh(u24c3MX|@T-@?r|PLp#YwH3w4Rb*aV=1dt!O z5r-S8%r2jKdMpAwE!byA=x+#qM%M+UUx21qK4){%`R!*XZl4d4wP|qSzH5ZjYivbw z#rH!>4I2cB!x#+*{8gBBf{6otKlL5?B*I`+IxBB3lg@M5{}I&j|8c$~RB5DtxS*oD zn_V4i(o`4l~Cql{x0g zX9C-5&#`)!G+DoH`yg}Ez9vZ3bvus-C$Z*B5>m0sQcoh~#9TVUzTVwZUkUp#+N64Ub$IZ(#oYW6$XsVJ>$5Z^P?1DT1gCTi*B^inrppd6K1bs`F!#B zOBwJq)@5Bm%!cWSq4N6sbQt~xLugWwb(n}FzSD%U`VZ4z$E2Fo7{3>rxPP>zP(T(3 zJh(NmxN4x4NnxMCsA2^26L{oj&egvlU$wE8e{6AjqbGg zcMw{b11-8aMi)}6t)S_o_;|7JLSO}#@#_6_s0%F23zn)To|XsRQ}kKxE*SkZ=sVf0 z%0!=hEaV6miZ3$flj6lIWw1))SzAJF%TcT3D`$CxnS;XZ1^arWAvp~sq{Fuv74fEC8kJjfa1+tSXgQ>U*YEoPD(f+_W<~K65 zyJ!7G&do2^3wnAR*3V%?!4cA?GOJ1SRhY-(q=JeR^lj`Lmmk%xK+S&2*rS_)@<^JB z15lxS$%;fc<1lp5w4|vwlz{lPU5+}nmLd(T=&tX4-iIDRpSa3W$eO`)|Lif%BN&D- zaR>oq>$E7OVyF#blU=EE6Rp89k{KLaoAnFjl>E}pv^r;iLmCmK2ft&!EHM*BWHyD4 zRfY|41E|FFXsRzxyzL7v*Sei;xcxnEY%+wn827j~4zH~m<O&{sYZ z1Ue6ktq<)FGSt?DK(=A_=}8F>x4{2YSEn$5h}P{Z@k-|y$0#6~1EFJi_$4*$BxI_6 zPj){{?Cn7M`q@S(D^c3EY-|QgMLTRyjQG|{=C5yH{?d4WA=NQ&+Ce8PX)sCFHue>+ z_(N%CZIEF-+pzCWeKlDTmzO{;kB#QQ^+I}d4EB?NF3SsqifEOQD(=HIXA&rP%vcL= z_P%IM8q_P!QZ)@|;T2Fz05R#eRuJ+=zUn zjl1^?J-#FXTCrKP-QnN9{jO!(w#L~+9qSkE(P_-%LWvy#MX_9z%(}`q{MQ}W$rrzm z#fnP8-)t7o$G4*O_f4cB^Tg6wzA_Qt9kgrQ^f*KL-LKWv*pOtRXVsMxR~e zX;%NohpVDvQ}I6>8p1demDYb?t-iAtrTK4j3}Bg+0kp`exn0v-5${2}z=AWOPr8tk zyO=+Y@SShm4@S+&wXMocV*hi;hT3D3Cw?w;3#}7$$}~il$6rKgOeiY?_ieh3Y-CuI zt}v_iwdCVp_$o)8(cLbm@4I`ZJ~e{g zrn&C0*V~IuTNg_|;DC9Z*GJJ% zX*nvay?%$`cS?y1fl{acyJCC?VT*X<%U5{be+u$ikOd3!`vDhdnmZPejJj_X!eI{e zTW#ZUe_e^q8uIz5O?N|?Jqb5L>GgdpBD*X`*#B$OIv0mip&qM%(lX&!}$5J=> zbhM*J3#WNB4*(L4VwbYx!L^sOV=^`f>#XUg;+y%mi?NWBPobtPt5e@KOjv?Z@g3BR zv;8CMotV=%A|C;(>#_zZZc7hbxkxqTYg^9f7+0HSNpco6O;9dt)63vaGXD#y7By&~ z3c?AJ*7bEorE2P)aKqkINpgnrc#?G<+t{byz&-94YY-$tJJ6OIf9QC%?dDI zDU0&MXPVWU5D#}4xs#iDK2YF&D=l2}X$5He$7ax<(!OICI_7}9*v8{6DPmDGUzX`V zPe&43zLGLS*{d+5WfCXw9P?=UROgvO4GkkmiH8j7ABTnq=MB6x`FNhDXehYKY}9|` zqXh*O`#OM;d`hINW~<<7)$D}oc9BtRSilF^v-bqM0uy8u9H5}6*7tVCO?oR=&5exI z9%nVeKHlY}inlS(d%X2oRFOF#*ePUVA81F?cQ@;qRhc}ytkiwGl_@A>`4n)dk6MG* zs9gdghe9LX{Jm)Fo@R}Z`Sk&t{ugE0GCd2*MO~^6`CC(!c!xFjz#LWnj!&!5IVh4h z?fhA6iWXghz=QVT_E9_AAfOPX{^C-DB*Kq1>S|P!nwb~WGMK^J!DvTji*0n|r*Kx} zCRm}q(WBkkYepzgybMpmOiHy2J@}$i5zcBXxDK#UW_QOst$A2+iOw#WO6M_C{ir6l z4(B^=P&9IB%7p!`A}BcO)E5HrR21)xOiBHz(`4PswJ`WxbG5dbHlmT~%|ftJ`ha{B zC7I846-!B78)~qpI-AF@!HO(JYp&Q5XTk@j?c*GQb)^8PjeDMM+#oOtnZ4N3Zc1xP z=Gm%S7P<+oXhkQSRp7^u>Sc<%Jb^~}$h6lOXfpz{L{BcygNu80IfgX{e0t{~&h=Bs z@R|{J&6G38cf`JM={Vpte+LVR(nRHmba;&&a>;0+WE5gsuDbay6Tw2!C6q4L%)g_lQDx~?zS&j37 zr>wkxo7MPK(`IY-n15}(_W8VQ=-ZmLfeCT!n6CN{psY%~ev2w2-^J`3-ji6%Q#lP$ z1o_}mTa&y`Jaz~2%KOM9J5&RLj3Rr7+c@GaCipVy?sk^{;)liM%T<%KyG-=!)&i_INzBLUgXA3baBaQtMDeK!|+;`Drxwk8tjmuoAn2{VNZ;t5dWCzF&3W*KUWawre$&p1O;)cqed1^p-`E=Txc zbcdJTq2DDYe?$eg=+sb_Iu5K3nf!2%iA3FhcQt7g~kHPJlX> z9+@J&t{Ax}KpoVIAmkp~4m~Tg`QbB$Puhu8JNn68b2p*Z6d(nFpg*VUDX_|w(eTY& z0EUT(7@UZ1!=q4H)KBaBeS=9Vb)ZlzDR;k$Z@);`M2#2V*}8qJVB$KqSZ(c!RjgQL z{#QJ9`z=FWIR?o7uo9TWkiV2~1Y`HXLUZlU>LcV+a|y?3GDm03BdU7hIVS8k{!goB zG9WSLB~>@-zQ;fGC?+o{-W6QEAn>*27gdq7viagP2$ELX@1p!4y;4c9GHgOEYZG+~ ziGCU0;cGnIxW0KxzK|U*zzU`+3VqvFa_oqDvlQ6hU*^SoVQJsKRU2sTGp$EnhX#Vb z@~|!K7}{T)4qw(k%4N6m=hq;CK6&MAHT8GY%L2v^^48jhoDU@Hz)-uHbN z^Z^DxvjY`$W6*Nd<+Jhu&8>D{l{gkT^1| zjktGx;8lFIP30ud$QkyXy5StBI9fEPdKaHJf=k2|*J;|i`NQT|dw!sF>gBO6RtnvJ zBF|TzcfM7o4WDDZoR7qcq$eLwUv6EpSSdQSjV=U1qiDBh4oPP^@qRyPzu~~FT(TGw z0ImQ(^HZ}e%k|)1MSQ0m3T0bD8U;;P6S=!Peb^Fh8wWyY0phJx&Zl(JRny4{X!98h zgLU4Oj*0mr3o6Pl@OYN~*qTwDN9oF2pH(B1rxRwKr!RviYe7Fe z{Rym7Z+M_t=V+5T=a&~FTAmxaS9rbj$i*DY=STSff;Pc@+BjL!gBtP$1OOQ~4K@{F z_}r6pd5bqqZAxE{&%}_#MzCi2#|~nThsfZ-Xej?SeDZ(LKnH^MRbEZuPEm_%#=@C0 zLDLp(=|YsRhZJ{Z$2|Gjksk|m)6{`{U8z8#7IibJlN;S@*=8e-iV>$bxyPogQYiQ{ z)$_Kxo%*kT>aBRy2|UIltm0L7dxposuKO8-fbq{YNJdeOe-;)@>@ESQes0XoJq(5DOK)8ZW23F85o9y)p2 zOs1r)34&uQ)wyLw%nbo@pBAV7c}E14{tUhK{5h8`d6gCjA7BKX3nWM2{62h+y3yRT z`D$`0$Vs!L>(lu4eMTR?rU*ie)wo3XvVq~)48Sxj#}%V(ixFRQ#W|;-<52Xry^q0-{?S4<264}YLEoE#pHqHFc zXPI!{UvG`-sF|>lKuU)*mJDoCWpwxcTf?Xjy(iqr_>DN{isCa#goRk1-p%O50J0M6xHi9S z;nJ~zv=pr$2I_4NG6apkLL`f<-sFqCWz<4Cb?k^g`0|TlxgKaw?GM{^*;di?Y^TEO zDPBt^I+oS~_yJq1#ZY~EclJszno4Y$8g1zL|0kPejDm34<$jQCffaKvANG^6hoc*S z;V+Y25LI!}4e^rS+~q(+cov;I88@<-=;H*(l>_>T>oLy0)ItU73vX>YifuM-H0!UmT}tb0lb z+);x*qkfn9>g!_Et=UU+KmA})Aa#aR;xl8{j&xdzslUL@(1O3xS8RBZx7j7IRLP8M z-))FZu;G#ngR~BbQpYNIdp|gDvEjc*8-ge%aqL#U8z#pEtQ^$%J?GmC=b-9|{2@O- zBStN4<+O~u4v?l0oXc@1Z$g-boJ5(tBRV>bUWs{p^Gr6P-nJ3t4@4$`g1e4|y5(Z@ubw>MY&-rhx11@-)QBaf ze$?n|2>9nsWZD?%=3|Ad#+=mPG{6!rOF@?f6lhZ1!QIYDXD<1_5<>z!yRO2=egys( z^H7~92=AZut=mvbEhTyWuoW{b4P_U@h&oTh-xR@?afYCfryf{t|6T3 zTY~f4eUBv9ISt^>^_{T%Rm2_&lsh!?;Ak3|t4cUd?3BBr%ygcI<9&H+1T!6={UVS8 ziP@X*busp%;$OvFPf0{{ye^;@vb{urn%1c8?Bah;ugS$9ANL|*G~ zK#Vrldh?MJ8wPA4He^d?K$IW!jg<7V^gQ>4-RSXr9YjQm0r}-BF%sNLFuUWqXB6OW zIk?pF&PiMyHoY9-@>h1LNoLe&4USNn$J)`50X}1=0;QX3CbEJ$bMfqoT+S7Wq8c@}Ib- zvQJX6^=yMEzznoFud&nceZf^Nli!8GvCC&YYIcO7wdKEOTFFlyNwz z+Jkn@yZVSOV1{7qpYQ&ume&uPg0!0AekP{VK-v@Vg)uW4`a{ykXpMjQtcu<%C9+)B z?B6gQ1FipKYHL#UrZ}y1>#95)ka>cH>uJ1lxi*yZ{R3RTIyviV-$u8}^A1Qibcm&7{NTKQg)5noRXlfOz}A5al26amAAtA^Y=u6 zG#m?C-=aAEO0&eP@(DcBvHH`1_ZJ(&(frrg-OQVnLx3uuHRA@FoZFWXKY?bZtNxSZ zi`+$ckSqdjFef@`2C0=g@8<^;R3W}nCtvd~r2f7T0MV8mZik5X56p6%*@ibY1!F$w zRzOeY$d+hMVG1;-xPZ^YIfP4Nq*?fUs%JnZ;ruV(|)dxlt{IJFMgHY^u zyW)P&w-R^MU^~Il(vW{Ey$#0Ty|W`uTN}~Ttk*5+_frC+oiN3~E}K`och~{5{EYPv z_xa8r2L`3PmWN*5f6F!kG!k-aWOCZD z1M@zipv@zIvEEr#vPwRu)-}45pLH|%il2tP)@2U>1>kQpM+7c0+fA!@xc8;F)UO0@ zRscumK!(Rj3)dGmn*ov&JMNLfu9+X;(er9u&nvSzjA1=ht)JcWz&r0R=o`?2gn!>N z@aPSMVMX@B-`~EDbtb{)^h96xzG9(OlTKEdR6A*&IN@YasyH=r@2U-5l^z&oLr2j3R3jBBvW4G@tXu z72RSzdep>6wS0jFdM#2!1ucI-sBwdFbH+s3z~HlBW%^x)c%r3)dtfE=MIJZpw^5~u z&ctupwTLB7etz{^AR>Z2$EI|ttRC0q7+A3esh zi_H}R9?Jp&=Vx~uClcPqoJCPi&DZgZpWlA}lRx}zV7$sB&%9DIycr|p?U>!EcQ>0Z zgZhuL>4&K;?{dgf%8i^c^U+VlwX?99hAz#mxiz``4JJ9G!R1EFK8&&{3VIara!YL7 z$78RZuTaE7=h~$3XZUgFMBq3v4I@oJaC|$a)bmbirz2*b?ukWb`T$GtMdpXaoZNe) zwu{=e90M{zJqNlE=eUZ2!HuC8=D~7>Pb9p}#Hb>&hg0gU$vp$!Q~r8~KQQfVq^H;U zuY-pFyk6|J0@`cRv(3sr8S2~e;!fFNXaH&1b8ZXKJqqw+ZX3t9Pti&MaV~RYAwi@;%sabQu+;BB;{12e`*Yng#7Pa9r^!}r1&i}GxG)& ouMxx<-X2Eu-#-W%|Jl35DWVmBX0O)Ne+@X4@FocXy|_OL1?J;*{dsWnwe`PiB?mQ!$2iPg@J*=crE|x4Gatd5ey9MHZmggmEHc9 zJ?Imx+Z#D)n5uE|T^JZinAfkQ-g?6xJ&mwR%s!D*j#fS@uB`Y)^Fjn@lr#JzDkk%F@tM zM)!%_OREQA66>>RQ_*atu-#fY)*J47EiQyRi z3ubs7#bokJaJ=s#+3&;;Y&i|V1h6>R|9dc{OBSu+z70Z-`nN+e@j3S4#}2I$H(c+D zd<{1M>l^sJ=x*aHQ6tr5b>n|fLVgAObBeLH;y~G}t-ra33v?(xd;z{%oWH+^+**LTIcNUc zCd`cPN}KCuWpQbP%_#0Xq>=Ozeu=32)5o~Ko7bi?b*7gB{eRF?iUb!Z!&s%X`2JVo z-QT=|niQN~i;mPyncm+;|Bv&t%Lvlqm^}k(0CUR4CFXBS%zZ@$FC=<*?-Qm6ZwLGX zy&fM{orFZL+mHMLm#KW&zQP`hE(tZ=5?kLO-3t%4-y`pw^x>YPAg#m+S=8@-?Glt& zJaaAaJA&MO9rsNTWqa4!w2~llkJ(PYt&74e4gV9_0BzN>nksB-b~CRxfPjB0b+ZSV z3dt9jZP~haaT~$gv*(OlCYL$mD>{7IKy`dtUjS*U|7X+t=B+!a`I%~-znPzhEcF+_ zwC><^&K<+a!0EBqLJkzy3y?3tqm&8H#`Jz^EABTZN=NU~i0K}>1FXpS$Z8pY|E`MaYuSVQ^>Nxt3 zTg@jN_tqMj&#UD2d3T^7iya3AmJ!JIe7L>M%ic14rWnj;uXLa8NqZnu-Le>AL-M~c z4%VZEy4uv0)Wto#=F6CZP8(uCPl`7{?H5mk^kPaL+0y5J8PwPk4F5E#H3^J7aXG%? z=-SZLYs{UjwA?iV@jzd_JNsN~$VqD*B>t-Y7woa{((dZ@NxlIOWT*IAsDylV;&wPk z#0@pg7U4S`T!ZSGyL)g*CO1Bo1cKuFiMWowPME;w7LIAbqhk2FlF$RvsUfS~-D!IN+L^47poa9M07|TTihmmyk;HUjE!TT{zkOe4kJTwqW zj+Xu%e+r6FpKE(tc0r5ReAFALfJfNUApk@bs&ahzC2@JKy+hDKqG`NzkMz7>(^1># zdPF@s)H7hr_i*w1!XQ8BOGH3^qEw>cp(j(7OhdNWQP^p& zcl+Wdt3dO+%khJYel{OtT%~{?T!?2wIk_qX`h1Ks3?6ysB%m9Q}rI z!%l5ItL9BFkN8Ej59nOb$yA1bfCGb?KGznZ)ByXP)t_vGnVv0HJHs&xR4&c$-~7({ z@^`SIBqR5{>Ro3E-o*RBA4pY0?{qKr>u0>BEQX|dSo0JWr5EfLec2mny{wqH7R!l* zsM%5rN`2=>aWIjp_?8OA{G5%g2alc&lu2(zNwVWqH@)^MSyue8yHnj2)_}zWMt>0K zwZgq=9=ioqLs0Zt`h?@wGvW>6n<-O)HEs)o;I`t7Vlcn8ufC8URy4sJ#+-`h_5@=}?-j1JDA!KQJsR^Pyb z$S7W0-m679U4R9+gIFs%ekFb#DUh9K?@M381#l3~RxWATU0D^*YC1l*wr9+_OdOAl zBqv#_w|BI)%5=V87~JO+8s`ix=Q&?cu(vABumyd4+*tC196vnVKK7C2I#k_mk1$ej zMz`!NCc^7~CK9MI5q&UuL;uDxoz#Q`p&-nQi2@S zT>vjgGLl1`0Z_;mEe12O4eIjHs?9)SnbzgLItV>@B97c78&jJ-vqv3!UKt6gA`N_u z1;1h3OpNMOuiUqKh?cpq2VR?uCU`#z85fU#47&O`fsPvXLW4sa=`7VM#_>Gxq@7bb zv$>1X8z>0As&>FOF0)A^rGX=0a7Nf` z(moKwkf?O<)-vndp?~9TfH6`DNa4cA>gz4EMY3L~YfuIjV$h30BcdxV6b;$lKWWc1 z`e(*d_$NL6j3-|(V!K1|an z9!DqSYO&X>CbNrCQ7W`HjEMIJ5kJ+zl74^P^GE?DufiC*wfPY!&K2f~n)PnplM3|4C^!NqL{mztORct0;MXM-pY1GI7{-qQGwxa3Yx7k z+lxyDoVJ{>7Vbv-N_~JeU-RHVfnpuUAKX3l- zoY^gJxJJ0TF7)lS)vnuD&aCfmDnp<SBg zl1sCAiZIkUlFVLJ>Cxu1L$CGR@OYN-d2C0=2@=Rec$ZpSw{{TkebQH#s_a!^vT|6j zl!vW$VwhJ6-zukrD^hY;i^rhXr~Axsv}<@njWf4hI3lF8;aiCf5Cq>B6>-RK#b7H9 zD(FS2V|*i0>M{xOM`KZr2Kr23lZc2Mk9DwN)I(kM4o1WJ?a3$-;e7Z1}(Bb#gPGOHFgngCF5GKhsP*9y zghxv%fVxHPrv!OszhYwi!DeK{mO|ucUflN4)kn~VEs|URv$7!-xFi6uE7A9C z#IIjnquZp1ampCn-_jbU4^SJ2gJtpcK~7Kct2i|DZ|kzF{xI7~ph$jX zC{J>`-N0#sGLa0_FQf`guzuXO_9ku-+W#o@*m1ZOw*V%L<;0ruqyU*L3MSmhTJ?l2n}4+Uybny)#m;81gi<}6te?Y)c8 zZ;qP0u#|>w><6G~qo0o$6CTxReZ`LG;U#g~yruh?I&FsbdCUBb9Xb)9nVISc*nLZg z2(SIKsgFcJtF!a(Jo89bU(zFv!|<*vrm7MjvrN#|r7_v)3_NL;SbLBSnG&M$ONdiw z;l9wHEK$8lOFhBGU-4WrG&e5o<>^I=ux1g+W-yQ~77dR8Z)1IpH5{F_@m1s-XdNGS zvLIt&V8cAhYDBo7YcpcSk3Nf)?7;hVlz}BTD?)s!3U%SuxLkOZ2OXYO=*Z&mi!HeM zA6~-8$OILpv3nnetP-{oL*NhCt4q2``(Cf}njVLN5!et)lp<~&Tv#F$zSI12BW;2Y9v}K!A?ztQTRCF$Qmv|*PDZN?KyvL5^vgZ5Xj#rtjhrG zcDL8?=Eq;81@Tn3HktXz1u|i&>-HvIpQsa~r0cS;zIjNUjvXlfx z>-H|Jpl2=Ql2so~P?h~6mTrw_&k*dQ zq&NH^X1YGSX_3X$@3Cb@oIu1JV@ak#J1l^IYyWcn*hVMDqx>LBbjjjW)V_JtKzE^p zy?SRyA*^}I^ziW7yG#kr$`n%Y}76t!v1DhhT{MX<;SywY9`|{}2jn*R)US zrDZ?g({rSzepBGYg*9?L+dRJy{n;#&k(RV+-n@5Gl|lDYi|Hufc8q#IJ!utm^5NJ% z!li^V?@s?`%EE@d3S`yh)>S@+jj+R_ou7;R0|2pOLht-b`fE z6p~39IwTn)gi9w-iQo|*in?EigA<)i7y}aYnH!4(FTrkf{A6S zfL)?XtGOc3Xb(TBNVSO9+%necu^7pV8i&t7GRVh=e?Qy;=d_bMJ^PDSf}h$3X<*X8 z%#c1zCMKrhIw2LumS|A)K5{N6>lk9#t^Z{974XgE8Fd92=t!YU0j7-s&dzRD#bKyV z1=Ha7H2FEK(KY3sRnsGkdf-xSxVeH3LmWco>t?+KxHN}1xkXGIA+6i(U;X zdBjIz+oA*ghfl%T1a%T-A+_R=o4wVXZ-Jb^jjylx9bXG9DM|S!*r120;sF@L2g}?sF3!0_XU^6D1lju8#+>~*mdlEvNy_~Z#7_Ehb zl>2g6$*nE3Dk?X}Q=S;9L{(Ws4(pt;Yc^DsV-Rrlb4BEZv95{(%i%&1m}PCh)SDMx$##-ln}) zvmnOB!^MR-AMF#N%^s9Q$QbFA%Q z{-}{UCi>sNw@25V!}=2O3KJj=O?v6N{ARxMjjxTTaQbchGRYpwVco8c@Ue3#-PGl<2Ha?)==%XwX|*SS2KHXL1TPqK!cDWOw(OMIHAO6M%*SN zFhCO{mjN(d=Sn!XulXpV&XX~TY(Ih@%%x#0D6DbMMEd67PUa`+$C1r1$O4$Nj!Dt` z2j3MHsv<0J@ws(X=GAmYEK;et*w#MMAHJ*h>Wvt)L>|&Ms6RJ?NnDnQ>1ErI*U1ef z=C_bG{jl4+-Rdm)Q`OW(w1WL7j|BUECwZzg%fMFkS47!$euB-kW*uyC%PM$SOc){a zv0y4B-eW4~u$StigwxG2rxWr`ZWH??NV&z!_`8Dd#C9tJ-4h^#E4H-cNS{RIv&-S# zqJ7x*!_!vKwTNq1#?cX^8`C`8tpvTruagGh!F3b4qsiBD`bM4{;L&_)|6;y{-eyTG%>snygM0I6Lk&y2MdXovBJ;W%Ps4Is*(fc z!%5&c9=1Uw(ge@pJYEOYRo5@SAYaN^Tprcx++W{b+j%pe*qe*;Zj{csBB%k$&HlX2 zrZ=Po<1AFfEsT7}ft0fP9vv4_`{=*;-K{}m1 z|As3{CBwVM(8s1sc%j?{Z!DQyzwmZ@a6|brf+~Z{$0oB#48~jDj29Vj=ji2ppGC74 z8Z2;sOKG55c*4e?2g@r-5$)R0x$%?(dwe=L0%~Wty-{uK{R$ud+UINB;tS(Wqo3%J z1FYAECu(@REL6JKb^|`qi{`D-oxgq?#@DAN>y0$^fo+S>wMCMhcQ~Zag;v)b|Mvpm z--_zcOSRW%7JGjr(q1~u8yn>3qm8t)@R5OS7 zJ~F9Hm)1g=RiHw1upS_JZNPRxz|!^iO8@V@gK0!qqw!w-qIW$fw0-tzKc}VdMJzvGo;goKv=h=k@T2eX1<^&!4^8+*8Hdpo;9Edn*au zz&1t~?u*o|e{PQkv(P#RCNx$^lOt42bVm$uILZ|dd9bq~40qAzc_;Tfr&#h!C6t2$ zV10$`y|eg$W1erAI#5yzvysSDBJ*oWt9V9ds8EYH9ntTlB>I}Vt@PG)oZ@~c4^BEv ziX(!5D~tZ$?a+K3B*5;({+2jZ3g_F~ZC0k6QMT|v>~$}z=n?!;*IEa< z@#Ic?(^ct$g}axhCydLb^((|edM)$us=>AYpqVH0B|-5c@#s(k+_mZx40N-$X0@Q6 ztIr$0+4P(tql=D}c?>EQ{NI+uZ`mv~$7JG%T`};(JitRB%szBYhl^Dn2_W(_$k8tK zsr&&KlMGQvo-z#n2+ZfrXf-QJiOn#o>HUw<{`JHzzMI#^S;r4#*JU6*huR;h4PP5) zZ+%F^!nIUJ&W{eVYqBpE@lnz>IOV5I1@BhLrz<|c_l$5|Sg|&3vvZV*n5@2a9(~Jy zpLTeLmzuFF_HF2`8dIErWaL6kMMHtRDJKG;%9U|nXg!22keNhxCV6EywG(wljTK(y zPMQ@<$zz50*R-B(?IJTA0^ZFCWiCNnsA#}^;c;JxZ5hmO%ZD~%@@tIQJJ=f<=HdAe z+4$8{v|-`eMJ>n=7)_A+tb~a^6q0C*KB8VFRs&6#)lz}kVWk(7I%6K6 zV=}&#oBH8R{GiVG-kWo$n(qDW`p1Zf{DIR1%$?i3CZR*5Rn*=y7lZo! zu52m{(%lEuixa<3&<4k!UN{H)$NV=HqoUS7L*jo?5Qf5cCA=iGF+`Ws1bOr-p=kUp zw%y;+%zIL_KE8hJEhp%s1u5?W&RCkB#!fi7>Ulmyx;-B8@359U#-3++QD!yx$^6wb z;Rgrp3rAiHf$Cj$~gGg0JTwZ^9JbhW#N`a&FKqQ1dSHZ7389DsXsd zkWOvi!K?}4T#+agv5AF5?wwrk$m{n-JrrZr<4U5Gf!kYY*E#LCWaef$UtXN%g2)l> zef>x-%!rq`o1MbWtwhf`29rym4VC`{5y_bD`FD$Y+})(XHHOpCg3su3i7pMiOjrgA zI40ck1_hlNWvzU%?`eM?yNEeCv5_vf-OhgW#TqNX{sT^>9yXrK)25Q z_vNjtl-Kq5#o5i3<+n6UDnW)JoMDd)oDaaYE*Q^5lqrqBfHnUeJfNZ8aJ^;xQ`Oe+ z$3-_xS=x_Dfvsy7JFZ1s@d>$;u2+~$ zDm<(U`@wf?e1Y^&>r^NZ%=u^rCDp$`KDY|UMxxd~W9L1vEtOTyZcdlYlpBfaj}eLF z9BAdTmQFjI_Cr?CK0x2KWC(x%@O&e_j(1>_43XP~vlAOYM|f!WN99d+YGA?vGbs1kpu*`s`+ynB*nWrg)|2;n7}flYy!qVu_)A}& z^TzmacC_@zDzc1@y))2Ncu3n;cz!eX)!V4(9(0k1CH^tZIP*5mQl9@35J@Zs?AZF_ zUO_um6?jKK?Q!o{vIbtb{R)A&&2{xLe^_6zD|C26hvR3N9DJ)}KHcM$*!*@JLTC=z zM6{E>&~3kEPRXbQLVv%tiR#N|+gS!D=WC~z&4bkhq2`b?BE0_r3AFu|*;T_V>?sTM zm`ngzbElHJ+r(Hz?Ht5*EsIljf`xvP_YZKDSj21piQTtRP<^5XYd6q-h=y^B>-63o zcX~xiN(H#e@OuoDhQu1Be65h~NUlXiT8yAGGw=&FT+Wm$qltSHxV)!rq`BGsm zE|neh8HqGEoLUgef7GWvj>x6t(SDMIliJ~meS(9*%g}mqKDAL9R3gD&YSi-8_;-Cb z$Wc?B7GAyoLKrKFf!pl8o3MQ~4-rL}JVKoF0H*{~ikgOU`O1oH2%lOKt#s5`NK?H6 zU{v=_CN~tbpCkrBn^8)5#ecPzAdugCz~{llj^abpxE)G>>^4mV3P9PH%f}BTk0DNV z?tIqKfNPk_dLY5Ri}%QTV*b?qykdZQ)FadUV**8~IfJ$=;=1dfZs)%aKBBvSOZBqN zW?_6Q7jGe~8l)`f6dw_FtqIz}>)kM$i^mwOxYA#7d8xG2c_nXOc6xS}7PV7kcoZ4xs@0#^e}iRdf=;Yhd>FsufPqBXq&CWTd-=y; zPOt)YkT^9I*lNshMC{8x+=wxiANKCD_?<1yo^Oov>@XQMUkD_&C3vA0H(a-aQ$_s7 z(jxCRD79F}U*+!8uq^`Z8|ffL_nc&{%gNdYjA8v!)z=#%pT8$#Kmu+`x)ICCl(TdH z8xGVa;00TNH}65OqJrGG@82gcYtLP3H3{&IFZ%5dtwoUlK;(c+Y}7$BdeC#^1l9vl z_r!M#wE(we6rGvorsY_S)pqB&E;q}ii%PnUk2h~NMd*i;usNM#C`j5eHY~6?{!BHJ zqoE^1sVMbjIIvmW10$Z8EjGIem&k2%goHvcVil7@J;<@6paHSKE7H)B*Rnbmg7E#f zk1Vo-{!x!b%!4el_PZ{C{7xZk0x&Gw1HRzgS}jpUx4U2x$De)xx+)J<7+O<}zTjWx z=}U%oXtBBb4v~QucL<+3kvJKd;8pR@6F_QZ~d+<0>x32Vex51m}1$vGVV zA``(-k;pYIMY85@bt5xM6jwQ&#F~=$LtARHa6jNJpfq4uCWsd(M$&>)}s=~w|Oj3WIb&v(1@u{3R^ic5R zGv_#cwS#SZvfO{Y_FkCec>@rUzu)YfYo&(+e`Iae+TgoDTZig>%(xbgnZZ1oZMCIv zU4e6psg%a-DSp#SFYnVWo4%oZc*m@SW`V)Gc7Khk+R%&tm;w&+n$y!Xn4}Q0f%DvW zu{?a;;%rNdv)k6{1K^j$QnT4lq*_dlbK6yDa!0Yr?>$0SHD~`Nab4n8+1L2tyh)gB zP0y!wb5lVkyUm*{Rm#|uhn5Y|m$%pP6?ie%y!#il(fK>yQ=5WkLY6g--LN6OejhlH zzl$LwHm6G-+q7!)jgN2gi{Hl8uW~g3ry@*-3?Eo@Ejg7%XWyyDH?x(sf6%!gDAC+= zgAR!@bff97B)nLONQq85EE8qytn6_}cRk+3%d6=_^DrN|)SS;u;_tRFoHr7)o!O!a zsQPhMw-uT0&Cg~%ai7I^A^|I*&7E3dq>=VltSi7-kIoCjHtm}#?VzL1)NyRc>IFbU zVobajKsC@_ZS+tl!Psj|3WyuB(>pJPoa+_mLx-I9@{b*gQGy91dunMY$WQ~r*>4gB z?mI^!R&>kW^H!Q&h#pfnejoL*Ju65e%jplJ3ob|PoNbh1Oj99!7w7HGK zgV`o%P#br;A=nQ|Cf6xtT`b!FgVt+!xm840@_}aKpEODG5NFMs<4zAmfQ`LVc-ez0dDR~)!YWJ( zw07!rP|{_HK`RhA%B&w%>r|sUs|WFxuuojHfqyaZVEF{$VPAI?72C7Gvc{&AR517*`)&7 zn|7HRQ)EDf%gbwmX=(YS1b^<3rksiWNNNrCsvcc}$fzUvBfn!`^OrBP9P+lgHD3eG zMO*u!F3=WSAir1NWJB;-Sq56eewxLOW!=*cXTbdf6vjA7cA!*+Ek;Dwz-h5A=X>@ilRw3@TZK3m#_ z|HJNAFF9vlz6+}OWUF@8Rv7z=&bM6jRX8rqx^oXIhFlgb1(OQZMaoF zzha|s4ree`X@|m|4nAGr)EnmU7?im9`E?*s*0)VsH4>;bZ54hInf-j_OEZBG${l6> z4(pw}3e%SIkMOH>0U5UxT8?a?3}B(M56;{>$0&f;2`(|7)+l3tKX~5A7iG8|#%v=X zHJ&Rb=6r1xUL?wtM=Ta*( zDbg$+LqlLymQsDlCb2) zOryr{GU*t()BJ5dphYG>6Xm3rFuO8JvvI}zqr64BZF%5S?M`;}v5U-vYi_!~7twaleove!?u9Tc~uXFMTk6mBbAwQD4wt z%gtyQ+Dy4I{}gr-Iu9B1+sJgG^*97R-x&3AIqnqMx5dsp7!lxT_uUD-nQS8WziR4oBlwJW z>1p78J+AvjP!Tuu;Sd{Fbs%N{$OT+?8k3rrobF<7u}m#D95rb=l5pG0e{ycqWWk$d z1JfD6s`F?Jw9Ei)C-B?AvZQVjc6~q_lCz@g%^PW~v@svYkvoJuWKYMp)>sOwEOD?{ zg{ivc-)wzh^!w7}46ePqwB~FRS|~)Y#TjRtI^<%$eJS}y`h zFiy$ z*jG-`@e(I`nD89Mx*Rw9REth^=RNwsN}mR4l=^PBpt&m(R&f1cebrp!>D44I;Yetw z+}znNNg^edI=ZN-xztHB1-R>eBpTzl4Ep zL|XS6Uv4?he3*pu%TW?BB`TNp5)tz29qho=9nzx0+=ijR1VcGjM1SbyO`Y_LR!67D zMd`(raN8-;RcPFV#XK}S_*;&ik6JVSY+kaAcN?*e-d8dqSl!~bjA8P# zbD3ZvTyP2X;3KqM)y*Yw;DaoPb;|B+Z5eiBZp!Ru#r~{!DJ1`d9J3y46=$5OJl^)} zWP*B}oIkJ0L~vke%9{H~&eUtnb$F&kw$8{Cb9Kke;5>h_nC|m=qNrKkDcqqaCa4al zgF~Q1tyxb~*6xVHr@A`LNow0JPMN4WNKJdGKj?;O=I5&@w>m%eJ>|W}NdAWe`h`&v z0ssAHr}BP6T3!hzJ~u#NT&29;t1~il|HU1~!I<#3U)q1SaF18imq<2s7jMQTp8Czo0R5+P%x~K2a2d{%!=$NweeGO z19}3RwT7&2=eW8s2H6? zd)hI9xRxyz?hKAG;v->m_en&J{U8d($qCc+E&k=LaCWrS_rMMpX!bmF(GpYq($L41 zfDV36Ba!4LmyG(Yab&xC{%+)s>xuB7%VNojz*bR*A#+__J#`nBt#~5DD9KcR{TM^B zpou_wD#KYdnB6WXr{OE=Mr6nU`UQ|^ajh7+4exL-wEA3Gkc8#|<^g}@Cwi~HDUJ*N ziXR?JBp&+UhHr}58f+576yB-#b*Hz@@IrKTf^p2XU9pQ( zUH~^$`Z+}J0p$vk@yf4IY~NmDgPBb{NHk&q_U2X$Fahrq7jk)Ta|Z_zION$RgmO`6 zy<7q=o7&N@!-Y0=d_v|pp%!UYC$73xY*A2AfO-eA(o&WL&EVdt$3d3%d+O4h#H@|_ z_9xG&#Z`U|PdxareEH4QiUfa(iETbHqq8dQQjHSq0x==$zE)2}@@ebA0yOUwMe{!@ z2oJW5qoa;02=fBtFX&Fpxm$WCC?shxLNb$-k@+}rq!&<=x0-0wi|a-_+CQHlO%Yu) zEycfEII~f(cbBDe0;KqU=T9%BK#e+iU_-7ZC;pLr09lgNS@NJUVTjxDftc-CAqP-P zV8A;2@?`oV>O$6v@+OYE?e5M&Uai z1A2ZnW*(V@lje~>3qS)F%z1k24A=n7b;ghz6B9eEMwGcL;VXXt)f=9SUCj_)frW!X zdqLSEZP2LBgXOWGF-&ZUFg~;$>ZJoMxeGa2kO z^idrdeKDtH`*~1Kt}z(e%3=h`e$zaVLU%P)5|L-)z<1SKfJtFlku9A4Ib~4%nu>FL;}bv zaz0x+1I=g>__8lwwZc;E@0)zC2CH9_M!gxKW^o~NUfg_D0IryQSd%pUSdKOo#&?Hv z&1rM}ZQk={Dok{WDonLz32jXkkB|h(^w>4|k}bQE!J(M(>`kc%k484i%S;S!Ief^t zIq}MRG(%OJYJGfOn|Vf_=#Q0i-jwjxuIN_P`n^u_U7IJiIsB@dvdbhJhq$>b>U4le zIY=lIa5I(!l2tkb^8&=Fq|+sgUIPvRNl8lYgtOktig9v%Gn6wcT4+L0 z;Qxx>z)*bP+*Z(!G+Lrd&g!ea&v-JpjtMTE8|&`!BiJn_f8to_cr*QoZmM zKiU^=_IDSOpw)^^v8wco;80dhGGR2x>dni|mbbc(EC34pcN(SxF6$6eblRWfqH-V+ z(?n(hW22<}8x1w%B=(lvMr1L%fS#u+s6;YoOQd!jN7$YgtJD-I8y*s|Djz9kkpval%m4)+-k1o>4 zmw9=YBy-u~t(!Oz^y+%l2b!;|McQfpO(FoU|MMJ_&#Iz!&U-H}vnk0deV`Bu0T+`mLdEMzlH)NFgte9SCCtPB)) zz+Ei&>2rU5t`|OINue^0X#qSvT}zHR(BBqbHYr-iRdz_Czb|Y?xNDVviKJ~EFCfiSS+nnQ(Yu;>DnB0k% z98PCsJKX$AThVvFl474Qw=q({0aVA0;jef=`Pv|xH|f~&V3d9Aa@P9t_eyuZ8=U5q zxhVbSZD{puf9i_Ca<}KF5-#G6QU1ndOCvFxGsecw&4vLG^CtJRAF91#6UiU_Hqd2O z4`fAzysxy`rFCuvrPH2+9+0g2xdnNTB9q1)CAe%3-05MOt~bkIEXgB18RPa3h8yj8 zn4ax$nZcMg#vt`HKoVOD5x16aaYPrq1 zd!p+FUiJ@Ia=x77sn9`$q}?PGuYvB74{&ep`_6n15N{^JI^Z7-2g<&s4#Qq|X!%C! zI|@Im!HH`O-$1=895q)T8l*PMwIYh};{JI>v>{7jizimPZ!Ch!tmXqPbt@=PGZ)?V znx2im3#=JNgmbdhnvJ($8m# z-m%~vqdA;ZcUafg-=?jb8tkYNTQAP^)EIF^oT0fORmXjB{x#^{-@%!G6z0;w4^k5C z(B|?{GCf|mcW7!d4h2G_IxGAwGMM&bqid&{q%=1B$Eoj~UU7YoyJ#aLZMJOPwf^4z zs4i>+gwP4pa;>KX6jJqVP^{3p*9 yO0-bG_G}sCsT45GowhF7{od3V#xwUQr8HN z>U4?3c|)DM9~dgk49G?1#h>a|e2pAiifRkLQ3@IOo6qfbN;((AC8jzt5V^q8sQv*` z7}K>D#6WT|RjqgNn7o@7h34%uW6``A{^otqX+OzF;%S$18xvgK6D!I6ju6HVb#%5K%SHi_BUb`oZb_G;3 zi?Yy~^mUtvs)N1z_BzDm3@Hmpb!Zr>8TT&7+kLNWBoU}RdIf@P={-m+BaZY94A$C< z%JojTHG5-%YRzV~=`0ag$tD8>(duml9)g=~dSg!E_c?ovLTeug!m$NMtkmY>YBh^v zpy5qHCCHpQuH@mnnO3T@js-bD12h#b;Dj6EhUzpLd0bx4i$#*kfO-dK;(SZ(^|eTu zn72dBRw?kO^MltAF&3~KS_vJsz`gELG811w0_J@`$}LAs4jNTp6KA>g@jM(A5~^dE35huPzxNULz6bld3;#dBfdL)<@srtJvr#` z?wg5P2hOujC%5f|#vu`EF}VV9A~VYgu4puUOkcu|j@~_PEA{2nG$X&udS7}rJ)PEL zgY&Hdr=`G!AGTF^RG|y|V#Uw>dX%DGE#?id08khzbW|UV7h=zR6n)VGxb7rauxH60 zzv#?d(S@n6B{Sm$V`h6r6duF6`Ifsz+p$*Y_z^19jI~wEGB-ivDk*GSa1|fyjl{is zv1TYkt}3b%bD(!cY793=YV@h0E8J*UM4l~N{=ygB=e3OSx<=URg375P&Wpe{buVus zr4JlgbZmDSnyCvv8GGpj7zTRczMtgyS4GS72))98aq4Y^@%l=0#8`_lv6ck*(^Hf0 z=Z)`Rtbr+7tkIG#&N~|kSfW=eY^F68-{{RPp*8reE-d?F<9E1g=Mvbgv4MVoi^eVd zDCOoq8djID^S1{Ee()z2cF?0Y9}xIwf4ss#w4{3B$g9?0jdUKCH~<5vsr!R zTO3eSNFIZV!*Sz+ElzTyH+A(P-5RE_NK)yt{U%h9_5{A75ya+IQLUFOqGYm_yq>ih zFhYUp;^)G`p);UUR!2;<0onR`1<}2cK$76}-F3H=FUxsxEIfMw1IEd%vO(r~qJHvAIjq8%-J#U*B`8iH{kAX7DjbEgg?Py9&^xdf#eg!<=?qMBCtcT}|yAuwL zfMUkZSP=rtk5*CNfeS|jG09ggs|?0Mxy==p_M4Iy%*ZjZYR1$`1z_NsgafFz_;r>O z*|~zyVxjOEUH5ejw1CutO)xRQv+mE3M&`mGy9}`Q(JKCPMkfCed0rpQuGT9Wn8ZEJ z)Ou?DZS0ENS9X!=!4<~3>;jA33wLKpaugMDkk~%KgcO-Q6V(L#K3i%)3XQ=bZC@zszsG?0es} z*1A?)Yr|;whMec?v^e#{;_EZ@hsOk}lftFanYdU;6 zyQq7NVuUR%87E~DO`z1(ZLZF{MLUK?1trXxGIDPkv#Qe_oqQKN4FcRS>=mz0%eTk+IT|v@sRj!m#aMw`w40p-$e^{RTRs{u0#7pS-wD z=>oEMn+R^w7LGGn3u5dix>)JA4?MFr%esrg>;^(bpj8(+b)GxX7oo;pt{Q4AO;MzJ zHCDS0=|LJJJ8R%Qkr3@_g&pokT#)5ZU&4~gsSw&h=o57d-1wYKl9_bH_lYkH@-@aK zN8{a3K3jtt)cgFyv4J((3z+!tkle^BriPOgROO&2k4cBxgbE72piSS=Tn^)6-(~+H zgnz_7{odvw&08}59}`wrItT*ub@SK#|~vR*Z*eFUf~8R3G9+lpV>yV&OE=dYK^PF_baqP z?EO&2)XNi=5mbO5CUR?&eFjL6xRLXXaOvL`SHJ0?ADK!>0==C;%BA**L`}q0IF{Pi z9q2;*rH|BNHyL{%^(Dtc#&PY^L{$eRVNueRC=6WXgl4i;WpnfOZ$R))$zY{;8v{qy zyDkK@fC}-iL$UO0oyte=Lh-E?O7!&J*?w`z-&?sHFc4 zfjKbAbkgu8)}e!BgdQTzX7u~dbW}Re>c1Q_L`lcC=2-;lOz+6pb;5>)(tDML$_*5a z%RikgBNC<#*sbK1Y6UBQvKWNNeE#Xe+{dy|mAPCtMVhsF4)|NXVW2$3{}eh+vqL|Y z6_a%1_~ECvT4-)L_w2rQ&C72O>Txbr;hqoB5H%Xt&DP0a*Gg}cV)-@u$@mSlJ-aOu zd*y;ji7fwlvWg2et((KUloPr!GG!Vx&$SWgqkPT9l1_UMld*#Q@(P*A@|sAzAcdUw zLS^r@!py#q5(0sh$&91tPB=05C3mhGA4++kz2D@PY4LrInw@f;FYYfeJQx<8PL|e zvhYjY3X!P}sP@LpiTWo*X96T^HJy_h60nfoS|~0JO~3c8*yXuNrqXJC2N3omQJ=Wo}BHfLz?WG5vk(t_#4!hNW08E)V}K~6}+XVPHO?T4St~E(S5AO*FHwg zl5YNPPbEm2%3{lQ)w%i-u>Cx%UvLh-RV5@lX6e|ypflmARFy}jeT6PKYne^L=u2<>2Wg3* z%m`3;r29m@5H-l?{*JzGCTH+Uvd zJ_*mhMdCnwKCAqPSQ4ZdR*dRLvbLPiIkhL%^;U5!J%wk)qv1QI3pHwLC7Ln09#{E_ zqlFb~ELY`bW!{Yyyu|($l_?B+U;HwhjV(&Hx{Jse`_@SPm!2w>{gvw;d;P9f@4oh` zS4_=pCk@fEm$=X7WxqSt{_faMiZ~ag>i+Pn!~BfGsGF-MVnsh_SOXD`bA+_DaZ~@n z&dHc%#gSnl*XLHK;#GZs?Q5Z@N)hJ?S!WSQe9MEu->mj3Rg092Hr+~;Q4k2BZsWZC zTDSQ=LTz665%YDpFQU<@>Bw=zfqtf{cK3yoUCM<`EbshF7>z)IAM6~9=D$kTW;V97D!$xm&`;BC z&yelN22$(UZ`O%rvfATpDjFOZ)~<+aEp6SbP<_cnQ)O@~WTeVcW@hEq9Q6KkDydMATV($wd3PB?{+JHcJi4T_sYLQXSVL-rZ=!Fk=u9_rx%e-MzV|T`1~qc-QDj{VT<4?;Ld(pXuIXQzpOr4*C4_#I8%h?0Zi_Z)yZjitU^jDT1_s zUzcK9K8!P*o~b{K&oBqqA!bTb_ag*&aHcOI@Mfa`D zV~lmeX?mqs^?$o4eHA`2SNistMa7f>##JZM^n zqBSwr3rK#nZQg;iE0D_MlI3!3oj&9_^`%F6V4R;iP4XGb>A_pOI~9W0s<~I_e;+FZ zZCn<3rAAaGczvGCQhNv9sdHqMieQfK#W0q! zAuOD4BiBcstuv)q?WWLt5yLZD+c-?;$u=Io{q2vCTKxNzmPAW^>Cof2kPz4FJr~lh zhrKgya48s6@jkBW$;*p6?8iLe{ZrOg*l;aDfz4lEeyf{4sk#CODw>)80;mFZe+k#o zo@=rH=pwX|O7yM4lnh<0Z@I9g4JClE{;El>%$8VyE}BnL{tOM z{iyHF3xb+;UIts{A6ZolyJTmOiYD=!?wU8tUS_v)v5~B6$!Ol$TXJ@nPL4-BND3ua<7kF_V6HzQnY$)?0g+5Fmks8Q zdzhHUXgDirN5X+d#mBDj)g*2mEFb>2S$7B6c0E{$&(k8SR>oTfAGgrgDKpg9q(nD)<-^_JL0jq zB%u+4Vy1-X!nDW$3OHT4GW5Fc=FL2bFDe%3Mqy#+(>Wj4ppet8V0+3zo zgo@P?LDVeug4N2m@q(0tT7bTY<@Tj{&HboM5GLFwb?Uvk7S742aTq!s6;Y2Oy46lY zZ6?Yn%@n+?@Aid0bQCMf1N90YrlR|gtXhP*SO+y1f`Z;=Lz1zden2rm@W#XJ+|hqBjffCFXBVO&m(c!BYW};o8+8!r}f^u9AT; zs@b`4x?WS0#^&_MqKB(aK@_HfWSHu*=QIc4ho;$nmTOmiy z2o$u^L0Yy!%4_k}g{Cn>U$*<6PuyYQ7N4}@OlASzv<_h;SGu1{kY%6)KVC7iblvx} zH;?)o#?0tR_iLokLiLh*#$A_x%jtotm!1gZU8Mod^5oZNtwclwAASuZMZRh~)zPWS z)@pF<#+#G*aNBbiQa9Hy%7ngolc6!0%$~maT&yGbfB?qB^o_v8joIbX?1xKQ3KCe* z{a%$9OF8AJ-j0cjSx-nlo_lJ_|GIk%Rp>3&+4X7x=$2u_Wbr-b*rYbGWtfjy_fdBH z8v_=ZGSdg1yxm>$$+h$_@EqtRs#OvxwEvlX$l2-yo%jc_Ka83JIY_Ugi`L6|Rr@tN zNEs_(mc6$AdYZMcer6@!vExaK!-lDvynsREi@EAHq>`q`2Jhm$mjZaL0aw>aE9=4I zD#66ZN3@+9VdZPmr_}|fuLX^4D+C{Z90m20holOL&nv!+`P(Y>w;(i<#`6umbVWU! zDP0r{h3MC8S|3I2YIyFZv(`uUL)W8hE!Mc0mlu)W4FpNv#8Bw;(LNq&n37{1*@1^| z+4x~Gf0*mgoSzU(!igNxJorKFT!028gvx!*Wj213UgFRbVXyCfe97`E;icxs=d}|x zyH^V2oDc2VN8hUKk1K8@vTQ{bm$!zE+~>uc<(kXLYPa$EvP%osg)doR>tFfG1h!{9 z$E(c|v;vuH6wX!N=uI91%mE3gkMaUjzv{i&$xRJrEDeo|+s|iO-<)eYA*xbDj{MU% z65t!s_}Pa5-L?>+b>{=t+3I?EujAA45ei*NW^b=k4Ohsti3L>Y9WWc#%2uyk$t$Ta zz|`}5r|#-YE5{>D*E}h=EYAFgbY5=}GK6k#I)&h`MVe8!?JTvNcxa2!{?_`wf5EQT zY_0!+ffL=Ao>`85xO;UXgbh-zk9@=_NkJh`0_UbnGp4Hz^RtOZVr*DX+tdKyE)kXb zpcV}Lp~R_(Ky5-1U=}a{vpjcsFmMDYlE^ib&P?(^46jhV6yGYcz)D)hlL~167K#b8 z$wY+aD_JQ-h=HB0UdfH87uG^Xb<040C>N5-@5Ha}t!#m7Hk_ak?3*aG-_AGJOP6aC zI`l7h=^gYT=N^tXBQLh)v%4IL+7pe;dnipvfqsMgdg2qcx;*A@H2+FQYqj-@UKDE2 z{PTWS9!%GvuXov8`CVIon^6H3tKUN7a>|?1LKB{L9J%?uFa<+k8Ht)IdIR3%Z)(?E zntKVpt}NoHRUy$fEZb!W<9voyt@R&TN!v% z&9%ysXBE@_^Zp;!wFFj2qOdsJki4H^)UFtAqs^A%UhPE}u50p`icCJaUO-2X>q#n- z8og@cb#>wzVi;c`TrvKGLGK!QiTcYrcM<)tb?FQ7X>Yc4mSR+ge~W9Ev2QO&O8;a* zN+>PmPOQA}mqfqM$2y0MF}aWhZE zF+&;9=!ToUsaWvTN>=Huu_sQE1O$Ul0?ub;7*cF-2i46XcP_;WR8${dZ&?(c)9)_a zl#r!64B892K7S1SlU_WNYV$wz1yq-ye?zALi2Vjd6O>)fp$iQi71Py8N%-(PlqY;X z+5?oRwqH2DFKvynHtWR|5lW_$-)_!glekJ4DXPiIgrD1sjPU(Wcw2!QP<6v_O*Cqb z$F8o|5hOdK;oMBB40$Ha_OMHAQ4-y;hPI|hgC(AttHX`oF%8MpEM)p)xC{}BIoI@1 zz@O&51;X3!{}Ol=+Tr@@tOy8Q(^SJ&zb>@)tO}H^*e=+4#*C5Gbq?sbz>KnSUfG6y zOmHjX>ntN8-~2bk$j(^}bZ~_5SEwlcD*b^?i4e|1(c&NH3rX>@^*ZYn8t=z_ou;M6 z`zVmN6?(ifsDe}fv%9ra&bGg3UJ`<@nsHPsu!V}mI~;I~PK{7}Yz{Kunt!e#|EI(J zryu_RxAfd&5z-t&)2mtzBA5N*6i;Dz)z5(mg_>bD<3q^r=6-zZYy+XdliSIdk zdUP8Ow>h=19pY3H0xN=$Qo?ydJv?i5f|ve=Qz0$4M%8Tgi3K()%DL@rsi_Y=;GNyZ zM4r7v+h=@YY_7p?fU_aI zgEQ`^b=-c~Gm}LLlS^rLR+Xk^(7VAutr(#aw)uvgsUL4oF;D(f2o_9LA*RD(J~a0) zW%au(RQ;)L*oRk*nk=>F!{t5R6`hWz0+Sy1WrlNw=c1Xf;Ks}rSXb`n*HAM`dei$ zV4mrgbi-54DDog>Y+^-&-uK7aU zQzZ(}+{H2(;QGdvCW%h$;zfaK<>D@`Q-X|Epf_M&?I`;-mUi3J#pok z=RP8^{4_hF8^eZj{Y1+ZgbGpFXh>^ptMne3_Dq5oQ}*!ym-NMEwKb~E=Sg|I)aTTH z)k5Y2M8#$3M2;1rXn9;7lsArD^iThSUF9PVo*wTV_3wOhm5#l-)7vUp%mlXkg%FQl zB5r^BgnH}HipGiGLvBS?mnbVXPso9&fcj7IqB{(K=$Lb>SOhjf^ellDjDD6tH1wD^r9JAuPX2*-|bDuxb>v< zf2q#2RQ~egZkM+Wc(!81n@1kET!Uvz3xrx9_o@U4!1eO| zt#%aK#)`_BW|?cd$66pE!b7(hZm8wEJq%ng)os_2ty?D;4!F4LHH-^+R{zM02NSeh z7Dw9(_peYs$4cG<+7!4tv2q(*v(#D$kNwCF;4|c7N|YE{rJ_})f#m}z94nmf39Qo6 ztJbfnpW61!Ns+Gr23jT=a6l+sYTzVh2xW4fwFwD?h>*16u_V4PYU8n&tH!fg8mER3 zmd^1JvPp>V6?)>WOiQ5Q!Gx#dUuVLD{xK+IFw4~MT*S4o`^-zrHuH{H&$Mm%y7H{m zCvuER7Zf&GEg5l~>?_cwQL*t<*dkyvP-r;TpDAozu2|i&O~o;r#*zc!~)gV>THlqm>Iz z3ua!`k0*K#UIQ#Zd7^V6!(Wh?E|Ph-Xbnqf$ntF6$X(i>UOgVpT@8AuDWCXa_nN?6 zl#Fbx+|aA^3>oSB1;#u`v`65iQ6iAdx%=@~ifrOP>ENUn`Y*{TVBW$!Zt~i#d6tJ5 z(WCan`xN4%X9a^CR%3tCD;ogKt)4JWg&z7_ZY(kG%RZK8<(b-#FCRKL#Hjx98t5;- z@qX=G31wc3O?+X1>4O}Gj-%Oj{(rW19aSxY8)VvST;%YV_KK;T!8 zw}5iPhr~Hq>Wz4wnK^=`H2vdV3Rtun%KnnOcpY(qj@?G}c$OVIivyJAJ=4ZqkI*C159 zWpMb*|N4TI%3^c?*MDKty&5X=v*2HJcp$Tif$uIxMhdg3xmebraiQId&rs9RfLdSd zsmxUup_?F2AoYhJED#Qm6w|N!*eS=bw*w%XBDr6U>#}*Rr7}gFu7OaejiaqAtf*fd(}tL zc8ZhV(ehix=mV_j93VRjb*+H(ew;cJP>a_OYA24^w3~E&J{YeOHc;FH%rPHXZg)F` zNab}>KmWs}6^IqSxRia$LPfnub^FXJe%g6P+)+vT3^>w@c>CwyNbc7a>ZQf?mpHU& zHi(Wvr@kfag*{g-t!E)X=~aL{zu8~^*FPQ7#RUBc?sr!_$^FL65fk>Q8VkwOiu*hX z8*x$~sUsC{=$T>2ckW~|=S1#D>r+RZ&2%O%T>^+j1Zp(Xp}=Tj*a@;DVx|Gs$%l#U z--|F!JC`|k;*w7ac*$T?q%z(6@}9%beq)V02TJO5;`|p{*>Sf1ALm^j&8@V-^s=@TX^Xnp-(jjP=xrdmewN?E6}nVoR(MTdA+P za1LS93Zr6loG2uD&;S=^{|U9&cboc$IZpQOU4Y*3Qv#}vV#&4eJAU|hfJHtNeR>j< z0m|X3Cb{+OKKAp)uu0aP_lslbEzsQ59#HWQogaRc)-_sj*0m6 z_pe&Cn?neONL||yroex2XoCuG!w+=Sg_8{>st_~O{o&ymY$k_eb0UBcaZ#EYHMl2k z%NPRxs3Y^N&4^py;Ls%i$5M-l&*vyzjpxcEPWZVteAi)oWn%FXywHrz{kF()9h~oB zq7S7g8Y{wy`3j@KPx{le_NQseIm*5QmQKp(Os;P%7M!bVqy4uxag-}51uT6JbT3R; zSS&`exD*42!oOR>z+vEF0MCAUrD581;G3SzV=8y^=|s!3mt4?Y(oW9tl;0CasP30!Xju$aK(a zYT&@T(d~^)-g;ry_u%W&`|yWL2IsV*8;xjVQcem#C>W$qiu(6lB<7ai>$OW)%L5M` z^Xasl6*fS`4>e#eMj0X;T+EOP;fY)k%i*|t@NsUFbfAf{ia^$bk*sVb3dv(9I~Lzd zWJfrR0%_=gP-iDCA;6A(vll9ZMX$`UZM!0E0AZc&*go#8_S+O@4{s}Bgrs)3*E6M* zrBsieB=!FGKAXcMarszCR{|S`)ziYFbQlXm1F)pS(y5ega@IKAWYgnx3WrF8p% z7sd}?~={CCKNi5Ufpwho3t;< zgG>L)(cDFl$kCFt#dj!}yB8SRBjfbZp3Pjal}0hFpt9tB<2IZ^_!xNOX&sx0@Ie9J71^^%DQHhYD+3!z53ByLb1) zKj#y3rtU3%?jHD)l?g?ZTu|cck|}t?+@pM%-LY;hn}B6|%E)05&rJ`bYGFPHw_G2; z+?(NC`e9C03iz_@tWh+L?4+|INd-KhK z{v3?RC`e=q=)eRmFPYMXe8sdrF15?uWV@AlzIZvmVNnq#>|O-rOpK&x0zIl zO>cwdDutg^osF;(2RK676yZ6uDz%*J?%4&INpKt5J1=>zn61#Ys%2D8}0Z6 zF^&U1yB<4^t%@_P3xbN=GT^oncTifhaBo~bT|eEDg(~WlKNvo8>ak9rwC)aJn7x9o_iYnz z+(~X7%ZoSRI3MU%S%rq(dL7^!5c&ea%G{tMKDo6zTKTkiAS8#1 z@1Z|u-YVj;UN%+kqu$zPnEU-?A*Pm>TBp*r>voFmf9IBd`izjn;&+TO<-mQ5iGnY0O)%Pgr z6wdm;&C)dC1xN0%YBn_YE zekgv98sE^1Y@PDDM`ZHe2)xnrJ)~DG6}oy4>>H4m&_2YvKbj=nH|RF!|?rRZ0@!1&V%(HeWJBh%|pjqL4 z+IubV`^cB+g~iJBlGYU|yOa>hHM%a%qH6@GV!e^$UU&L;+>B4{oniL%$s08VZ$QD# zv?IFDtkoLRPygPGP~803l7&W4CEjaz))HZA3YSIuZdZ!Nf}|HYzD6!Z#q*J+c4NkV ze`fSUV04Pdl@diHw|4kEn1jQh=b|XLFi~~HbB`5$?_N=g_dMB1XdNNH805R7yPbnH zbEM*6fW)3BT4 zp)y&SR+827ZOi@?7;46L)xZK2&%x`T6O|rBc+uuKgEZYN{5c@B7FWAK*L!u!|67nd zB59G@YS%Lgj~9LNMX3e|F@@DgS|w`7PAd2z7Kc6wl99d8>QXtPr zArW!g#k{V$0a@HFR$44osKjwo?Lt(;AN}ypPo|lF5hC6wl5rlzff7eGt>#x4$gM+0 zk=WT-&rVReElEZDzfVBNF64B>%rJ|2vI|2|&LR-d{VO{m;DqJHuB@3Rfn- zg4EI|fI0p%@X%Y~8;FyV!jst zcOHMYB&4CEt`szr#`Pp7;I^Ap!{P1T%S$CxHQr)*tco`-ATKLm;I;TfXx_c?QLX62 z`)+nE+Emx0>3*Oj=JnorqbeN6fU@73VY>cd#Qz>nFUm;cBs%h4&sModZ-V#>Bqx*- z?{#PAnZuOms#nGbYwHh%b?Xm^d^Vn@o*3%vA0@@J;ilvSOq1^)aZHnDQ2F%gEdpDw zx8=*p8_YP=fm3=12(kfP$%p^DfC3)>ijFL}m-67lZx*K&Ptj!`s|a%kYd$k3?@UJb zHh}$MU*u6qp1}ayKfnO&K{dLTrGlSrr{!rL5H$VgVgO@Gk)5upJ z_j&?X9~GIQ$+r>}L+Z-(U}tm>CiHk)^2gbvrBF5{zImR|qm#eJ=Re!_L(FfamqFO- z|396!j@EW{Gy1nl!FOR$jIw#-w@ZUkRk+5QmmOGFjXb1kSZ+@F@UB8XZCdZk^k?8# z1+_-G57b>9C#yiBTH<-n$K}k@i#EE6n5Am`PtlrBOT4w_%PA%?(VNtye5z@;mD!%` z2X%aVL3}ok)%Q;=@FISfrjfeXR?!ZmcTdXqnz$g=+!iDGB%;Mbt^J2l$LC>2PIcX; z-|o2&;CsCl;t^_m_tx!xva0sDj3A*}$FnbRAp{D>8My3~$2M>A zzWncHhkTA5_u}1zsF}6_BUoJweTfurMZ9Qhi zWujq5*w&89B;|KRQ7t#*AZni?D}2^~PK>!Iw8ewxt)OGSa#bfcIdZoYDinK`LtSxO zxxf~E9cO9J_Bwa(h9D!u0I>{{?Z!jaPEls*ia>r$91g=w0vf@|d;l~i_Q!kpk9N6w zOV(NOK0gqhQ1$muc;cJvhcbnX@pq?TaF1h84zBNhvHOm>u$sRo#{Cj2NKTb$PK_8*M_=J~RcAg=6JTb%as zd2=!lvrq|W&QP2!xBn5rKO@kQ>EG*(I29>wvYG0ky{;-3@S5@n2*nMA2N0>cpCBGe z-lp*XeaN2{=_Sc550X@+^TOFmnPDjH{_QSax#EGEZ4GR(ghB2YlM~yU&>{vz$jo(s z7vHJ;M;}AP_dCi+>k!Hq&)vv@mxj8>xN#aioi4VVIy#KW7vskG@B>afFG-t>uer$T zCdDI*c)Du7w|g*DuqE#xSiP1OxNW#6ubuxt>8*hAez>U|PMK{rPq@&rdti5hjlYYJ zId~2|)+@1TcuZ(K9`k{01tmNZllXa^LZKntEp7|GK!n(TbzJ5(+}Cg$>7q-B{w<^l zlh;f*3i^!EGOyahsOpIhnxCM$k^nk`=JY4hJ9@! zz)?Vog(ztr=eLCa*SuvO!ffV6C_P;t8wCb}Z$aN%0z*-bbw3~QxZV0#Y|X7tr|}=*;nKr*c|8kRaul9^&F_I#aymfR{@3#TkX>s)m#J;$ciSbWO7d@%t12fl z$|E2Znq{CKjY-xsB{0`OFd1ZTq~MST|5i+LF(w3_$1n#}dsLgir~l9RlHvJJ33(b$ z+6dh)dexeHUq9nfRk#5rr7FgCzd3ktVD*vp& z*zoXef-Dmb0n83wC#(2F$1B# zML9N`Mqj@89LX#7blh+@zCT-hBE?hn$gCV#&2KhIiAErWjOyG9sl7 z*n!&nM&}{+Y@dyC>6u~rS@__gyAcOLtg_VK)rPuSDpjy`d(Qiost zj=4c|XJ+XffS{DImebe?G9wEpW5={xs9q&X=WSE1f@_Fb zG{>GE+)+FZX%-TPGjwxv2wbT07Z!honGm~#$CaS$z6W3;3Wl~!qY!^dp5G~!KLX|t zANzDSyKf$MjaLFBOP?>>0JWBvM+%II$6^QXZ*lw??6|3;FO#r(2 z>qg=?ikSzw^c-qp$AW0#Rr&HpkO`I)nY>S48mCy=^&qwfKdcm#<0EGFOqY{u@KT&oKxxk37-;!&kI_z^JJ z;<-H`Gs|nrilnb9Y9MeOuk7*5Yo6;4t7gs~mp#M*uWp|iYdjF*o`rVHUHdrr&f>S@ zmT@JTQi0|Z&EX~=T9}l#+h}q%lN_KA98(~ zFN0kgU!&}JgHls&Y$+5Q8}Ed4N5P=D=EbG4n%iG-0<0w45;5NrwcZ}8pup@1gxsWS zHfT+=?8AI!+UNW9ySoUEGDjYjL^aOC^P-tEknHfM;650$K+gzyfNLSIFqoc<^!?OT zKgFt>G~)8)7qPZc>q|*ifziv2Xbu%rreu$IAEEs3v5`DgV@8^tWO1pu?RH0RK*BjK zuB*2I5Kc{DE)gOsLjv$19T`C+*Am~yr*UTRoR2>pja`Pjl*fBy2msv|&Rh=+U_zi`jy~FLF<=W{j8WhS{Q`Gv%6mUik`;%! z^_oq@m(wJTFTuwqr=4jqy3yiVmQ(tMX7#zYX-6&L8cDpVzY1a^$P`ljT1-=p@p5vM z$CKh6{S@v+9Un9vu265by{znOcg%pO9wM|mLM&PK;~&F^l!FIMfW>=es68ln=A6(G z0q~{mR>w&zq2CXx#|pmMUS|POJ~8O>-|%0xGOxZqq=QA^uU*UVQW!<(cL4-46A60o zHET?ZX@=u3#L?l@os6eGg1QV5f&tNF8EMsQK1p!%AdD7ssV-NKHf(W@$G<^gCFNhCsnDY4ZCP&pX&hX3Fp@`;H~F%nfiSW5nqbCU@PnDd{9oqTu;((!o{ zcQ41tNhPQPe_b-tksfy?&o7^j>dh-&jeuD^Cu8C3zq#a%b!~T-r@G!v6;Bs!TgC{| zN~L|S>CQHNl>Ut#B+G;Lc6QLa9+J1s25SWo+MYdEw9kD`BiA>>@l021d!-(D^_0{^d z*=oHTv3xyo1aeu}c0$Qxa37EL$yaXAl;R@$@>V{=*KRp$?Ih~6q10F7`5K!Ugl5fs z+LO6CALDfIa!x{7_j5s??UvB!=d!_7w2Kv+Yk98m?ZqkLVUK?-fSI=cS2U*cip9Io z@otOFHfZ*ZxN^;Fi%DZr@1d=nxyF{n=U>@Yc8d265rs^4jGwpgp> zA24_wpJIU}lcn#8>i{+Hlc*!CwOS0jGr%X=kuAWwu>Kf#8nlGC@zs0@fpXnRb_y z!$)1AC*sBh{;56*D%WeB1Mq+~5MRVF`;R0TDdoXw-U!)cYF0b6KCwy%gaDA@F^7q7 z3x$0z+o8WM&En#~c7ZjA6@UQ(dsSZ}38F8P@&1Jg!b~uzgxXsMwGVTF!80}}@%wsf z@B&$tNjc@c)0vqI@#caa8zDKP1{(B|e6Q}_v#Lt65gi9R_Yk9yfO# zlJKrxkGXK%NY~G!A1Uor@=m(h^xHCOq0E3!eC)Y~=|$z4L; zeZQQ!76@29$8@tEC6sI?RbTV`4!1tC)s~U$oSQz}fqtG&t0yiyCDJi1d2j2;wsi8; z+cojDv?-&nIMjOE%ofASz3sigeh{B&Y>_}r-L3nap6HNcwI`qWCm;mDnzX8!1`vJb z0^UyymLXAXmQ^p(gXUth3qvcwVa-LL)SHHuHg{1gGhcnWQH{=Z*yrho&&JJ5*BeW@ z?_@2JXQ}t0#+DdLD^1yKB=u1iy39lKBhz@7_qLJ!7v};Omiks;QSk^X07Ny(4xgA4 z96#|=Fzc|H)|c)2Y_D4N{dAXisSjohHjWwgIZtN*jj%0Gp)*R~E76uP{6U<5jv^N; zB(5k^lQ}=0M0Iz`%Rq^YY(|$vbu@OoyC^ngF*3?sonw2jn+_Z1ihRwaYRgkLdQYuT zawWs(`Zq_XM9|(!u@bnv>B%X#I3Er-_eCwCI{QP@3B}}yT$N5D#1oK08A#yDO}uw9 z_DIMoQHqPY_hIs;FiFl?9gqoJ7+8B}ve;l#{g2lUfF-0@oqu=E=B#4Tx>M;1CANr1 z8|-~AhBk~EFQvR^HI>iAvur|^cT7*}L6x!~V{5%=f&0gaLW&h3(;}_4?!GxWqpHwC z9dzJ|8KmX<>tX#_@%AgTd6Qq;qT!kg{3YT?5elSmL(FSvv@yq4-QxARpjRacg54uq zXuL+VERej*5aT3q9zK>HUxW={SiIFs!bK!v;+l0-Et zw)WEFy%xOM(9w9!Cq32A#bGZe9Uw*+Grjs~{Et2+$#EFP}5%#oAO)iAPW z02b5ZWpeWoZ6*~%d=})8B~TKh_GGCN`0JFyQ+S73M%>#jzu_IWo$$Fw>oD~Pd}erF zv%Ij@S40?(7PLW6Vz2rrRG*RfIOFiipY>n=-TyoCCxHfTh|Acx(WvIEdy`5Zm|*-7 zfLVt+fEYSm^5yM}y!zZ?G;M}Lt ze|?L~zj~nW3-?;GgBV()OiK`Mhwo{xF8Jt8zqfvoj$FOMp~Udm4D+Y(fnvtJf*b09{H$VpLat zA(h=M^f7$k2Bl>B+%5cHpGQe0W~FO1l+b?Z;toX{Kc&70t>`mb^rX2 zmX8+xfQaX=E_<&-yAO|EEOusZ7X%Lpgv9W^SbDNLK5;;PCskMFRlpJRxg*CCi*dAk z^b!a|CHImzzMW#fIjCg$+*@|k{9)FmYzQ|;##0}YaEZ(Uv)7Im21)dja8r3yMx&a0 z6AEFuMeXC{OokNiP2BEk0mNC95|2bn*=oJ|eo-d>8cLeAeX7+|n!qMB{%*BBzke0} zg_$nk%1Ii3O#=~cN#A?CL?Iu-Q(;|`&dr}DE8=E>cy50d(u@45b-dS*Idgt<+dzdx zVU%*1G)3S8B-HmIWpv;};%DB*B+*9xlH@gi{*s6!Cvzoi0(bS+Yx)bT{?$CD!i$RF zndx9rclD<$)VLEz##)jM%1$k60NRnYr=Wmu!=eT8jHAIB#Ti=pCOSSQ6HDB+W9REv zZ5cw*(W+R)YEf+eX3EmXw4BD=C`qruTb=248G9dyf6-kKyd{B#d#xxO5Rl*np4tbZapuh6 zn+sgpyQmupHkivg*SdVv=OZ^;wLTvg`G>6+*TW?4<8i_pg}X%g_s26uTf>AG3^XTT z&!efiHf9jAGvmN2{zXZ{({kE=%I;$VlWFoIxWiv+pG@a0@#b>ci2iKxsM{f(gyC=| zffH_yR`1KzYJNtt<3&(h0m3eD@sN=R5?IcC-cEt<&ftHJH{+C`qMbvDZPM4z2OD>cxDwk*wg3<7vtC z6a;kA=}n@-hq(JIAi9Y8NF3FatDF6BQ$D?$%Mav%-jcl_nDmxgefExS`*Pf9YHz9Q zW7EnA32VaPl_Eg0&~*M^(V37bfz7#d-Y}?TkBC(wfZo#wH@COR1cg^v!2}d?L|Ne^ zZFMJl{fLt>9Tewzs;2pB!kdrqE?2wW>aA_v*n1*wH(@0N=Y98W0Smwx2chRX*u>zN zU!$_V-Ay-z%`x$A+{F^bfxYPDxq3b9?K1>;dVN?wZiw7Z;#EndNwf;LV5=Y6=w4%y z@!eJ!?9VIb!v(Gk2D@Ishqw3+odc2f0>MY%vtj{Z=RWi&gXoVtiiCbKtW+4Z-p(nR zIkC*UaW^a8H>+AMlF2#gtWuZY$8+C62bp)Dt~0z2)u}Y1u|{G`T#`O}tK+eC_rI*% z9^$E%Rf>2FcX%j}5fI;Z0f6>pBObgS%-}QYH?1bYx5~q7956E&KbR0GvDK;XbihEg zVe1p>-iH}4e}?{Twt$CSxd!rEp|}(K#!oV3{ViNcPs^%ICd5GLhHxe>91@euG8Ks` z8@TDbB6%UWw3eQ1)Ge3rOFr(Sa;e*WZl-%p3N!`jgg7ZD|E&VgUgC#96lXRPssd^P z4_0zBiyw+GiH<%FKRvc?}d`qFL{U$>cjjjSl43;nD9axgNe&3;g!mrHGVwWwc-Q{(0upUvTrnWVKn>F!rTQqN;G*DT zchfVNZ0btw1Hk@@*YbcRe!3ympA9_B?- zzzb!0{BlUVAW1H!hH*|ml>cVP)765DP%=nmtv8kaX^jD5koTQ`Yxo~*y~@QjJDLOG zQjUw9Q+;4MfjvL`OwD66Q>ErOv=nnbdJ#{=49=J4Ia-QZioA}nPV;|OO3&6dZiLp_ zBM-B&AEJ_NM4FBeo&o=mrdel7a38Rn6LZ&SvyWDUm7VCARuSyT7c5D(9vPB&@lf8y?5-ryq51qNFlfm=g@jroT>?kh2a_GNa!g5cfa&2fzb<&GBll;Yyv8%t%^ zq?ES8Wj_(YPQH4Ecxgz2!U>25e}0af8v-Adr}~5+MU%~C#{WOM-ZHKU@BJUv(Ws<= zsD#ojE$L7J>F$&m9g@;z0s_*E?(Sx!igb6QNSCyLzS+hNL?t(Ehk#n8^qy^^ zAI)mD>aqF!82f+@>=wYvgj7c-ss-1{d^TM9Xl`Cq+~Si*AM5RM|Eoy3<)`a;Xy0y) zr#gPtUx7028h$LqK-OS&F1HZ~x_)jz9a{;tsymq?>&s{6essx)9MZr2arY%-{=*XE zPKmp3j*We<)3ZSBVkS@0v|cF6g<>6owgJ6karkTCfs{o-o%!vki#iKFbu*>>BZ%MY zbrN?arY;!y+0KmLLSGCVcceF6K?Nzl)@a{QJSrAi9o@FHZ$P+N?>R|2J2QoL-Wl_X zfipFfSy}}%MM{fc@x$5)!tJJv-uDrT@6x|Uc9kzdra&zh!Q!tA?1||^Y-kcKn)I1= zHCJ(V+m*EbKQ0{=HH|{ixqa9xWjE7`c3`utHZsXDV>Tt+f|gIVUjAGr@t&}OzF^(8 zUr-VdTU^AJ)RZR_tjGW^)$6B~=;tO4=>Zt2^yW9t;Nr121P+~C(V<`+(j?X?YQOEN zgB|ff#&}UP4zl#KPwa%^Imnh*(7nWsALrTseZB=m!=d>&Wk#GhI%fDP`=W$eMqb10 zgo#P5@R)cBXL=`EgWHFFLk1sZQu=#&)zwCB@V=V)=bRFHK{W!5I~>uAcBg45&wVaI zxWR#o7i{j3TsZw-_vL!k_~L-fz%PLI`3@Hb%5TAL!!KV~e z`c9A~RcPk^qMA^9_)%-{t_j~F9SJQrAFhQ#f~)Qh|HNUJCunUYc>vcTQR$rX|Zlkd!BeS!5HMw+m=BNjwjF6BpY)EVz^L95yy>4tedP}t`TG&AI^#Ev?yu#zQp+60>bGkY*QTjaQeNyFmn48=rTG=2<7~&uv>-n?|-U|fD zBx6qa-VpP6wGjavG@>aQfEyNqgb*avp2jv{`UVhW)$U|8WIa4u;1PlH9cC{|uPS^7 zjx@Od?x7UoLw~tRS|fA;B7JGgu+$_AX$UpQxL)eo9uOk?2iU_Bvx@D!9!l*$AlCL~ zZJpuSRzFT9R{bJ=`dxxtqr@ORT@7zj;l&^yi2AWT62nnFz}BGk4JCUL|QUXL}kEIRdVOxl`K5#nI&_`+sAY`{X`u*#77ZwH? zb<5-UfUsVqIIO3(JL)BXxD8jQ+@?$ItRV~ud_1tL0pEnzU!H$Bbyb*GOQE7}^P5CP zbupKZQ|XUM5&a+`2k(kIUb0@jEtu5%h!6D3ZmI)=i!$TRO7@%izZ5S^)F)LIZsZq0@af7eOKRrh z?;#RY>+>}-NyhU7xQ+)iOR)RnC~+XIh%A+RXr{{I7-IiK$&xgg-%3 zlA*DURtWNf2B1MlgPvZsa^*YA^S#Z>02BOYyQDZsDxK3oBK_t2q?bhDLYb>JeS@ML zx4TRSG9=2|T)CPan%P+m$ISW1<5F-^Z{7t>pzF69!$qE|E_>dU5u1lctXkQsU`N(p zm(p3RW$L^P+89O1PTl^n{g=P#>Bed`v{pLW#b z7~rcyARnF-cPm_w^`kdkMqEr(LP9u;*>IHIX`>Cf45Mtpkai(LgoF42@8aABBftcL z-Y@pOBq`BqKGI+5)5U&J2Gq&-NqS2+Kh3{wU@aE~W!H9A#1&w;tn}cX_)|&WzHBDy zS0b)pPz(z}J3!sBI##ltxk+{)^BWR|(H1806KefXg)RU)d^;5xEn z`T{eg`iAcZ#)~q^FJgj@))Uy*%|#YWMr1#3Wk&d$Jn{jz02gT0W`23s_+Mj&sjLj; zAOuQCOS=0l8||!Yv4_j9WVef6Ca0@xt&JqiH?%~^(#|imUvSJylB~ijdl>22e1&PG zRCJ5ych_gkbS?RA2Va8hQ9&0BP9~+mPt^Z?4^#TfW;d(LOBILO|3-PmvUS{I^iJyw!>ubvW;aW#=d7ro}9b$#v8F?a0QXv{cc*B`!7(3$#+W@!T4;${exfaH1YqJE_4vH8@MbkQsw}5gC;C#ZH zOltqOfCuXum)=sjY8z&Nvvm{Hh9xCpg|xZ0p@;9q(6;G@Uz&b5Q!|B*OVes{_CbPMK8WN?gyC7hG<1 zg~FiqOo!GiZ0Bh`<|N{+5Zue`!76;Z*Z_K?StfAx4pM4UKG@eMG^tX-LtUi<+-i~1 zkN9NwXPnRK!WikAq?8tb2UJQNX{sG;Gs2J*l75wXo2qcCQzrX0mtLgVkLV2%GsTXu z5s#z&irn;S#~KN^%fa$U&Z*NT=dflf2paZvfxC`Ane+F+UX*OW!?-*v=KS+t(45A} zT;HujMK!gxy*Q?wTO=Vz3(gkiUAzsshBm(~4!tTR&}Z_aZA5wbey^IvUe~5|5$~Th z05*q8e!WMxoJv&YJT~}%diR$0)m&y!K}-@xd!CfvbKI^1h6YYB-3_+YI5TKvg?DOx zDq=`Uhvj=_XEHf^#KPTjpSNrb4vdc|4-o0Qq8Kq5F~=Y)NDO}+&TR$37$_b-msN& z^Ptp2cRAb0m3cOJc0zQ1|adaRBErSyjg_AYp;2|KRZ_binq2iP>s7SY*1 z$Z_jwmSh=jk$cabSzED7U-#4UKs6F9=kpcSeEp<0ZZCdosD*V1KC;g1-oAIm^Z8HX z<~ApCo$jN^6whqqK}`>()t+;-Hehb69Hk*))K7l{P#TT%81S#tbj^-9dR1P|mHVZ# z3tm(lx0nimu5W@GU3OFhw>DC(; zJwVbG3iDZ>aY-vbjF04QSF&nXL-51FZ%O>@A#{Otbbbb)({oAAdUp9lBP^R374CMZ zwyNKFPiJI7=$WG~%U}%uUF75e0agzxT1$CZ)+VuHj%TT!d5{!6cM~(`xgyiYLw67? zdYF&^#k5S#*g2M{r~(H`WRc>0^%P*F+2IE?*{X@B%OzBX?Z+h=%ZDpFuoyl7cWeM- zBNV)9%PuP&=fesv@h$GvV`^Rs?A;g3zwW4~aO+{F$y4^0uG+n7!!=oe*@aGFKM6DScqXFvcHyoEo6`>6*~}Hc z&ehwCJXO(}xR3QFeXUp$4q6;--~Ng(D}ezSm=5R;6J8+F%RcyascteOYUi5lf;+3zW29`!P;y!H zQHIo+QiIg!{M68RY&`A2@9%^T79uc~X?xXPsq`sLS97#Vjq-pV9Z;ewnUtyAew%3% zkCjNGMZ~I|y65zmNlIzw-Sea3Ly|QDG6YLV-k~DvS4c`{DFMC5uP>JAI(=8;x(s>a zw(LXJ?Qgr6eG`ItodIu{nm@gD^_$e2a7QyeKuweQcc>&}SI;M^9-?u;Sl|zVH~)jG z1lI#lx?O-Xa(sz%`zX^SC!w&m2R7c#1GcidER?fD8dCtp-QZ}n*j|*H$&1%}g_uz2K{<^A-9rzlE2Z2GO|nJpmAu^JRVb z34!Bnx>b`|H!{@ZeT{Fh>wpLa@on}KE9M})u~RDwXW*rW>&$%W%%J~Gzgdud ztZ1-QDQ@n$IkUUo%3#qKF1nxcUjG7q?MRzss4lHQ5VG*@<14c7XnKc!d(j0S!xMWi zzZw;(Ih>0wa$ULFvRBx!|2gaB3|}#or2x_#@}R+{kK($;b_a~^9c#JIAU^KY4z&jt zO8z*_xpr$>JEG&*+Zv}%qhTxdeah>l`FG_EkMofMv?(@nfa_>0&tCKU$2pf_o*Ne! zS5V-wTgl9|E)@19878{@qZo{8c4=)n&gY(89d=o%h>og~z_^V;Ly*I=knM+a)5RMV z8xya}l$lC?4wA5mEA#wRaNdROQ+Q>&+$Vu>5=7-rNo|mx-u*9MA#Bg8g{@sjip5tB zJ#;!ce@WB1NS*&_)dCEgHi~#a87c2~Qm|Pkq?dm(zb0Wl@VQ;>4JWV;V`!d^UtZXN zUjxZoNPdKSFL%7V-mF>IS%>S3Q0d)~YS~PUf*IKi=XExjR4mp7{}v=v^Fn)(pM+4C z?29oZc2$3{PrOi|ao>~NHGlFBP;wFyI+S;;uWj5t&`&U};yadLT`;q&7Hw7Yb{Dm@ z^FpRBj_mUg)J%v|RKFM!U~N*`x2p#(^n3=o{Ejy1Shu{X+2N#ULBHhFS}NxaI=Jz| ze@=mU9pLqf@yf9&2zP$+*h^0J-1T6(d36DkE807@I*U!nAtdwmlLY7eJ5@{iMgN6s zHULBMvn$8;#e>$k!5#jJXyp1JvT9oD)rraz52w&+9a^<)&gW>ay`3ASwO6u>X#E#c z4J?od2otUtP>kKH#c&BP}+At_U#lkY`X0uM${`JGO?IaXfWq*t8N`MWK66S z#PI6EOa0a+C<@Z6{E!)up|n~^NqmzZ{M<2{8~`gc02hLW9s^QH2eAMPbmu)oWD3ignEL z^F>1=afB*N%USZd#$@#MHS7)I{mlw6D+^X?oeJH+=H_jl=wuRl^HJ4SQkf(ffgJrUZRsE6P0f-`_T;E{S!dnM z%d=A!bNgU?Gu5Xkvq3k6?##@MUBDqOYzh#j_ormvU&pL8yU*+w!;s(*AWLr;dSI-K z5%l-%6;{~|91KsfpAG%BEzspQpa7A9CA({YSCm2<>hB9QwLNUT#X(DWAnO$xM==iRUl{q@`0 zgI=h*_tc`~&GR&17w-3hoB%S0N@A~&Gc8=9;jpRu3|Kb=ZXBskE7p$$O)I7dd8&20 z(`CwicD9N)9L*sNkJ*a^=9^ zmy^VCXbRTX94M%Rh9(P&3l|4Kev-FBTGe_Kv+f3qkpdn>+s?>@F)>m|SdvPwTG{p; zT)|~sv8!0}B&o#A(IrocLd>{@{L563n*{)IW`+)pPfkug3*oEG6!Z;1_XBE-3XzfV z0LVWMl1#6~_X~GTBL!OexyGCH@&8c=C}k;Fpi@|7VkI|v-G+UG=}0SYJbv~Q=3}MV z3XN2!W#&i6%^ySV9KT~I%c!}{TlHjM>ERt@b@PurgBcbmxC?&WGCyG(kIW!1gK5c) zz8pms5Z8rzXA(Hwi;P|*0xa2NXX>O`!cV0;cV+)LY)m5%Z$-;wX-;?;1F}6Zjm=TH z)(t#~0`Z%%jJ|dYMj#M{wr{H?PdqvMen*3USy{?ha1G0cD&re&=4pJ6AJhte?ct5z zyGfY4Ua}d?(}C7$g`OrHIcsp)`}V&sjw=UzJ8Rly#*COO6*&PwyYdpsjb_kZ8WX^g zC6(;hH|DP;bN0@)T7_+3(;zweHnN}g+em^>T+!hBZ>DYaT&B8G>QFsEm@BNnf||pS zgyO?fwRHRvg9K5~1VpnCi&b^!vz_(d#DgZiLVC7+*r5|N>haws)4un0a>`eIR)k+Nvb(m~KMf?M=mtC3;C>Sa^IGLiCd@!#( zXMD0^6@W^@<_uY=SNTdDx+dvK!fP=izBtRJ8h(Y4Gw`9Zad+>(w4j$3v8Kqnq z9Rf!7e&$7i5j~?PgsYT|Sy41a_t|GSM*m}2leUm#{GLEZAZy!41=iC-9w87a?ZRWv zo?*~z#M;hHW6UPDa5o$CpW}N&jn0RBhMzm#BT4_c_df4#1PG3vU@kE~x&@?`L=^ZG z4XG|snj3@7@)sGxg#xq!f546+yT^0r*HtYvZ^dy=d>>_>j0|B}TLmH_?LlKU-uYGN z-2ML}zH|VVXrN8`nFsR%E=T5p2VC^l6=azATlvZ}Y=#%W`vvs0jY~WM3H>{kdH)TI zC^bNQRU%HU+e}!@ql7OfE$RSU=ng2H z{o(uhrzHX;{q2G>{H^*Ro`T6-7)rKz^pQr%iBnhdp86L_Iel|m46?r9Q)l&4A{lMcxzaZcF z{-%^j0YZ$K%#=NZ-$9qbck~L-LL4atsIz3_0Pil{h_gbVR16I2Jbae|&H2+37Cu5Y z2Xxu~J_#QC`6_qn+;Vv`C3h=-#nfLQM**0(w3XZ07^|{+8sGGN>CF3Uh6_muOnh*C zU&SF6PlIT_hQivtU$+mDO{$x^#!Aec6cG0Ton()rD`g*Z|O4GvzfP~wUDAXwqw6AGf16Uz^4U6`=DbqD=wYEzDN!nch+E%DR|Qu>cb z$N#+)I7-SLAWSz8nub5@)WnLe8WS=XO=uuV^Z|AE_~igs&C;4Ex+B&j7}-BJ_lDwv zs|!8Jsr?|eW@0|;wj%RO)>^5N45U=IT}X+0WWpr88fS5l@&8Rb2_S`G_fx)qSlzJi zqqAqnc7DN6DSri$u&p}bN;~)jq21_v`tiFjA9M1c`u`!t%3$~GyF;GqSCXSBU+muLrD#7!bSnZ@MRgi1U>R|%xE+wi2?$hspqD~OXCa@``pqga;x5R)&B0zy%-E3=Zk(1)9falJpAr>p_zyKE& zuNQKjZVFu~iFdRuc3jApvO3F0gRnRUrJ9xbHJw*Y{zpS-B{0I)0Xb~(ZaeLMmNbH1 zbvm%iKvY^q8h2_+Njw^iwQ*jjO^KQ?)igD~HFGvX`40ylA%lwEb(`g(r^i~m9w5Ex z%}-{;9DD)&a1igK{xbzGw)y~vsZ(_I|8}5NfQLHo7}@B)$~M3}O?Eyrs=XZmjq__* z&)c=Ea&nI)Zv!2SV`FbwE@{=~`cnwFumqvNdeS~qK6qJJBksRndxJ_;?ZC;_3DsOd zKO)=VA&q~#_Tc;|^OHJlWkL%jy8q_7d!Vnw{_ubc6bqys9Wr`jI{%0oprUEd9FE(n(xsu(0D~*Ho;n(mgqxop!RjsCAZ7#c_ zZ<6s>jq>jdr+@U%t~~wVm(kaJ$&UFSPV`;^grsr^i>=+btYaVpBsy8^kqTZG>E)i) zZr$CJ$dr<6dlZ=Gc`CUAC#LfCe+@#C0r)v2PdC_BxfA|$1gx6@JH zaRvOZtaTyH5Tq+O6#ZVe(WD)|jm0YVtzP~bkmYk_blzm(M-~doe@H>&|M7)B*hrWO zok-=$}Mf+P24w7kN>r)j>x>5k#{s0 zmqTTF$<(v?7p1p}OwIi#W>Y-FkZkDC2Vb>;9RC@#Iv1g9$dbcdUVh@8acLDzEv>d+ z#)8tuCqw-;$oc&Oa9CG3Ch=2)`(gqYTOy9+;)z@ePsft<5+j^x_%1l`+p{IHBBQcd z9@(>QsxH?o)UfIw(;)Xp^P_v5WUdS>q`5rEGn6V<6c%M>^3Pd&vVAwIqp#@$?1WRn z_Z9z}=j91{20;TJ_sCPD7FQpT>y30|uFhXRTJ$W9|8K#<5Nj!kSg9Os9KG_IlH}sq zQriKM21v8+#8LS0<9&~4kPH^2a5pPB#9+>ueo}8{#DmpImi|4%HYH9C17~vprAE6@s+ud0oWJVHu)&S1t9V(QK5ZttnP&)2qPyNY8|G4*)zW8U zPk&d=d^E3v*IyvcEXhBUGj3j8GC;N|pzIKtVjNKR6!&MOucBWZ@uIH3L+bz3v2Wt^ zMnwWLri4;mDcc;+=#--pumzR-^8~T?J{T_*`=Wb%N|`R8yB2q&mj=5M8J(;`KohG5 z_~j1A#JAIwC;J?;VP~5!MH&tdimP{%qaBuy;=H*$an zdU8eZR0I80&pPYQaCEHFqHfE{)NSZ9-J>7L3-mrG3Q zCAvbck>Mfb=8Mky?$j;CW5St-%;VG3SE(oDa9L-wJ=n`LH#bw2>FQ#v zH<6orXgd#T4T?T27P<3?cy zQTXgEM-xZrPF3`e&m=bGO0^kJ6Ej#{(&~5nln~CX_+B9uA6=QXVorDu65pr zd;Dug^X2<$x`}lHt7JB6Z*?Y=f4&w)li4 z;`uRmxyk&)`Azqhf+#e;f&+XL+Wat*Vowy6hTxy#{YOE29fKQQKl=(+IZqxEO&ojA z?rk*8r1I;!bz4j6EO9S$4RNew={vf2+$^`~ThB&h)n~fXRCziZtq&i>ry8KVW-3&p zNH3FcFBRr7N+NaX-pKF4*KR04#C#i1Z|ImBzqdEuC43RvBo(893ocDF@}D|!tN$sa z-BhBq>L4b&9ezpAau7N+$)mdNVl6U0tCJYkaD6RS_8z@>)R3T2+<4u$=`=eyIz{M4 z5&3G}!gSke9DY&V^hs!CdaXZSszv7UhbeSb$LbQf#JL9(WR?-zWfG%MPLWx=PvL3a zIec~6i2es22Ndb!Y7esa1wT1-1-BH;;Miq62h`a>mRZBHtgs` zJU21?KE5gd#khK~el=^^^0RK3aMo9wx?!s0g8A80FsLR%Ix(Z1lUj#}D_DDo)h)qi z{VLgr6P}bDyGN(@qGKmVz_GSs(nDj|7^?=ZpL(b$+}bM5mm7=E5bNvOkb7-!PpWZX zz^9pVRYA|KBbhGiQ$?4=NTT9xw%^7r=`xx*Pt~aRfxpxr^5HEJgd{~@V_sRPcT*M4 z8`txsZ4-w}9SHvnq->?VTry16qbxuXpI9dMHC`sZ>*b7Y;ke#&@xYrIxF=hvK*An zUFw3(fq?jrgLYWb>T z3NQKm77yA{6^Ic5v;8qnxJSiV&w-3WvVEJfCEpLzPRBm$OP6RTBp!<>-_j3Q7rO7z zkd=S+3UQ4_RKHw$Od5rF<%1`sx_Q);69*`IS2%7c`Q^eMT;xQ(RmjQ8aEuNEf&54t zw^(u6A+l$<8*5N){`L*HbQrwIrg6%qq3e_*$|ir0q5P-HLyWmkS^7fg+iWo^1_B=_2!ozJEG(oe1m@8uD;u!p5Mo-vnF1anbk z9QQ8w2l6UC@FoxpHzDP;mF#F`AU%*%!|*zALP)^1#XUlO)Ia#X%$VM0wk*Lnju$M2 z!N1z%DRzCNMMHc-$LT0znz#tvnow_YRH^&R=FU&-WmG&GIIVd?!7H=P`NY|?ZXqU3 z0XubQ>p(buBm_1g2alZzX7*)u4D?bS!P zAEE{PoPB|^nymsP4EadWc|_!mZ(22LkjP{R`~#jv-_->ix!>7J{Yr;k5Co2eC!rbUwU?zN7Km3?GG9b2=nOG7~jQ zZ^B5s_;W@i>MrQ_6LO76o7p&Xh&G8n^)!Z-e`7rqV?g+fe*%3$^K(Md*8po=r?&i zs5M=}k#+IOIx%gO;fgVZ*;|7Il+}57)y0%=Rvfeoo#e{$4@7vp>Som$2I*!fDjv;u zQ?c3Hhy^N-$0P*hc4zUhYRFD%J(x=NzS$N&khf0`vo~SALsFy`yld3RHUoBON|#GuOq2TG=PzE zU0XhLNQawIV)&^V=hsj@BdjZdHo9TEcx0+&R|gF;_EqHI&ArvJTTA35HJpEHqCU6L zLOB*4yn33Yf$|l{K>ObbIP|6u z+#|I0Kx@U$S#!$ny8b??%l?q7Vmr5pXM?nOUP(NGftJF=zz#5~rv_&;;8@r~X~rZ@Dyb6k&d$`s`NG~=#< zK{zuc(%h}eu${hz3~x%WJN=1e!gE$ZJF=$f>CD&mM+v)h>MIy^ibH(n^Q|A52rnW$ zdF;zBJOMTuh5;+wr_?m5;6XtngkVFu>}f*0lS)UE_=It3Z`lmy(Be0?`Q6T_(W4qZ z<{P83L0w#JvtF=J%EaBXCD~9JdcUq{XbvL(dI#BRJWF@=X5BD-IK-&(-X&%}e(?0QWfpFsfD^!HoRukr89D{-Lh)tX`y1Jzw-lJS}|l$<3VY9<;PiI1~i<5<4BcEiv&t$#6j7h!q? z4L4zJ`(Wwyr7y`?y7!Z26qSfDAFXA{*gLy=XC9z3HL{R6MbC~=!ca~j#6AwBsz&(P z)SA{2!sw6IYx{i0VvWomFfU9W+LP9d>`N_`DIeUb)D95*27g}}`$|?`k~i}rkv@Ni z$Xhp?)+|VS&6~CV@%;(=uKlw^Pg#}jbmmG*EaIB3^eHQthGwggKtovugG1={9DnQo zU_jrPdhd0mpk)}vs|w7h@~-o8{CG9;GHT%iNO#ar7OGZ1Mq1?0*&Oa`!#}oC%NQL= zHx#w(Ax0Qxe5Dq@jq!6i2R|jf$YK8E5=cZEQrZ-(#ewUc-kE%qvscxF>Z}eg*cDp7 z!g#uWl(b`8=-kkK>LT61rQwNdj@(35>xiI)EOB)~*h8+ms&sCXOly5e5@ShZfVvW^ zS~wC3TvmeC-(%P4=-*TiiBO7-dWb}^{k^{=eXo3z=(8}|`x~!~2Fx5amEzN*#?2h@ z^asc+VlTypKNWUB%zxso&+_ftiwpWRwGJD*HXIYw&{a`Xy&GUCE~)Of z@R(|H;Szb;f2ZEpW>*%(v@um)9a72s4PV_TJz2t@}?vOwk^!~9b|Kl&uata$w5o-9rf z42G?VVLg;AeF`R#YqD#|yjF3V5RkA#hYVK7NSr#FqB~UBpB*q(oUV&h>rP#+DHPz0 z;t&{S#bNH(d9aF~74FT|XG6}za9`)^6z^d7%gS#K<_A}>mIpmE$ptB+vR#&*=72=Y zDS<6`&W=HA|NWWQf~%2o4+!Oj>aVtyGw=WBjaFz9o(~+GHL_%G6C9a^^j)^byKLS& zBeSCA{HKvni{vl|;d&4-Gw<34JxcltA^6Ut6k=teYZSw~F3`DgC+0(FbCh+OEFx7m zQN5u$2J(tr)>DUQMRZ|If>%p-cLOHDVBZ@B!;N^`lSWORq@x`{{c0FXZ% zvNfVcOEdx;r$pgL-6#9Wu@OwiWr0A!M#xuzOB!=LG(w;7D`EarnOBU$EKyWLR?E4)d zf2*6e`oWIbOw`X|-Bc54Zr+M=iU*c^YK5X5G=`Fz^vUFZIM zv~ChL!1CgnZm3k)O5}Uj#?%6dBkhPr+CNV3BF)~T1zG&`2K@+Vd61$RU3cb`fF(}| zj^sHn7JD9ppSl-`*?qFslqUWL7cPYm$Y5?kcRCn|e(r48uKNEJI^-!JTmc2Yo+X9` zLHYx$Mt+1@KJf4NGPq~p*1848OF6J>3;q0QEfnBVCO3N-Dk=wNzG|&LlHAm$IT7Xw zj?NJj`s&)BwS%b04pJjAvD}|TdS2H~IzNl|3)2FpN2z6t?>)3YluE5NU{w&m;SJLD z5}Z+LpRwsMS*2p1kBHTR)%VX$QpfjpdTRm=gJL^@OhGgynN(J-K!Q7VGDI zGWfn74JHiovP9|LP51U`g6J^(Fyr0jL5ZM1^?^(`C1IwZZNd&jjVoBHD%QdH6^+ zl^({(y`05Mg7K=4RMfpkQ1nU?nk zObl+ljBP6+FAsTf+U@Lmd)i_4If8X-Q`D*YQWeS4A};-F zh*X_6SMaW_i+SCIo~tx3UE+<~i!cFNY9@z?Nd3CQ7P7;d7Vi6d4u#3f-hB(U7dh#L z91?+6zxo|hSs;de_#x0_tSX6xSfiV-u2mTvYYjFrmrjvcX`)+Ubx1mhhebkRD}%1z zVt6p==GRYKC@c?QdSK$D~J6XbFhH5oMEigg7*Bvfu2G^0|%l4pyF$qjKwGsapx%wsuDA>~E z6{KAxy1t~27)|Kn@~U9k_1$$d&DbxWB7I@aX*!TLzD8&7jCX_A+id?C%8RWow%8%F zMS|;GVYisMR&&^SJsxEUIDc4w-QUJ9UFlV^XDzRc{vNADvXtx-@=2(_th$dsX78eb zXfz)dKyAoC;V7~7=rjZJ5Y_(cW6~LX#YPl0O)O)qF>V>lrCixgrMO|}K7A)B8L}DL z8Z%{QO*!1u7X3n>Etahb)(U!z`L?=g5n35`owLj(_A8By72blx?m|#0hRmseO(hSx zqlYG9S@mj7^y9Aj@jq_ma>C>zOK`&f9o?hEL-xWZUzP({%_WFPi0w>rQ-ma^DWSFl z=iG-9T*6IjFsH;DO^+6nl-l1FBjd76Z#5IJ>_l%@hbm-Dy`XPo5ic^j_(DVIh$b91 zDRYcdYSPXnSvHmZ;R?azRNXh-q17T%r+cK;8KYS~ImVx^6V23$N$3S;i_BuHg$?}M zp}mG2FuY?UKmyvjckyP9YRI%{D&%5CB}Qga#zf%CytuK~YaL>h=`d(!co0#r0>21r zT7!a@PD?&&)B|#~Sg?Jed8V4UcTEuMk?Ret>BPLX={1iVWjqx85g-QU-Q)GD92N29fqK>NxWotU@XvTDUGlZTj>{l-B z7n>Hya6nFk1R{2;mw2+B{6e>QN8nQf-3QEGlon2g<9_y&y@GdLs4c6zX-~rVC0@-T zbNd>8yFz11(!=+G9g%*)6-`g8ar--kPKPaw?ul&_DBSs5Ub0NU)7ROHh>mgq{S>>5 zMJ<`=4-0+&#<-DafGq`evR}r!!Iw5`37v+B z{0>s!xea*PWWCc{50<76k?bv_BN=j!5Fa!`lG0*~D2cEnRSi}gN85-5)MQ5pH(Sjr zA=BGjK?Jm)qRK>W|WTgqI(&S2!Ouvc`zHMbZwz zsA1Jh3(17*<%r8th!P8tBU?*y;c8Adg~U>j z!2|2No3vSF$7PP*?M^s56%2F`{J6Sm$g!M6Q1{*0QpJ1|#kU_+;WJrOj#R^s2a3PA zfP()Aa+%85{aeW>AEsJ5o=pEK)&1^ZMj#C%pG7yARKO<+@x+iJT+|OC*6B&B#~K(E z0EuRWmzaIr?pc)8Cj%_naqo<7*ENq%F*zeoIoIgB?f46wAAm+grA(GzJwB*4%gB6C zlM+h1&8g|IqN`}4wdon3Dj_v%zi9Yi6aSNxP{rs7fj4(oMLLd$>}hByY)MgXna-}K z{k5NdPHkjkVPN+ZsO{IY+2m5*%Vxq0begi6(MJr5!&B85?`y8@*$ZA&FV4-oJR9gO ztH2(w+#Z-r2|O=<&v>(hqkZx7%idu8uzjvPgRulbrU7kJESfm z!*ff9GWa4>()!(b;UcEB>ZR8VB!#GBk(s)Y6GMmg)ycjDKM@Cq(%r#q!;b+u^SuWX z@2mYTo7ZrVPkGQ3(TgZtl)HbMT%UFT&(F)_H33h-U}CqncK7Fv!M2vyKJE{FNfkm2 z;X;h&GA7oVyYrgoH(1T0=G70EkzJLtnbvM*(fB44+5saC-?-3uy5kJlx6TfP`lJjR zCi~T?^06nHnO$WfkJ?JY`yvKpI7-UeY_A@L3>HqoM*FX7JEEPHpz)tfO(5h_W3@74 z(}e4A&rBEhR+BQMn;ZpX8s}>T&YX37xwPX9WHVB<5k?%JXS8$oncXO-%+x`dvn(k96r$7M0poVwFtA8TCi9ko}f8 zc9?|XE-KH^-|(ESx1hI~10iS}e|Cn8oipA2+EcOj*+Prt=mjht%ct| zzP#}!z%7IOwqF`WW$vM;`rtvRq15fT_{JLQ zXwDjU5g)tRW-2ps7aVO=oO>(}I(NUjv{|dE)p-`K41`H^RZfE^vTVn74d+_vy{@(# z*x!-8RX27QI=e%Shb!6M|F&=Jwf~9U*|baQAk`9y89Q-*gZdzo=SaBYz{eQ7$ByX4 zO|>yEa|8Tv1r^=*+ih-A96jE9veSDumKU?Femhth*Ka%e|`ogH;Ct7q@5e z@A>bNH(H8o2GrV2pkm@tLcK5f;NfZS+9N**(cg1hlG;-l3aA)#gY&``c*SISMv9^? zU4{(M;bl|Yoq9xM8Tj7MC1`~44R0cO(RHbSTLISTQtuS}-pi(RV#Gp1j3q9&J+bKH zN)Z$jTkvVVp;C3U>@|R`wVFE~Rc(+vh=Ve|tR`%`EAFVzMSdgS(=Nr5Q&CI#q?Ti{2*2(1{PS*oMW5Hun*>*XouiNCUaqs`)OHf6nC`Nu zIY|a-if_V64nN2Z`R_VWVWoaa$@zmov(arBy$%}I>RCk?=;nF#r z+S@NV>#@P*H8y?D%l zvR|F&C-FvxleDzN8y|SV5M5J_37yWEn%|9?;qMKgBV%-l`?<};RpF3Ek)P2wLA!3Bo7;$!| zr>N)Co92qX#jUXj+%ttlwLsm#j$Gyd%i2o?wE{dH4HFOl+`fk87 zf5XC6UvX$l5-5ThiMotPQ$@+orr72WR=C5jW9=AJF${MPMx`9jc#^(|EK*PeyneVd z6uz-5yuE?VigU^F&95um$OT}#D~m2~OXbL=cBJpBM~hu@N>pyy4a0RCRIj(q0*oPBPkv5adyZqCX-=iP^Hbox zVV3lhgp7y#bksA>EsvVVzneOB=faGMmxT(Tjc@Bm7;R*vBQ)H zb@%1AgykNgwhpDCfe2 zzfW=lDEs|OnGe)&sLUT7(u|lUA3(D?Q!7GI4ggSC=!xH99cvyaLowLT^zQ6JBXo1cYg@6 z@d^DmA(EIVt46m|i<}nbBB@ppRFwwIWdz3Zq3ds{2R0V;v>${moV`vwTP_RTYjY(U z8^afW#B%^+`$NXX_E?J-r-S@^K3JBYu7>}-<8&(T3Yjts{jE8@RABv&>_@TXL-Pi( z-Idt?Z>uD=cb{A4a;I|NoQRIpH>#@t^md&+C$lHz&Drb;P)#2yg-$Tnc<0;OZ}aZ| zeaKNYZNZbHN^v{ea=}HgL&7DdrtZmNavT3fq@@1SzP~DbuH46D@sM2IwQ{l3`{Y}1 z9fJAq0TXN5xA-f!4yDPzIu~1-y|L(PwCj_zd0KO|^H-H056NkTW%0R8N?x~D-o14F z&|Cg*PwaD6rh&>zaL69e6N~WP?6m4)_h<8awQ&X;ihq_KJtTX7?d3~7HJX~L=k{64 zX^B61eV#K;PhqmI)Q&&zPAi+QR)XqJkO4aPaloDivr~fI=66E33+$Ac9{lKi1c%!B z>k`3B#q+8+tTXTHowxFup=*>Pq*Uh+xUuQLr;pcPu^-!tNp1RW>dN;#*uKyZ^+GqXybWW%p)SnC27_rR)rN5jPK!@X)z zkMG3K*ZX$=_u)Zjm{8Q29W(`nz+#dd{AhzZsZ*1-0GJFZp+RS7v=pp04WK zzKgwrEj9)J4?k6JSC!V+Uiv@D<Le>2C__shgE`} z9Xb4R;p$lb4_2_d_g} z8O<5M5N(tNHdya4mc#9Ou5fuV2oe> zv3Bd6;+?ljzu&lUbzStW4X}iTWafbtz%1Z1b(JA~AizDiGPS#sjPU|D~jf z&uI&WnJRE&DNyyhOZvw$;U0rI0$9ep1r|-a|4J7D&uv7ALiLudYzopr0NrP^q5uE@ literal 0 HcmV?d00001 diff --git a/docs/docs/pics/wsl_pr_schedule.png b/docs/docs/pics/wsl_pr_schedule.png new file mode 100644 index 0000000000000000000000000000000000000000..bd6c892064d0195b1ce0de9bdac435f54ee22b23 GIT binary patch literal 89327 zcmdSBc|6qJ|36$2DqEL|NR-OHMnv`^WNB<=Z&4!qG8qk7qRoPZ#g^Ij{3NujTn#&Ut%mpr^^Smvirq z9Xpt`E}n<%*g*$=rrpCpw`0dn8}l2T;KfdNh^E?(+%HEa!7sG8RdrQ&?8pz_w{~qe z_1F9%^kj}N`C9Zp0=dss-yR3(H06H z4nQa2?Uqu<3t3_3?|oHe)95H?dd6z?@|nw96rpNdj1BMgJY|A=$lR|Y->d$^LGvf- zsbdUHyEzGAv)8?2c|1SWh zC`Ibqjv4N2C4Ay;hKbiS?ZUubD)9`kjNA8qj)``PZVn0OPGO&BXwK(5@Q6F50+8&$ zg|%Xpefg;GYr+hy0O9MJ0^(jlyNY`ETg{T@ipDeT$7$+OG9+|Z>k$Pt{~jCz>F7?* zd_L*=Faut%pm{K!RYA=gP<`#Hd8Hhtj!TKbmK;z}lLN%;`}Wh3!Uw>5W#<$=e8EfM z8t`p3VJ>xtZ=XP5*2}08AIw{$ha$sbdffQ%UcwG_&kmb=Jy2&+lrEbFbi79~`#JqS zeGZd=;w@x5D_a$I#rrG|g`j%&i$M`n98l!vX@<(u2{nzNg5x6W0tANlR92~4Rri^O z#w0X?&4Vq=7q;ZfmLs26!Pa!0vViP-)=v z`2N-w>ge%|tQ&YQ-S5~Aq*K;uvVz4#pv{=vJu`|c+OnO8DZ-mTZ>$maQhx($FCT;< z^8`I=$fM+kq&Ajx5+A1Pd!>O%5T_5`7OIMOkzX~^~0$o zvhEV9#OwXQt+3ZlWY^8+zLwD;V#@7aVy-oA1v(8wA>XSzx+x*e zvU0gNYy=9UEk7EG!Wb!$x83mQE!tAZZ&bES$GIuAv^OV>O}xBmdkzU>8(-}<%%b>1 zY(oxCB$ydveX3C+?%tW6>nr28#l*9mN{50!k9H7xF0fm6cxcL5P3_4&=QoPMq;tB` zIMHXPvGKRK8Lmhi2bqMHIMUA`&n|p|lvm>i{iynM28uhK8S%UXmBHKqVb& zKE5zx?oR#XVG(w9HJof$R7Ucgph0h(hy0#*#B#ZoJZ0P#!E}hP&(CJrF0;

Zf9} z3h=|p_G`;?aUeKxIaf%!pY?Rd{dg+=rHJnZR;;q5R7WG|5Xb&TVSSBz8L0bjQ1_pq zE|vCP2o}OI=$JaW7aG6{NU`%JQoHyzc9K{_tBq2Dr;pgBb3VC*%m^J%3a7 zVZqgMk2`))+QPyb%iwuA=$)^N_`g)HK66K0&|&du&Y-`RgSsDsGHP*0*z-fL2Gv#j zgb+tmiqUX9#M#Fk?uJsotbUdkJPqiVVJT@S?50v@z$|GsA$;nUo^YS6`aCCWzTj zxO8zC2MDxJ_R<4ZckB>&zu4<+o%;HFBWZOp#3xEWgNf@yRfA7wma)^`3tS3bZxV#~ zmu7l-@Mlivvwg4C8+^<5-8emQkQV82gQhG zx8YpB=A+n75DnW?a>Mre35MHt?Jl?r^tv3EF?K&&i;Wj9&H6&)TuR~|CnjHkidaa# zf=R*%b^w9V->>m=ys6Wl6=U<-6_Utn<#@EG`1_tCUIF)9xmORP&{2TCaRIxoO%td$1B^qcu-IrihjHey-88VOQ|~ zsMp%_vs&5re2i8?NUzon)v=2Kb6e?#a4dQ4jHhMSLJ{Ezf8K0*%Nu}ty1?{il~2dj-@z0 zeNJ8i(fKjuEaWLRD$$*+xA(b1+{`j!=7I~u#G8o}xl<%=yznu_&l9l+Ob-qY2Aj_| z?X~kcO8YcKoya(o&F1Hdct-o_0QTbB>W&s{l=MO+!mR_H!>fxZ60$DvzJo)eCX6I7$$|?@~O?YHu=s8#DNHv23m>j_o8Q#gS zFiw^3Gm5or4#=OOUET21z&)sL?t{c$Q9ROiu;xKYEVs6L#4@rF;{Tefd+JyH*HDf>Ftn6#Q|dqpI}szUaRa8 z=Ci+i<9rDlV?NtG6Kj#CT42xLy#hR^Y2hEq=ung0tloQw>@NSX%)1I`V(3Swz5jB| z#wxb^!!IWJo+Kvwec$tuV|@!^OnG3?Cj>L%SNP9SW`LN!&n7$v*VG z!S4bl_rW&YWFKl!>5`|Hh+7P0v_ATnl0#7V?1a=mMXT}T0#d)+=ASU48b&4%9t(dTKBJX6MBNhL9}+x}WSem@#1*ZyM?@Co zkOhaMt=B7W*G)-0!-8W-)d+PnGzX?^+q);M`t8p|n}*1fd}2A1V=i`||CV&J7xvmk zCacAc8O@MCeWQ+1_MsDXxUZKhRKeeNOQ1N~ObfRXz-Z!8W}N4zA}9M3)T!cPiMWhb z7D_GE15|EUoH$5m8Y@NK)iDjOthQ|f7*IwdKplJ-Av>@~EN(p_A;8{KuEUAmfDC9N zG3)-I?m0wOjU)f^Ld?&;j*A0|3x{MMdMTZS#V6S3H1hdxT=QG$O>nL%`}W4`tjBf7 zj$@<9)b(?o39HNNfw}#+4tu^+s_GR{AP}&{SHHdS;QwT16`QrO>V==gLsoc{S}iqX z{*vy;=mmRbbFzduC3azMOB>%gajy_{$tRec@_aTbPUI!MFQD;o!V&S+6+t%B(T@DH z-*@&nP@Ok=&gCNX0Se#LfA(z*kIc`P(l-1qXUO^Khbz^-BR-E5;0F&>aw+>(Py(j; zBahe2L$>(8YJ~|rOe$Kq5wWNDHCpw*F1EE#>bD*3AkCF1^jJ=<@qlDyrn*?E45I<< z)}O4tlZY~E>Pv^cw`BgXh;Q@pM;YaLA*L|+f}~sz#J;}tab;=)ct-IJnVGh=Z1ZJf%d3WD3%-CXB7J=X^=nveo0AZC%gTjgtECa}dL+s0KS% z2Y5l=nqI{EYA_~UJG^53@!=giqU#Rh+qm2iDz+1nW#alR-zU~kX-cX4ARa69eWN8F z{;k}@NC>Q07g233L~@6j)C^SlR=-I4GPSyEZ_yR)jbkKbTmCiA1up->HYH(CHF!aK zYM-tAkzMI+_Vs_C^h8959SYs3fh`VyCyqQya;{Ls?C}_};iqIT;Ggp~l{2D<)|p$( zIo+CjuSnc!J0WWe!Y9V>&_&~DQsT62vEA2KxAIJa$MHAB80i@4Oe8sYS?B_vT$gzJ z6urZn(M@8R1*wqkb!n`4;bCx3Fe-I~@%j zO*RK(gy!WyRh0EB=Vx#;xSKGP$Nidwjl|omPMg{H1w}>d+|ntw7?g)@ll@5ozg0L_ zy-h?{GF_lznp0Gdst{>kTf22+o8iUqn8pe=ywcbQoA)*b#@OI9GC3h8xp|oUx2U{t zS${>U`c^!!xXeh`>ACjsU7t~LMCM~ypQBHmO$1M%>IF%T8i(*CD}2WMp0gZ5O1d>$ zkE6SOGRsxl;ykB}pw0L1?L6!)!C`bDP}Lg;t9j^TFCnKJbdG~j=n0zl#fpwe$(t^2 znQlw>-yR=vY+-!de*RM+I{6g`ckOG%EAdK&|wh}{XsN&XRJ4RK- zr*JfH^&i7m$P>4ytNJq1&0UuOqsSb^xAU*go#wGKGo11LZh2bo#VNI;jC2QUZCWj+ zHP>-2@eJOK_7eAEJG7Cw^d3^ch}|otvyl|NM&FS54jq6;)ywA8g9M6>MvpDUCy+7Ko%nm$bPrXIJqImBoja( z7{tw}5z0B5F}3z_Qan=+huddic1>bXdVjg>r!GdZXtPrbUvB2&4?(*H+{-94zQdbU zlBNgdqXc!aIrVz3U2jg2#bBk9D}W93x82;#!FHRsc9SsKP98=&@MsArEQ==)eRS0_ zCYyPs6Nf((-$X3a%VPWWdgA76b>?ReLkyOnRb{yS{7Dz%^bhI%^EMK%UkkC&iJ!fB z-Ld2=yG)Mh+5_}^A-cddGVQiNG@1=#RO~f#`nCL~TC(PCW`6{l^Q@evZ2kt70M3fd zhO$AbTD@F>xG4taZ@H}q8}`h{5%j=FYtM01%+G2yu=!@&KSV!hKR>Uuw)J@prrv^s@dr1f6;LJ{U?cooAes#u_4pd}EGxxRSm$h9XaiNn$wo{E zkDu6sFNo9BQp`{)6Q#J}BdEmp7<4i}nH8YdHd&$>lII(Lg1os}F!S~9tQPrlWKayfWAG{?)tHRW^^*fvQa>J9Sb?n1`q^>pY1-;A2r zlZXqTs21pUCf6?V95+Qv4Aj-tNI*5B~WfI=Lbc&EHa5r^`eaxD#`J!m1%FCc1 zJ(!wljRS2IiulPd^C9ojnYBBAt;4eZs(Qe?@6px>)s$_*t9oT^6vCO0TNQ!(3;nct zJKmD(0-7ek`Hy?#^`dzXubiIZhQO%+JIc+tzM9sA>9CQYSPj-o!tQ>1cT3;seQ38S zMFC`<68SC0wt0HletsaD7}Al_(6^&(Yx5;XeUP9Fvuz?Ihc+hdjr! z@JT*Db*5)jZ{}k3_3KpqyN{?+aYI!xiK)T-!WgS`8Pw}D;OmzGf z(oMP#w-u!lKBDk-(&lY~QMWGtOlEwbdUO}qkA;Rzs%%A#B`makvd7Av zmLfHdJ{xim(P4l%99lH`kH+Js9>-*xunx^pX)Rl|McD zKsDv4PwwOZ8i;11D^UjV?iSfYF)vREMN{^~=p<3Pus&K;#7LK33q*?sQ6lgNqtjL1 zKs0Nh>eWq@>ENXgJiISfk|Ua96JeO#BuZDRY>PbqF|lH4KV9Ho!<)dT%1j6PlM(`_ zH=&Jl#74q#y{!jE7udE1j$lkza}?TU_-%slrl*0ZBJ&A7*i)v%!KnRgGq}OH8FLY- z8&{|yfsBe)^fd72awgl5o7IIL9P>exA(yE>Kp8_9*qngnIlEt&YSDmnXOqp_JNMji z#q=DE*S9uZ$wUv1wZ#>=Su1O>(Fkm(}9D_ac?U0!D_}Gt+6l)=eZ`ff*lQ-O1 zqE`9+&9x!hA)$zik&@bX@tu;%3H>-qGHUap`YuQ>j8N#o+5D1n?tlg<@o6Y_gZ-IHI`jC2I`adn;yQY-UsUKueM9Q*+m_`{)9v5>Le|Kywr9L#gQu zn6ow#fM%p47B9APFlIsHU$)@VTIr^uQc&41je7;6_pmp+83wxd zxz?XVC@Mu;f<5=&c_mHa1`^Fo;ClRvzQ??!Qx=R$9K>ai*-&;gqJ#G`B+X$Aom~xw zz%#7$*X3*;4@2B$$^>?pq#uXNfVuY{WGO|G19;t?SCsCKXV^CN7hYGL0p7FZW#?F0 zf8jHr3oO-V8$C>WZhH^2O-R*%qW2?g2)C?{M&^TU)O((Q)CBfy_khy^wS2GZF9kj~a|!9Q7Mr02a$ z?T99hm{6ds6(+IV-0>yp`_kD(4cUYAU^P89rdN{>KzjFzF7S-5wTKVAHhkUXv-yNksk)2?3C7Mr|nFZhA1=)r8jHki3DhWb7&C|L2iw06zdvd;HB7k8z zOnu|;N|VO+m0RJi?=~Luls8GW*I$`Q#kb!nA4n2S$6BTCI|5(xJDJ|p?)*bnnO*4u zyIlF&2|>%;Ril%b6ER^r#Cns<#2IaU9ub)=rL)&poYP+#BwczbP=SPm=jq43WVX_# zm(zy^NN>o>D&cXlh{W|o7VXI!B~Qgzi8t44i8nuNh+0R$9N=_ z^h4kHOB4*JPA@cMgxe0gZ78>u-cLk{2rKTm$U>K?RavMMH|rP9ObpM_KeaZG#iWlT zRvjm0oDhwz{FQ{W?OCK0(~FbOEzO9|YuV9$ZTjL##}@tu#o%Z>$8|ZW*46tV!irg6 zb<#EK?yNhv^9@Q@pMx9awDrFF+_v;n4hbeFZ|t)iT*g-DO(`ri1X))hI($G*$+6)z zI?jb~kT(f!GDS5J+!SprOp-)$>d=D;kPXA6OUu>c{E)mXz?PdXt9~|_y*hm#yFRNJ zx6qY;I<*=@!4c`N}BliFC7gSIY(A z(|*cu@1$sZi;TsXMl-yAij0`@1tEvCl;ZbbQ39&tVqd7W#9O(t1O7 z?od;(k1igTiSmVXVmB(4%9^9zb={R%4(eFKM)6J9M8vKihR?}oV|8qq=w_nYz8S<= z`9>Rk^SP5zrn6OxZ~ga?vEwSPBvatYtGxo=UmVv6!w{E&4{1#A@7fzHLae_@aV%dh4{Z0quX_*MkT`k|ISZF-~0#NkT<5H z;;JID5R(CPc>L%`6&rS%ogzZRD#@}_>6z{EqD#LB8x{MjR6m}4MZ6i{pKQZO%Sih+ z^DYf~L#wh+HWLfWZLtyq zdDfevGlk!rk2qajeIb)$Te&nMj8s{OO|8GJl#v*FZhX;`cR8cuC~yT7pI?k8x2)c% z`ti3q8^|1c1cQ}FYf!Q0Oq+~uHYS5x5muO>_O&ite87SBG$#EzuK)*)KgORI`&iLZ zJeD)fYRP=W^c(wA9?_zN4tZeC{A(M}VQ->e?gDV)Z^Q>(n!m>THoK7q9US_x$WuKb zK31vwV@5%7IG(Ra2QJm_cafB=1Wp<%?{hFrXU(f5>l45U%Gbg#hh@7hPc!N?K#$p<-t8gmqRYyy_(aiWY12cN=LT~b^Ry< zOtGE^#yi+yAje317*M~?%aGYcIkJp83^5lF&+Cp7Dy zr0;2X?H3OZk1+I;+QEtWpCq9#T;hJ8>JUMLj#gmre7QlI({&i9`RL2DBQqFm{?mIE zn=W4Z0A)+RJP>j9n7V5zhMcR)Onck<^nh**+aKcwE+vjCe=@F2o{Z7Ya{KLVbQHR3}^HZMH7ksIzowDUECHWsLI><=-h?Qypu)_>am44oi zqw;hC6>>5Mn14OKnoNV%_~&wmmqfF3%vw|)EK5-B1EVtA+PI+W(3tqtA^R|c+wis_ z*~_rr;rv8(;3Tq$N+SZW@EnB5_5 z*;?bCMkv2Z!3^FC_!6e1c&Xn9RaUOJ)Yq=dq?he)gA3MeK!s#K0g8)p<-IZ`X8evH zmX1Bs#a}Xnw^CxH9a^b($xQUp7BkKZci=Z;q6OCt-e4;2(i5_el6fokAJl>Hyw*Pz z&qT}BcalamDF!DKK)BJ39qxH-LgQbhX|w(@8$;P$ZUGbRst!pLF7;f7^_)rLMw>q) zt!)3j)P#cDUtY{wXa4QG*7i90k)e9do1>h$qF0K)G#j@>3mWzvcq7=WGHZl%2S*ui z{CT}!*}c{PUao$eLC2csVlUWQln(a+(8Y?m#G;*jJ~>GQw!!DHwQALsH^m5T-sMinwzcW4j4O4qbd=Zz{p1ocM3$ zDZ2RhPW7$`OIvOR*eFFdUZlaKl5#;GIhmUjAPI2kQ9R5@Z20lWwWRFN$6l36#keCf zAhWk+*>W6%Bc$^!QY&9+=>iP4Y%btBe5u3S9@=NJ zv|YD*+r9}-;iTL~W86_xrs9T7oN||z=a1!mw|`&whbQqo&eMmamnaQt?cY{@0n)^I zK2do^nY7%$%fO|Ua9^4@<1J59l>A9QUU79=~2>$f~1 z==6l4`pj8ykt!zCpZ5hhP}Gz0hWCY6xMf~AZdLCZuU+sKy1WKJJN8THba49#8s(&^ zi~>p4pmwb9(=`gmLGAc_>_R>UYeW}tYkTw*aS)lKochS0_Xr3VAE}F|CUFv5#W@(_ zhdLrGDIoFT&eHb<;&D+(S^OS zI%h4#D*fBe`M-b;m+cK+h7d#bAqsGS7-Og|E^nvSB!9gx2Orn8>z@l6q#?d zsh=uGf{16cpNvWY_d)?YhXPp~Jj?n*B<9FhEz6vJbW~XQFuBmtDi&fjm;oXwHGmzI zVXZKo;$LW}sn*fW(`ODs`K(9)IpOmiO4ejb7ebvzn(ZmKYvR{Oo zP8~(&mnZvgk_0PThL^U28)qu4dU>m&^TIcdUl|9;g+4Bdm8Y_=hpNxT5KSew_YmfW zxN}v1Fe#FF9>9+$_sn5PJcpkCn1&lvoTrHf{o<^9pV12|#qtvW_flkF>riodjk?s{ zE+5Jq3A+K8`fAU+oZKOebZxM}Fhlg8XF2wjNM? zF+@yQg#ftnCL=8!UDl^&d4H~!e1QqrwU8gUVk-t-oea0^^|wjyfI%l6QjB8RwG$yp zmkK!(}RJPe8jL6WDxU#2fhe$M`lK>w0&q)%GP>BsGl?+;zg;Tya{=`eeGe0M--VwfitNAiS_b1zSN0 zH^I#l8dbSUe95Q0*ZzZ}5SZA2-=vjenQQ{@V?|b6n2DrKkFVe1%!^|tXUGPwq zNrk71q#;#T3KR2f;)ug1=mJ~-mg!xlLFe_2?uV=|?4k?!OK25&j4l98_O1(68nb^v zk&da^*eIG|YKQ@t!NEWWoPunI6ZIDkP&2&7r~Wp8*iF;RG^+LaDf(2lDQ-)(%YLDB z({C(E?NF9g&I1%{B70uFii8|Mvh3IXD%QNdP8v*4D0UOgPJ+EEK)OB` z2f3{yiB{`Im!NHs2;Utpf60*+Iq3hDr$bF!reuxKa7T>dv3h2|5Nk)Wv&@h2N#MdHV+>!;H5 zj!h76PX*9zh^1ql+!3337>F7aqXxdV2cnx)gGsmdN-P9UPtoOXtJdYFIHVO+sFMqR zS{hX}2LlKJ`e2hv;9N#Mtsk^D3b`0KyQ(dw!XXV&-u?0Cyxqp!_IY;OB5c7mTMu4v zg}?~o@{uVHRDD5vu9c9ws@E65&D%kT>F@dAphwjE3(k(W^owQj&m_o(DFuE!FgLJK zy)MH>O2m+>7moAp4{9Sv)F;}s;jw1I3*rzW@j;%*B>%=O8h!Qi@{J=K;2Q}qc0husa^Za+YE|!|7}^E+ z)a!AIt^?VKkE*$WpkNs1=hpjy5#OmqorAC+cuz56x)A?lkfq-=-A{wQ1Vs{}_4iX$ zU<0(+NQq{WtZ9ZjXjB^{Spj+K2IOT%9Na&yB~#>#9+Y5>W+!{yRPB#f6oqd;qL`}$ zP{b%XVo_2FMqs?&0QRU*5cpBWz-lX3`v2jyh=GmJGUQ^YdxoqiUBLX0t(<3Gc$E#V z@Oy3ps^1#2JShe1r*MDX?QMh8Y}&bBGjH%dB1@AZx(FT`a3OIERY5NHV_S|Osbdx2 zP9Q;1es5c-|G8B5Ga57+(x7}n3W0+luOTlt^c7Q`ZF4R;bD!Lq+-)-}hvdfPGGf22*cni4_Aq^bE#!3CH=Rn*rJPi~mfyjNJn2;aoZ>7LF_Z)28&+v$ zp#<6g4`oK_AN()sX?>yebp}?Gma$NtdJ5%G_t<~OA&P$O#1{G1y*6>5k_5-!;C(H& zrLN?&bOBS_8vlR(DmUvTCuaxFWDg~o9ix+Boi1$Ga}p8(Ul1K`vt|Zo=p}*+b$h9^I2f|BpJWC%&ifgIm<{?h@mwHZ1LqiJ$hY8Kde`_a zVT$KU%f9xvSpqGud3p5V+b^C^zb}7d2TYt1>|~v%`ESXO&7Jl>GnL?O80R)aX)E>O zH7k}>{OSElvya5h#;@n0Gf0(F?kE4`u6jup!0`Nr{*Ah4T4MPJsgs{@FsOmfQHX3b zwX~oJEsKaeCQT`4lFL*XX@~QCTXi4Ss~B)^UJ|1++BM^Elcx?Y%Z|PWX|1w8Qz8NJ zZKIy!F}WY4>P|Ysu=ssONY?l$;GR%E02{ZQdyt@(xj8w{KlxeUgX_GHdUJl7lnQzT zZq5RLD9-?LLl>B$4B`Dpti3~*vzz+_9hF@eH$&q&>g#T2&mC9F-2xNTOEyV1sTeG( zssIFdY^YXARf$EvKW{MkSy0qcN9)OU9yh~J*>7mj5F}_u>na|XK`E6nQ8FGou@)7B z;zM6efIT(58+fVY^)f+CyityIA~cITZOwzPdMzIUE;RB{!=5{E#H1sN&APRq<~v$( z?TugK8P;=Gr}~IDk2~<*OYKkv{l{;f6yHxSbD|62fP$*drTnZ6<-;dIf_gVy04aSv zRVnThEOH|-`qWNwc%o&Rudp#{Wh}wHONS-SfXnmK`KX1qXSS1V{=5zeGT%}%tr2Av z)1K|RvbLZVN}T&l{sc~MsdtIT!*k5-_f*}QJ^-H9Z=H1MCKIi449tpBLx5JOd+N}6 ziA>$&6C5>inhFcuQPQNQ?{j^+kM&GxWBtW#XyO1SD@*3GqlwL)|=r|I#cc(lkkKK zkeUp9=FUHWu#jRnKsA-yYkIKS&nFvf4}yrLq{-9&=WS#s zfb2uqfQu6YZh_IhzFo4O{;9n`?4XmAWmbvGb(XO3*tV77K3fq&-;xOOXL_6U^`cAK zJk}U-y|25&)_E>HZ`*iESb8~erlYsDn)I$bIgR;w+v@jajummD{8um^(dLbzo5ZzG zm(AKj%GX2$F&Z(^{VK#3gU+9|>oaR*9s8!D_EfL!LS};Sw9a!LDnP2AgTeig7kOga z#Fey_>n10e@Q`#wCN%7pviU6@1kniY7DOQVy^ChX3stVELwvgUHAM#Wr$iST+8h@D zPEYD1tUdP|o5q+cY^iw8jeMaAzzSGH@aV;#Vs%3IAPYY+2bR>U`7C8=1#(Kf6Y&k(i zdm#mqIf3TL!XCZqeSHr1r`PTeyCp?)B9uqOn#ADMq(Y_cL?dt{HA-Ce%LXTAP0E4` zXyVORuIAB~hvAo6l(mJhuALncnc(PVUI$=9EOsL%<8$MkzI#ihO+s25p`m_xNi$yF z-RXRHiAtH71IN+>{oC4i^pB^lzXR7#kCSu3#(mhR^bEh@=7ollb{J`JU^;)=!yCpG z(@lbuXps!l7MG3`pYAFnUowewZCo8RpB z(lv06y$DDI)6nu--$Y|rS)IsWy0Q?O7rs0isyrZPZ325~?K4tDP+Ij}A&|Q2iB}gI z_BV+~!N9f3nt5+Xvl3n9C3#)=dQn8(C`)M*VZQ!ibFO}@711dC<$Q->3eob^Iv`oC z+diL%CF>rP;CPHfpN4J6J)ZWTL|BV1nc{+L=T5_P1lDUVFEtH;9lX-u^^?GJHJ$yb zG12*_hL-MYeW#`K$TF~PpHvd>>&y#luQx6u)j>9o6Ot)bIvhjrn&|R~tF+SAclKWQ zbtw0?0JmlDK-LLeLV}8IhOqee)h5xtiWmJ%lBBOk_`V=`JbF21ka#XP@e;zAtRO6w zd!5NSg&ZKT5oVdI6u0QIzo{0OfvfBrr7eP={mPSyI=TB#8+6l zmd>xE#v;-U!{?jAb;pY5o4Ax6--~ZRq^REkkwL)d zQSq}Z{n93A$}3_JX2$n zgg8i*wY4&bL7sbRwC`*}o9zbwsB`w*xaa$?ORN(*fhxCAzrcjpK5Qh9DXSCWZdzq-Hz?JhmnoY&sJ z$ll4%=3C~=`@&Z`cpQOMnVlh@{`OBncRmm#e$j@fS`ej3PCEL^QKv}AoJlBV>@Wji zxC(y9%wUrL_C}qoEWDUP(!ERQ2ZAiI97nf{bkS}jP9v*(MoYYFhV^1~RCiwcNC7y31IOa^57TBsto!YGXA5Eh z6YHs`dlZjNjZ015emk*1IGs@8MdROgM|np1i<)1a6_Ia+un<%ICD5s-{f=^yRc@Z~ za&JeMl+inX6>;#1$dX0sKCx_MlGc>P+6(t0oWVgDP zk`N$cm~jaSjx$kbuShxd$;s|^s0-(r$#_z7Z|*ut{;4501T_)V>rvbL_L{mQkud4R z9Q*mDBC6QFQwWedr5kV5TjBiXiZiR2U==h&4$^*Iwoxi@i5ELL* zrfYgm&EGO;7m8uG*Z#fJ#}9nJAn@oZpv*`9>FrYk8}^jG?_jGqj2?y=frfb5Yl{SNYHx%emhe zrLvb?wDG)6VI}mYlR4(d#45IbT5Mbku4_+^)ceov+2L3PE2Eq*junrx5ESO_LE{#$ zAx%o=4hcG5osT(Zb4$UxYE+ZgvFzr-Bo^8SYNJDB%$eW*h9;&}DOyECV5b1!D+ zQ|^;z!ty)7H=Scb+zv(HqSUJf*d&@w3SrnO4xfIvIC*?(`d!O6Qf^1eU^4{9U!C;X zu}RsR@WT)SOw51!m$6y`!)Op zVF#8xv!v`ZEGUJ54|Cf7mN1v2+9ET8v(7^ON^!kYB7&CdZ8x2=xh*|~ymzZ)%`LwB zT~%jty59<(eOEyEdQ}(h76i4Qyb(v zs}^%2A-lL8)Wb?rDDVC;N*#^z^Lm&dd=#fnHQ_(rPc*k?4E3d_48k)lCY_M?3R%7? z(fTm#c6Ck2y|-oe^SvZZ_AVk zsR3M+TRcW}oIivxvINS|F3KMjVCVTCXpDV&YM-Y#bsU_+{>KQBkN|NKo;cE6b#-8| z_YV2Et}o|5S1UNOSIN6hMoZxBlf`2_AvP>2ykft6sVQYpy}I;?fy!!x7GRP=vRu067M-R&d1 zL6tP%Y+;}vep~cPc^xKRv5=*k0x7KQC6iAyDd1xIIc2cVVMY?3QEmDkngNu2p`GW{ zR9WnJUcng_S@-@X=qG^5sV8MY0Y_VvZvM;EBtrt)#S(P|Ccyp*SuT3olNVP5Ik8U~*E- z|BEK$Zm6hPTu^|m!8Da`cBk+%{D)1c1qYbptzQNOfa{;-6c+yFEeE9ze71NivS)YpH@3ZM z$?!*u$G)ro9uCM%tW?1SuILId1pEgJR%-t19pEz;W#l-106y7LIs1ODs7Sr=Z4j_P zYLxlS0vG<;dxqIr?+Zacp}Ip<_0oUncZURELrm3j2>O{Px^20AZyWrpsqnF-Zr_QM z2i|b?LRaSpa;x5JqZvO-PEPd?g%Q+K5&TCtMDYvH042tLYsW5wy%U~Ck( zcR}>}=kchsYCDgs2z<8(*AlWq51+0B9USnbU>m8cIW(;aA#(M{*pCh!f!6T9It?Ul zOKvDgzS`R0H|9tX{OUmPYj|&+bl1UJuoBPQU}1?em^BgHP>3XsD{mZ5BWS5@Mzp-H zdgb-G9`~0&UpU5MeY;EVBP_l4lvgii!@T?I*GNfP0?W-wD+7=7MP`12E)o=S6)SR} zWywo;Up0f8qx=CAo_vHsae0!eL1G$pcr;|I@bbrN0+r+N~LeYHmjV^-ov)rmmVt zPcO-1VJ9F5`+x(9S*y~48R7etf8<`qz?5-cH0{AxoW{7%KgUrg`M{+wThiLAbZpeX zN;eN?Fg7|WIDkjL;v;2|KLNvbGA4kDE5u<&VK%-e#LvajvE8LA{Kws~4!_U30q=Kr ziMr`D>Z_ykqwpT?Z-f5f%7~)Hl+o-Ed!M5&p12z!)=9L1zJ)9#r?yHhUODJI-098{ z=>soZzKVcc(~f7aY=5u@8O^tFy-5w2P>@e#;G(4Aq0*D)fN`1iwX7nps8-2GrhG(Q zb##kZRnvd5O$x1xy(1t;{>v;!L{k0%K(>kp2&r%Z6VAQ9u9J8FhqYk>6Q0%k$_5s- z)hV~NC<^=&>0ZGFto%wBU-p1AgR@Yu+|MUv_;3GKbWg|+M;>xQ=Rmo}(N(F#ub<`E zZ-JLpzae4A^JM8@mDL}S^A-Gc3E47MBaf=1(?G<{w&N+Fq|n>(6Q@4cW-x1|w$trj z@VaVHfm0V(x~GPNXR_{XRqP4*5c-ddXKR7%ltB`>9Wqump&TR>&$KeYFBF{OVI6|tHV9Kf>vLkMJYAX)rZ=w3{@cSm4(_D;|)kMMt3 z4?BRn5*GMKA-E#71X4u6k5dIx=pShWPQ4NbJ+@@dw3|mU(cl68>pvqnkUhxRZd9K* zknHnM&mq$>>+5j$Yrcrj~)l_wzl@nO~YLIrl2L(xvRc zq;3J{RykA1^6*{n#6Aac#e1(n5(yI3e|jt!;lKkc3JSO!!n)T0WQCi0$#N68_o5>g zvTj01sIY~5@JRlO=##LoKr4QlPCTZIT8%e3? zD4GaqZ02iV8+HQsx>MLo%g8zKD~k0S78ywB*c-wNkkuC*bM-aqx7HA_SZ&P?D}B2- z&+{i!W&))qS(u$0#t`Du8zWTU+!BX_z;f5eQ)QdFBj3}X!hdXZ#lZ*Sjy z9C>B4Yy$gEO&L5Gw2M|{l@+J{HnY)LI`{Zy?zWiXf>TFn3)~wWqUllRzmM)?rq02;gTTPTf_FI{ z+xw7W3;?ZPW)-n0s!zp$%k3IkMX0dMl1%740-UsYB)%lg)ZQ0V+o=ug`~glqL|pG1B!_8Qyiu7j85RV<(w zPs+fcJ#6=;Lt}#jz!yJ7I!jBg!*#Z~!mq?i2X}Fc`!Mg$LzI8%+$G4g+gHi5J?(z> zm##GVxvPSe>za{ejwfk+;Zm|B4&%yDQ-1fHvUA4gtONlNq__}t>+;UdAD1^9XxCWMx{v=)yr{83bktKbX2z!Y6IIUbUJU8XFcr zJw{3O^(jDTdQjcJ-$sg6XVQnhFi}9U`m%kc(X-8jswV{Y&dJUY-zF}$T9UeFw*V;U zD7t(Ypp)L_Ed7nu1qyfD2e(EGE;LMRR=9;Mqsns29eKGql@rN(iomx=icm|1+mBI2 zR38%PyT`?u@;*1~VZa3l3+Ox%U6VqSG}4@<@Pzh|#=32H+VCw|id!&X72Za>Z2OeK z3lLZ;vANDl>a|YjbF=9TKe2;++YYd%77&6C^W9vzrbf^nwoaNSJijPIwb45%kt%bQ%-hJ+RP1iR_}y{spx zpk%ynx2Dd?+l%tAIew8*mivx1Qrr21x>Cr}b{P!<-G=WHxJP~R0I&9cK=@iev@zz_ z27Xm}IQ114=6?5i;QL#<`{U&j3cB&k;DHXMP4cbI$({o~*T*bQDG7nu;$20F}yDomwQ` zoctxWrKEam5CfyBWrxvRXx&Ui8_mWDqqPS2$x%fe?SUJt`+g~#p59qGj6$En%yog+ zQomlVV$8!6)hNWs0pHYE9o?1gds0Tv{(jVx=bP=cRRiCR^M&4y_f@%EzzI5&JaD_|!&;2NZAAe@J~=(yGNnOZ84N59NmfcXyb}tbSoXI_YfOYR(YPkDVZIW9Mi&@@5yVU%( zsHU6!BxO(xDXLX{;WuV|;pzK!JN0u?yq4VA$vU(*R>v_*-z0KdoG-jq7zTMU_ z08es-ez7<_f0o=rp~MezvY2Hiu>7V&PLPR?_yipTVZYUJ0=eaJfagiX{=E#l&rt1h zD70>?s0Eq;e{3+pKt(`e6BM@H1~F>jGIfB=#wlubPYyGM9kJ0#mwQY`GPfc%&=Kg0 z$)-Fou-R-9Jk2|B?zpEJyg=AFM*gz4-=a@>vW|WXF}7n8_zp2Wx{Cf47ZoqS8nf@7 z&XDDKi7L+49ijk&<%EN>IZ7%ZsxO%no1Y z*ypu(-fc788ype>%8sF!T`09GNHRaqs9Z@NNPs{L=;>)^se(_G-*h@rotv|QJc z+bXW+#Vkq-ZumfBU}_qko3aB9tv6Q`jC^XZMcIq5qIZ`~rPJ%5AS?4n`&Q2RLveSa zPeR8Kr2Zn@m|c2K()5ma%zRUVF+UXvbN{Av9Zu0NaGhzRdrl7=U5T3U8t>WNld7!K z$iHR`QI5{yq#D2Qm;w2N5-ou%@B9N3&TacBQca-LO1t;rqW0JnVRc9O%IlUZo4^j5 zXa9$}_YP}1YvM<3tcoJ;iVdV#0cnaN3IYNaib6Jm2pF)SCEJi>Q*Z=Dwj0)J5bsi_l>c!sTN2tEa3|0~GUVqVCCGoWFzGwASx5oD~pkkoz&c5em zQ$GF-F;o%{2zN7%`=XFs)?7a)WjXQt9SHbu{Dmk?@1U-o0~C zNIYGAMMI>CJ&!20h&!^`S_pw&e?^g|7gugu`fQqZ2z-5dUmJJbM^_lN-|OHxSpq@= zk#~YoJ7`6rQ^Vlyk(dHiJ*s$=660)Aq8_eJAyT?lwxF=iAgQsLE|cWQ-{TOm4qOIt z?M))S2d4K|j7C8))HH40uWwNAZ`e+9P7Ptp6X(URRTjDY^`x+Xo6@$)?~!Zq?}8TN zPeNKsG=lRSsA_4Bht>+ZTzN%Jh-}&*AF<2U$N~6X9MeyVwOu#SEqcB6j5ir8DtNjY zsh|phi~^*M1lJ2nc#r<5^7HA_r$%|%E9WZgBA&i@@vfQRgDTW>4~CtpS*lPv?qbVZ z!Zo($kppz&DD(a>98!3l5vTgO}Yn|g7_!nbXLKU$t;&-c-<0}_&P5A6kJfgQ0 zw2t7Lon0@BMlz1$P!`JkvVc?clg#=dVneB$3NdvG1KPcO%U< zrC|Bq)(1AH9a49m`|!#hk~m0oV34rx>!rk~!~4w`H1AbNHexbUiVO|>7gFA}NHwgT z^>4MwQpo9!+A!SYTv4cH3sz*-$9rAT8;A!+c4~~*JxJFA7G)^)r|#q$=wbm z6}R4|hTibVptk(3{i&lcs4UCNrkcj@yUE%rp~B4FQhX_5jwUmVpaD;j62rC9 z6veoV6`iF6YH8u7+<+?*>(V;V5E`kANav#DcH?o>mkRT_ zlJdTKX%x)Nzd0rUkS;yG)nvqzuh%&0*?$L-h}lzIw%UpMNeI4n)ks^}%$wg{A|VTg zGh3_@QX-w-RK!m*E{hFvcP(n5NB6vY`K$~GB8U%d@9!)SKbb+o(#+VKCAs2Ut=Lkjbg=9O@KU-+HrhXJ=LpykAhq9tnwXT){%}Bg4d^2e~`<^Tv}s{5#|U z35EbdA?$D1chn$bBpJCMIYOv+E3~Bg`sy>+p<}8f0023aKW!kX*f*UT*)1~RYKPaI zPtiHGoGT25*-`$&rHRYOyImVW+AvCq+KKik5Qj*iN`92IUcn~QlJ$)%8I5wb^$*!Y zp*a+c20{6R^n<7q>O)oL5i~Vf7P+UM)2mdTuTCv0vpB*aZ!>y&(h}i`R!eTc4!mDm zAEsm-otNH){kYL@OK47d$ZhLdGBchiW$*sGif#D@jT16h8NC(n;K!y_Q{FhhSe|1Ni6e)Uri@Ew9m)EgrH3vHlS6 zZ8HYy$11ahuBpZkzY{KsQpcCOjKv1!)$uPnR=V0sP7Onp&cAkW*E+;_bu>@a)P@er z;!%CHYd}>aI>s;`4l8-@2{7@EJsd?4;|f^fQ97~L$i`%jQq{*a_|Y5H zJDSK1Wwiqe@7i>@Yv@pHDK}@Y{8p-hsMmor?W5* z_7F%{d){>;`O~O>^fRvQtNgfXeHee}lg3BrZ(QeFtyhZcPvZxR^X=+e3bTi)u?bs; z)V@9>Gqv3me0du6s5LEq&St!IQBrOEjc2pSrw7~vYO}e=?2V-YzQntm+`o-!9?pY6 z2C_ZTp7dDBf_EpFK%%(_wH_UGTv?G`>Lx`GebGb;wa zGPgW%&o+^!yz=jI_sRFw2yhF3-tiXzq5Gc-BWSscMipzCx8~H<_Mo#@g#;Hq89JAA zB(6^1A+V~r+b-z@+t4lo(&eH_TKYMISCimk&ZI99N$Io; zY~4>Ba_2dGw253_u!Uk<2L7dq93jj=YdBH=%oP{wnx1Y+Nr(9!X(nNK11gqIGK_E8 zQv1mUyj;!1i&kBl*o`$Zm>)!xJ@lY<1XokQ|C*6YTJx9S07AAGaplf zof}r5Nv9UyrmtZB32?7S8%MbU2Tb}xFD8=`R@D|8oR>zIIuK5I^4S6VG(?f(rR=AD zQYQXZuI7B&OUJldHx%m^{^er-RRT?Vy>mln$$U&YN0(a6c7}XP2;^04^{4>srP3s~ z#Fwf(kHORD9nGZ~l{B3o^@th&do-kqR=7DLrJ6GzFUEWwxaXKbc%pDOT}>>W?tjz& zf+3aRW(0u(&CbkJ+Ga9YkLt=gp`05kF|X?y?r1gL_kkkktg=}mXY>6y7x#0tkLzF# z(KhKu9J3Sp)IqwlU)91B81@~T^m0x{=QS5&f*kgt&?`%EM!PkD-e#pvj|hy<-6LTeTOWrGPC7$vh7-I zb5Y5Lv)|o5?{2;+7VfTUNF~u9Hg)}l`zs{fQ1XTi%cw#!b4#r_kVdxdX)j;)&SFC84hv38e=(6#cAN^g8&F7x%cL498^G$hHP zFC}Ck&j;iku6R_1)jHPRos1jTQ-6hzDSyI2XI$eK84P)~*t zXSi6}U94W2p7GP`SQji>dVD})VwxA}=;D5Pw(WjSvQ^dz;s+W>*PtU(<0Z++(x6gH zbV+uOp(GyXT^FqC@x0~w_jcoU>dn((AEYuees)cGa!(LCgAsZ*%85aKFL$Wm(ptd} zhEyiK)SKRusaGkN;*6B!;ks&5WF~mA}6_Vj%ha1Ox$?TMRxC zdW@x6yD^EOhlJ%}K`UO>GN6C~OC1)7svp^I^TqoDrb1erUfhNzX^2%6c0kbVP6DmU zHoyWy$z4>i1@?eDPDwkAVnAiuN;`Lk;xA`<+{rOMSx=Q)WZ+qh)f6ADo^Wsn>aa1W zQg5undaUeXp$)=&f{+{KrTk6Ix4EwIjo1*U5yRZEhe1{6wE&44PXhu!L+V5#a%NlM zt!1M)ecW*Ax4#5j5->8ABVnD))Q$!CmC>3n-2MttEd96fes#ug&9b$^77wfI@_hfTqk{V6#g zB?B)Al%7aC)EOOpvvhm&2Mh0({F4UZs)ZhLyFaREI>MMTQv;Id8QT*yz%pdA%|t8o z{#Ioc3N=TYD{NtIh(VO!LHsU5L)zaf|3+cQKSK_Z%u&_B#Szqz;L)&ExZrb}wGui# z11lv-8r2F~mzWpMVmQgL_d6H|(jGY}=;!*3t=Z_Ha`~sEt;RFDMa~ipGlTk^ag<G@DW{6&>Zjb?I!%G*np~r6OJ9(DT^77fcPvc zNEs7eYvCKjd>Ep6@yQAmfi9yA1zl2iDOzSN{=*634~&75cKcz;C3L*tt=jI;1#QCHST|xEj%Y5wpz$Yd{br1t zi?W%Maen}=PEM2UHZD<(4<8J`WrgQH3V10*b*+85F+SrfwJ1W-e*cSBif?j20Df10 z1e$b|Kog_N9UvK5k9{%QK;QR{T*MGeI_yzrQBUQR9*^67_teiYY6THLA;!wtTAJHO zZKV}c>Dq;Zi#a3;Q}vqIw7Mwq2QQ7Mr_4W(LY@9QC(r@Z!1$8GYF_SryX?^)oKNGo#aP_ z1LpNq=}4-+q(;)$u2=nwWHE^uc2wt^5@!25!_F6k&;Px0v6ZA=TIw#m%;M$<#=s@h zvQF(MnuQ+QbljR;=_Q-{7-F}U{+8I>noh!XdyZNEcp;feKUdv*wZ7OWBeA}3OB{8` zFt@&*W|zZ!r9)R_eO>vEGs)6i z9lsuT{9LP;7|YEuj;tHId_wg`p-e<$7+d_c_FDLrc8>yMs~Pi)3wv{iON-=1TJf53 znu`H>WhveyElgrmDaauIz;>K?R9#h{-laoYqQ)5=3Q#n0D>BhN%leM{g?AYfFvBC6a?0Fg)U%Ot-!dQR8|e*3{&W;DzuJUG-XzMrF@y@ zwOYKSo<+WypbJLrb?2VC^UMgk|HR1hPTVkk*FV?@?`_v(Q^#Y6r0b>U&zpDS5#6wu ze&dVeD6Qit4A_jC`!~fdNh{GhDC|A{cZ(V|6GyDA!Tb@C=O+IAnQ$A& zxD+^>l_0;KXA(}WIR6Dp-IMC5tG&5-8@th7xDILid5Pr5ai_AHTNY7$!=?-Rw#%}{ zs)O@Li3VeA<_FcKv`j1An}y26QbhwW#i1ynEMdQUg3-F|)a?OP`$7CwUl@BKm_?IQ>Q5l*~ zK7%;_*&?)%n(?JRtGKN21X--XhmMuiSeRy87|HmzPd-#r#wl@toiH}u(ybN{%6tkL zall-VOj=oX*zuO?IJTJq#h0}lBW2-`dtoHys%B3KTJ$VkV_in*rv`6Q>bDBpW=Twf zlR=%?i4~8Sk4>@&(VCRAvH zGU7aKNH%i=4ZP|RDaHR^G}hVl5Kv)dL8OCSIq2r=f7h(FFp*v&fFuA`A@M!_w4%Np zGU)4kUh`YhMpe9@Nuq1mVD1aqScs6#ooeTUmJ+BJR49K0!3w?hh9Tm2a&Vs4f1B*V znH%)|ln%;!L>gP@B{NE}QIyn~?*l<8BrgHiL~eM|vo!umbBTKV5`kxsWH~k|V-Rb0 zeV360(1BP*e9mdtEm|ZiB+4xG9g2(ISPL+#|Fo0V`?Cd2GE!>%f{JmHu~Jg8v zq@Kikr<&|yCLJmCCSCdQrlGwo@>a-nQukVN1HwDx>yrxkVZ(()_zU(>4ODhCF#X49 zcVFbz4eztJ==G>5F{^8_&u+n0ozLNDvV*wk=QThRmoao7IQ6R_9C%-3pOdcR?XB*?Rx*9z)|r{a+hX ztt7|s6V8_%K2tam`L3E*bl@E<#);S!5s&k2UZ?e3>pU3TmJ8`>7;jhL(wq@V)s4;s z#lKz1!P|3AxD5`WNxg3YzUCpuOx9M<7T&l-WA6<}V|=T%AFp&gV)fWPrsWu}SGB&r z7em@+rf}WHKVN$6frLiGv2R&k-!dCChAv5FcD6@49<|!I;?UG&lB_Qz(7;l))l=_C zi6C)zWQpp898#mMw)|`m8VYYu4j1>(?(x?my&CRMX5=k9(;vMc|GKfZryFEVnlvNC zE+v_0v2PfuE}GgS42m=++I_e%vY|CGJm2zuz=K@JPZEeO`V6;hi+39=h6^Hi6z;9L zztgVaj`sub+!OejTqX3{EjCXLqs9Hr`I4IjreTeK+`Xul^(QDX5F4MirtE(Gk3!?I z?PYj+rwTOYrl(=q^#vBgW9&Cs^6$~xYFA8-=AC_34KUW_(Hx4cEIdn!Ty<}r=c-{; zqf}ep^STR3=i)@iP&-s28V#)f)Z@HXVwn(|5drQh{ag=IS;>e> zs7&Y0X;HqKG+o+zBUS6|w4Po$((`d3wvAHm}I!bgr7%JOmkp+L3+ftAgyBspRI=D4)_g6!dj&dfOgYGKH|+$X_&{ zD+%S^dzT7`?Ez@SJ$?jhayKvOs8U~Bttk@tt2sFBacp=B4KAVO_48H;tB_lz1TJZAzW=Vg_Zut%n1LT>jCt z_}G&+$F%7(uBj1ENLYZVMieluVef8FL6WB%`g=7k2n8gZIga|Le$~;fQX26)`@FBu zx{~X(1W1P=LHVik;^)W>-}yoU9_u$xp@XNE&6_EB^N!E$()zruSEmY3rdQQ_L?d(C zQUT#~DUo-2A1r~hkZV!->JrRAJ-4Z4HcV_yajva(4%*{PQDF^DKFh#i?U>mEEcfiZ0GC z5ts}&-#BAv+nPmt_o?zBXOP~Q6fX4gvvFWd0Z1?pseyT0=@zxymOA;i0KTamql@Y~$)mnr#aPj}L zwF*wVhpM0BT!EQx6!D_U>u5{-sZ+jyj z-S+iSm2iF$sr%?Zvj70dvEnPIMHkGKNk8h!*SLnrr2^V>B^(N%J`VdFWVrt$;yb8* zNv()~0TnWL`74Rv+x##GWR^_V)9}Z4U1qdMASZVNuX2wEdzf@jKdlsOL69E5m=n|s z=8D5}E6AK?Ybx?1?+t+e_=RL$_tAYC@Dj!|s|0!DNZLUBuB_^}X|8R!Ei3>vE_-i6 zj(zSi0;?IjMdWobYUQH2(#rrGN`HHUu3UXbI7(`%G?#1l!i;k zCA7MshFRC2q?jYdu8dKebPcYBe2{HZ@(1TJssUQ29lb2{RbzlysW>n_zi`!lCtq(%K3UPWqmR<)7(A(q`tyhi!wuqt&A zh@&^VI)z%u#h22L;w47(6@sf8A3s~Bb^!7DeID6ZH(g8m5wjkXk+J4oG8CPt6QwLB zwYIVa$WV-oz_V@m(;yt)ksu9G(9A6Mo+Sb@Dmq0cY@c2|JZNw;2&bHM*xUWw`#apU z5U<|NGZ=o_8~E9O_m@vKe8UNKDNV0m~Balabu&$NAbFSsUD%t{$s@ zKNGIxnbsTkaut;%z0El)Put+FZ=#m5%z>oe1-mB19#p${Q*a|#XG`kf2ik{(4phIc zj&|zg&W$MmS2+bua;&gPFKCXVGnF8)?owEdy0U;c4{%&v^kZ-R=N}j@jT43N?z4rhGfdYbqYSCOGtWPB=ZFTOUViSS7aLpq*Nf#sZm?Y zoeRqZSzq(dHKXmZ(puZ%X`11mbB2YJQX9XVk19m>2m48Qd~8el7fEU|x17PG^lhkY zV@fP=>Z(aPmLV2g7O5(YH?(pZ{b6Gzd-p4s4_e;*#o0qW(Vo31cp>_{GPP zkS<1otFTT+?N2!LklRp z6r7A6>Sy%mW=M-10;z-7lx z>(y(d%Emrl-2Kn=ssr#qnl#B$cdDdVZR8Bpsg%a8G4v1qGGOpYRN#IAHAY8uB z^UeJ=IWOZ_9ST|zLHWZPwxoQOcVNe7+v@xZcS;w`pjFzPgzEPk~rEbHF@AtSb(iC&~CtL;J5w|!L_`TB8Eb_l7d^13XZ8tnr{B&2!%VKY{+Fr}obqcbCh3q{5_?$9`-0{d~ z3pZma$v>r7mRgZN!fF^$l?T|8AWAGTrh9l?ZqBrP<3~iE;(ipAXpcXZUDg!Dt~A>S zfb0NqZydGp&g(u_+l4t|ysLbBfyQa2%;^5?wRD#{6n^*^fpx6iK{$C>tLMZ>qvuew z)+)i&S%-*kyqRE)K0YF9!>Ir=v#2mzSX`6&tYLh`>upI;5!?J^aJ>9+r+I<0w0Z)kjz&aAMXfBufZ-kvZ_>PJUU0*SEf)Pja*lV^(p~ zmJJW0NR!`z8XvK_!l@ytV}av@#4*PIG_=g2rJi@pP_qmz$tu~CHs`cM+?z&umGqKg?U>G@ zJoaKFoKxH=vToM(E(Rx4K1{T4-w(GyGCU`k(|p)EBo;X5sh#?~D%%-%V!*Lw;X2_V zFOU7z|KnyyQK9fUgl`%FFe877`YVhf04 zI71@Y!tY_f$S&R`Tn{_%|xvFFZQVLS^tT z+Bya|0_nM;ApLxJ2}c;Ci{|zdC)jAqH?Z{DTN@lo`3e{EIHJhcvg4LSY zHjFzzV+z9?+A5dXMZ!mw;&bG9MjrS?_C2>Q@Q$@aBN)Cp6M2&@tkJDgR8@N>Iu3A% z(*@DU@jOg~_>OSe77?lCbDM?pLXf)`qLcl6I`n!J85femJAFPv%^+;>GYzmUmhgpO zsviDxD zt(8z8`T`!#=KczSBg3~bs;>@reSx~#^w0+R99`wLNPGwO)lmqE%_H*I7}B@j@i~)( z0WwW0R@02|KoD#hs7V6uqo2`3;q+imO_p7^(%SH1nl7hUg#=1 z`HoLOjr6WdmXEAD3QA6K!zWz1EL$po+_;csh8pWp)m3DL2r=qyIPIBi!)3_T9!7pi z(&Cku0=H{^`QG(}5b~L{`*HMOzp%Kl9=X+y{gB9nng*WXm)sN z_>*mnHAyF@BnvP`sg*qSH?2jXOnLkM@i1{5;LV3+4iru55@}>nk`=l2`&H!4E?2}? zg1Y-wHq3siADyCVjJtOs21~O9+L7z>#3J_ht4DO0|FW88&ajCe++?WGkWBuF;#g z5moL2TrAoQ^5x$=jUhnL#~T8a{m_kCB0oDYoB$m(Xro95g004g=+8MYDMXl80ee3G z4X;M{t*2naIAAMFQr8f?S@WY>kbo_>`ManaD4zX}+>)pWpH46Mlz z{(^u!^oXPbuw69`=fg2!c@L9ak$UkQ`wXF7Ef@0!GBfBmIMs}jqYY{`8L0k^;M*RP zxgJ_$h4zrdZ)|hoyhod77wBn4=+Qcst&!KWsuKRwDK+F>QdpaT#a~duP1f=?MFxqD|S`Kq5fZb5zMGDiW9>EtLPCgEIK~D6|;%&}j zSkgvV>^IkB0zr7MJ;uhygrP+EH{L}dJ-P79E7NgQE?Yi6cT5S;9W3U;-5*`9!FjRh z%svtNAXPg#oIp4tb3kgc$901A<2VUxLie(swt97UC_=53k;Q24f;0fRyZaov4`8Hr zt^~kkzqSDYE>{7Pw$^7!+7`$vJA$%KM<6BLfn_Rp-fJ%+wHX zk)%TXv6HfRb){fCKi==FHFu8?fp-Q@YuM(KHNV4XLZI5=E|DaHtw}CD_8>q2)p02` zS@EI;2fEHAErkq6!<}rsPn_3bEiJIDl(46KYrt)fXF-rqMdCf;cX*}lP7b~LZf5&A zp80LdMc&qGu1q+E9e=Z$Yi;=79?){vbE30Kn(F$tR91{@LFo-EQt>VrVc3cxoYWuZ ztX@Udq}W$Xw7g2EnUDLYaG9A_(|G0;In9z`tF7#BgZBdc78q+qJnJK8OYsaEJL?Wa zMUUHBS*j0CG&_S-aHWGV%j~OfX>ErX7eCMv z5Q^%bVE$Y~>h5RSnDDmygssq0fxFW z=Lko6ibJLR)G4)N3)T{+d^I|R9kd?Jxl)be#0&5aj=^eF{}G^5;yFYh$kS_++-jwB z*KN)@Y!_S8jLrzW_d8cGHJ0O*gmiL!*?V458D9*~l(`4Nv)p0gPN;vs!wnR{Nl>&9 z?F4+OJo^?xxZ|T`oL@2dTP{%nC~q=vYAp?ZY&dyrrtII%mhYvs;80zz0cF%_((A>b z4xH!xMZ*cXEB=Im#-50jHM~PeOGp48X(V$1G8KFs?N-XsF>o5qMf>TChx`fE1IQui zVK2!}2Q)csgO5Dl2~6#b ziQ}7G$dUJXRgAOTJ49w*A!5oOI#PtkZw>C6^HsqC_>X4;gA`STj$Tc%Gx2`{LV}Wz zmFwrYG-FUZ1U?;qt)Xrax&2gXB0c+Bt|1{h0hOYBey?c-IR@U5GS8_>UMDylt`>6) zXV$d~T@iL~DQ_RZk}Y!-%B2iXg{eUD3-GZ~b=L7`kqmmaFs#gO@B%?5NKq2vx`Z1v z%c@v*siX4^3l4WJL$8Y4V^DzUYDKSw3b}Pks~1iw%Q7cOdGeiJ7(s%5S+em3HJ$A& z!;IjDQv`V=VhFwwEbUrY9saCMmkYc_E2Sbw&w5irQ}24Tn&fnoCAmu1oUd{(F1#XI zQ9;*agkB1H%j zyG4$|PMH?fHvLOG*2yvUBqi!=9I`WmG%9^5a?02KxYxwaQye zRdHwi9(Z`>v*}_3ho;@#jtac($=%5+A||Ur9ib4Cz$wE*VvW0DxI#_t(Y59q8062J zMgCdG+Zv!G86j0-xFgDD3dljo4)vi}LFR4hMvH80qa?q0`W;oh=B+Anv0R58xD)0> zI*dp2vs@s+6l@>>Q8{%;V?s;4t?(^@(yNg=^wLiq>om z?kpzPN~}qSu`@{=t9_fBVf_9Zp+3Qgz zE!Iq2Qg!^paF1JxomH0c-dnIQYb4{mWf<~FP)f#?FK3Cs4&M11P^&6xzwFDAE}%qw z5kk%i0EXb|=GUr4xeAxc!b#F9_#0e26rzHAxHCak^u-3j!}ZeY|MqplC4w{rh+r!~ zpolKg{FBi1uQkvNQ+qmHxW1!?$V4B(@tX6?>BD1==a`CxC`B2gY7F!DV7fw%~(?`TF5AH)5IJH`qoU%t&7zRfnIJwRo z+zHC@N`J_L>;lKTwd9m$&z?8WOjsVhO?h0t3|WmGpDm0`$?{smqJnHno~liHiid)A z;cM5iws70@*kp{{iPUGuqWABc_V~ZwTx^$hq3jdl@u!p7%!bVKrIQ|itvB+b^6;S) zbt=WFKh!4O6E4ObBS+gaSvyVLPF??TJBM1GxOi5#g;yH3A`BeBE}eq8QY66TwGvWo zLPc_~Jj)o247>(hHAwAG?1O^mn*%>qc=VFW(}6kZcY|7+E)+NBQH31Aala4qtcH1> zD>>!n7*a*(p||SQa<6nsnOyd?RG^dVQ`*(^RG~pfXFQO9pNqG>*HJ;}Yd`(hOJN}) zAz(Etx$2nJ+S5dDJ?!?3Kbhk(lyj9rxautSQ?{2Xk}7#FAvipqQil4+Q?MI&HAd-3 zBx6J*HIQkE%T!vUr;;B$qK_S43^gx=eJT+&#NUIBHNHyl^m2mcyF_3Y1Gm7|AT5Y_ ztA-XAEzj_M&{eZIj&9_w8X=a{sP#Pq>XM8KgQd7pTVa94pg{0TzC*E~K-*zsmD1`V zvQ!G`2mV61O?oc?T8CSp2q!9@-fEn4NTx>>?SaF^fB%B^-`F;Q?g49b5QuVz$OVPN znUbdh^Sd%c`X9wj>)1_iVQuMJA(q4fyzSb=go;4BIO2hNOz>xiHhh9PqBy=MZc&+s>a1Ug&Amq?4sX2 zWxfJYa{fL(`NQPBq3&riqWV108Q)`-6`8Ow@El|O#$YWv9v`h}x|P?!b)c4*b`pUV zBt2-*Nxm*%3mAZro)3n;Fj6j3YSPC8-V)ITj*mP}L0O9ilns$fPN{HhHy7;5clcHc z(uK$#{bZm;3sNLt2&s!;>)zE6c}-mhFo~&PqX_^fEDVI%S)YR&On4vhagO}>5&Zo` zvboGFO9P5M3b|taj+goVeH)JpgL|rp2#S(7p0t06fD^IEd3yJmZV_>9P0LZ4k5gth zI@xytr$pDh_A&->Mm8}~{eUIlje}>!b&8_VW*yJCemW1fIyDn{@Xqc6$ufB^JqI$S zeIngDG9M8##mU?>Hx&cO6%GJTrUtAD_3qPOSm2((D zxCW351f7{K2R*n>d*SP|pL=M`a1i)t)maYx%-KODHh*%B4-0Ali!ya>LHG+0apPq6 z0(}ZXTl0U{$Sd8VcMZTX!y}HFaUFoIa$1Bar8?&(f`}ABMSPq^T*5t^xe(h~acn+D zcxFr;(WchX(xwfhS!6QEps_N0eNll3Pow49=>I=k+JoK1 z)51F1)6Ol>b5DW2W=F6QMLa&a17<*3K<7AP)YZSx&?V~y)1R=e^oVsFt-t^)D#%fK z%2h_#06;b28+OmL5(LwU^88a6``K5$DUGjGGdj_RGtN8kyp7O2ua6F** zXkPNmR$`U5h%Dj-O=j#&=r!dRz52qnK+OlWQt*H6mKlmLW#(prUbqPQS&1|r+I*H4 z|6(irpBh8XgqcuPFyn~-%=b|LPmRr(Fb}mqnRa#Q7(-hJdAKRVTNP7)a=tx^an1sC zO2l+Q6-vYmgS)&^-;>owk%N7aoPQd9M2IST(%uF>OGn0+p z-X|H@5dayosdEvr`~WgWz%6UR-B}Or&fo;n5WkjTawQPtjQ1ap(#H=d*GKKgPrMM; zpa=3O3Lw^hX{f!-cfs3)qY?imbiZiQ%s%`6UIUP7x3-@yIi}hi@^s}Az=h{)C@P-f zgWRt{Ge(@z^Jr$FKpLatsk4y-1hXg6FByRWEV6{!#-#$_p%v8uF)7bk>slL7P*%nA zo){hAdUj(S1j2r{IqjYdgv@`w+WQQqu?nWKi#Ls8C5!Ay#0!-52S@MxH*`THRD@iT zRJNP02KhTf1l~OXUybS=zM&`VWQw}`^N@5;2kuPw_sMe07_J-N#PGwr2teIF4oVNi z;#{IQkH0wn9BYMq5C5RyQ=0+WXE>b4Fc)XVzOi4+5P~^89h$kBrbCFAO@_!!Q-DsVnlK@wnYu17F-!Gsb6tz-z?r{rkpW$o~c{fck z`M*sh{SQI!12T6JR-c(J&_O5S_cc1&asQc5IO)nJ3pf7{^~NzDR$ zAfI|k5T#DcJ?=Da`;aTPh#cgNd-~w~8#-MHFvGd-+ZRKby*u8Zo+IQw#zu8qS<0ld zto!=7Jc ztoBxnHa>V}CHvL6LL*pu+e$%s;FOOB+M>b++(>itsXmY})M+4&3^M%>s|9VeXta4^ zyLhA2C1e3NN@u$V_W={G8v^3%mOLY~Zr6neky$*~{OW6qy#T>0q?p04c;E{}H_3OQ zW-<8cF6Lhc)9TEGxJ&%+MM=?j&VRuF=f%=NLHolaf&aKi^L;FZF!-k{SIYNpX-YE2 z_wi|!;LreBk6Ilh9?;6s9p0sC)|lY0SAS_oWq@4mCEW3A5F>(J^mG!5Pl)PgkV?pvG+xzVhG*+ZzU=k3zPhZOnHn`dpo7$$2w9! zm@TF6lV{l_S|tV>%TA+3=puq2-mMa}7`VCdSjYI-pEu6)f7ak<@rA5y*tm%yR1!7g z`#%KP8JnQa;m!G+Ri^8C;u{;xly9heJTf+0G)9MXZg%50m%r=*h;4*ZSIQr}^;4jY zFB9#EoTU-$5o1-;Z6c#)g1a}I9IZRY{n9iM_t8w6upilR{Mk;La1pn0l+yHnF1YO@ zwp8E|cLUC_Vu(e_T`KT}``=MHS_flR&s2u*9M+>ry6G8a2v8h3G}T(}!>EgS4r|+< zkCPlPNF+cB0p*9Oi(D3MdAdA(M8S#{_79Il%e~R>H`deXYMo~gjxJ(w`Dm)R%W2Xw zOw^vF^Fku9V6w);_KI|VhleuDHS-%cai<=7{P#(U8|1nM^H8mbu=<3GxkW=>j&T0S zqrw@?5nj!h*`0;V*P$CJ_V|KsZ&kkwydKksiJK-VppZQU8siiqFVc=wE5YF9^H+TL^=GnGQzircwM7sH@F!W zye}gHMgsC+f)q`Wu1Sh0{#I%D_CvQwvlyOe=^9YY+vmcw5@&9E@yT(Jixqv1pyLuz zLHg0i70)|)F9E#C5x(s`^AoT~qB}$`Dy0`L3h?4i5uhy3md}*eI=Ddr0P(fH@ECU& zKD$1}ZN=>cniw3vblcBoR|SFKm9EM)i^IgTfZn6e5a(T4k*99&7nZ2v|90a}YkNBG z+2xrX!NP0_t0yo~W7;jf)ND0T&naiNE z8r(IgQ{ff~BI*@hE_zw!`u88$MLzSnm#E6`AuQek#uP{*^@-qmB}HnT{u>^D?jB*1h(C~` zb%J79S;CE}!A`orP*6baS=aaETk^m00s|~3$$M>+eV=fv%4NTnjj&Iv?U~RvsMk=8 zuvfe&17v(VXcJf4>?Etgo3U4F3r{Uy!L>$Axdi@JMeyeVfIqL;=A4#8irO6Kf;n-& z%IP?^^k+%1y(-1a`3JTqTMT)2LdT(I<0{!(wWa4cXUD!pZ56R$S+^;OpWO{YBty-2* zat(|?*nmpH0lvnfxqKt|>R1q6hF-0CmTd*#eRGogEna$^ci`;$Q0wi+@cMMRhf~@j z;Wq>lHI$$RYU zQW3C;;ldwP@rKBI2Og$L|H4E4V zh~s*2=){wgEN%G*4f<@D>VmI@&{6F9_v4>na;t@q3@`9Sj#043$0p5>-fHB$MxHE4 zi3AQZ*sf5}JhA~Ue#b#eT?4DyJf2f79!dJJa=;b1p1cgZQ*=+qm6Dfq#lW2_Gw?w z=EYT$ah<`jiM1=nYT-SgLp)<%Xr`Ld&oc*=>xC8+_Gw`Ee^VJO`_Hnh_0b=+0*Vhk)mNlG3u*npyiCLEg^0 zDEY}7Tm5E+;XaG%(Pgf0xg^ax?>e|mBV-|0E}7QN+5D++9j0{$$158ckwFy$!{z(& z?7Qy1JX%Eu^GOas#7-^;4m7vm-ebR+##{I6O3Cm%c8BqFz^UoFB!8Hum9-HE7Lk=?SY+-7Q2krIh#^veZNAb^IUb>zYsjJ8PIBagK zE~jg&G%>iWXaa?kZP~1{lE=dM5{Iq+G-JwtQ+jmIB`P*4$Jkl-<^}gT+oQJe%;{KH z!W20hVlT;!euoKI+n$G>CrJB3R&zyjICG}S2f?iT#5?rAQPR#%`)WuhcX`5m$-1&g z4i*@El51~*wi*H*UJG2>n?zFAYk9wp{8T(RPb}X)J?{g&^;xP!ZaP*JWSVlKwl_au z#NB*1K)xFK`=3Y3MQ`Z%Ly&Z(E>@D&Sw^(;6L)=fo>%JbYF{kw_W2ZKmwvub&8Jv$ z$hPEMb4P;kOG+uQFGy{A3yf{gV;ksOl2T~FJKC?J-Rh*zd=`r@G$;7(B!_!=;FRz7 zG;iI6;0(vUP%go6x?8y76%T7t9+8 z3D11`_mz7zS}s8O zJ?DFycp2%;unIZVgMs)(~=`jcDt);NpA>Q53 zf!1rU4u^a#x+VaqG8e~-QpHxYIQJwDNc1J-&^HTzu_pOr8@meQ7tdo}Z}}0XQnhd% zCOu%}Pc$2UNcel^Xt9wb=L^~_&s`|WeD~eaM)gx&hgCDAH&j}aoMSPIa> zh{sJ7ooSUK+Kyq}nW6JpIMPw!!=f2|Zfv}MdzXYleCcS*{UUc_h~qgHzOpnG6=OSr zZN_hL#xlwO3QsUNQax)RRdo;e>?B8RHHOtrcQOdTBSSrx5s+^s$%-W&&CQ8^R9n2` z*e&-bn(6R>j**`A2nNXc2IQZ6FL#IgulAX?X6ze+zce8$q zV$$bJvB){$A&T{@bUUMeOR3P)Fg3Up?O1w&Zh9R<85Q3w+UYV@4<{epbTg*qdR0I2 zv}_?{CUYTbM&9OZIoA579wMUDl0z7F77R?|MQUkv>p}ygiowoXVcuuWNLjRN<;-`j z-}G$u*W4IS+kN9Y`}85<72v$@zjnmnzF|Ai*e+vff1{1@%4CDBpX=25KZ^OR=uO55Y^l zC+b`YyM?@9tzK0UE?075cT;35KQ1meFWK;SjPS|2TS+b*jxSfV#Ts`pug8xcbqlLH z&lI}jKcS8B@Tx2XCZr zmN|S-^7#dg^JP_=KIv0X&jRXK0SY4ks0n*^miAwB|AQnG4{~5f$w(n-98R z>Q3L8h~5_8$j~i0HH7+Zy{}3&7D$^cTEgR8tSFU_57oz8U#%*LeLa;vTe6 zRV%uNYqGMK7c`p}GcLziMa$LdWu%0@9|^F3He`X}?3KlYJIYp%eaN~W9lfZMk*4{0 zUh_c4=LutAo!ofLTBGV#E`42(<-)Aduo!yETQy2IlK?f99)rczV;3BV%zL-forX8~ zlxGVHu%-6XYVq;XH(d&xWa$~db;|jgPk7$k&#^mDC-;v*{*uh}49TNGw}VR`eOYWs zxRUufu!`ZYw2$Zor3CaBBz4gGMqL;8EaQMnO=s|O@k7+pkAOq4zTQm8rSO~Tzg?sY z##SGXyd@2LdzR_k91iUoyx=ridsBmTva2YeTnip$2}?JGr`W&fbwuJYh3T=tE=k8N zB55#=d3m_stAV#%d>@w6mfqZrNeF#U4!BG}==35B=6~}$)U#{YMKk3+YV7%AO)>wL zOO}DXS)#wOZjuY@V+U3P?PusA&wMO~G-BBiDgKk@?3Zp#7JW8Uwt$EiHmLLaBgPr3 zWVMz!4R7nOjt2=eGBO6csxdo*;<&n`#QB4Mbk_Tz(27% zB6t5cZ;YEAD8o=$bSy(Co@Ax_WzN zD#^vE6W9S$P48B+&(X_;>C6{+{|Tw|iu&ks`}1F3?!wf6{iCY;pZ;+)Nt5*f9pyCq zXFM6sb8{yP*wzszn_K*)abAJOC-OJX`b^EQ^R^}cwWs*& z7X|a$21R5!>r_0?g=@(@W_&mU#|~jjjO4L|(oD@f9jcncnSL+)wYtQoqta_xubt^e zT5=Y-iQZMmN$m#IYYrQ~xN{`noiOU)1?!8kH2US(j&c92mc6Mi7$$3@$1pm)Kpog! zg+-X#OzB1xz5k21FOP?EedAUgb&eLN(;$R0m61@gB+F0+Z5Twdwg}nEz7wt1Gi9k{ zX{;eal66vtMA@V4*_Z75VCKEXSc`t|AMfX_ziMWl`@XOHTEExw#MPy0G$q1-*1fMB zj>Ni@3;xpgoHQuFWNAu*Hc;E#|%IyRedsWi-|+C0{@ z4Q;xA78g@n6khx)=NXQyH8!fTzb)pO)TXwCioPE)W2r3xFfiE7s15yDwuz~E>hB=y z-APUO_}+?~LX!UF+5d2|B+JCHttnk0xj5Z)gH*rqO6%?oC&>0K8cCB={Cg_M^raky z2F`kJgRv?sb*(s`I$_wAY4v7eU~3VvP>Gx|MC^IcN#8c+sL0YC zd=>_^y5_sSQ>uRRaE##I53P`-6PT2);eywn)wt|Z6O)QF!DE&1nb}!&P%r$}OnRMg zXx~WKh04jt6Mrf;wSF)CAvDfpFl2t}G&-4wjJ5I^*kou^UFKR~?I49E{(&32)l9hY za3XSxTI%?oc64as(BTmd&VNboM8EtvJff={=)&)D&4#9H6EC?#2yV6haoP*#Qdi>8 zggNR9sY#H$Pv57>Audcfx_IpDT_aM2KI=5jxvNq{JZ6f&scQQiDFI%!MyuoWxoFqkeUV?*LJ zx4*0F#mhW?=Dij4T=F9v$I2$#Z;^IV za%ZkOv)ndUtW2$P=ujf7&!nRaoK^P^kIXh_>JmRFZ*nASHSBDg;bQ1!Rr%r-<@C}n zn$NHqy-%h|9bRU&hphE*jLzL$tbAe@YF2lU3-PDNY*d}<>9BO?w>A6^NBx{zgJ-#* zg%4cQG3Re+9n#iVBhucRu3yfR2{V7WCnzFEl6H4Zu%nzg9*}j*U#nRu5*WVveb4mk zeSOBS*Joy&JlUhf&0;E-?D z^cpkxn^{8T2RYBFd&hdkAMO4wPgAaWsv?_+5Bd5S%$x)UXIMW~L~(e1{nQ!7lGJ23 zzRu%&DWkCCfZOZsf|9}x`_sQ&CU1LQ(A)bs=4uT&R<(63HTNKW&d9*De9*=>2_$Fd z?3oyksQ4k1(3lNb19A#MckUmoElo6cQqWbWr+MNmziznlaBaG?kl8?jhm-Si>pn~pP^PDnk`7|!9CBkAXSxWeyjHJ7hY z#Rtu#cywit>wmrDM3?%mQ4njiFe%wwM4>mgi-UER09eXbTSsU1>Bh(pg9+F6=a8)} z>N}E}K9cX6l8l;Ei5x6W#=)0lyp2gEjE zUt9&@wi5i-t)lEu_@?J1R>82{Q6r_NIySa8C=Izvo| zaORr!HJ#EP+ZHUO*Nxtmd0gT(JwZ%SeZt2YSTW}CeB?r3q~?>*1K?>jwf25gZY0PX zc_|YmP5m9RRFSjan&fAX7d4@Iqtc(JKh#M~e_mWcP!!72Eq$nreo!%0F#hbr$rlf< zWhcQTi%2y2-zuJlSA^U~wCRUyR(MsS11Zxr5O(Y)?H6>4W7q3=Zsm}&4d-)6=~1H+ z8%hHzJg#bw({d{|p$9b~UJ?>X`r@n8M2L|x*L9T*G8)JwY~^jbG-|~y@wVDS*I6Z% z^a-Z25Q&yq@=@I>7-gbuw%VyA@<#5lGtcXyX8H`BkDqdtb^S1|97@n7rpF8fz1Dx8 zFoEbQoqjtc%AGSCByW1XxN%QFhq*34nhydUoIiR{k+6OQf=(FC{vJT`lggXqL-?s< zRTFXp(Rxk^!8IPTAvGQbu52%CTsWi9F`Hf)b$c|tGC$Rw60^@Tq`X%oxBPsNkTFNa zIGc7-l6PdcbHxT%37)X%nlBX|9SS*8r(8Q!C^O*`G^Ul*pw+=rdpr#O?a*YM!Tuwr zTi^u~B}SaKv=C6Y_3`JU46haFUKbJCJ)VmS=%leogR!KQfxBQ}ck6KYiJRz0>_MMQ zd!frz@@%KU7p^Qk+SVCf@v2>Y7F9^6(0EQclP~9mRqkPw9n4GqGN|ydLLb`s;HhpK zi7xIuJrD2ShN2cXvuc~b2vbP>XWS=Q&;*zj4JE)!5o95S4IYpp!4wG}x zcIFAA(r+sC8*Z%+aeX71;Zu6L+rgRa25zC`uqwRpNk-f>X4o}})R)QE>r8>04jpf@ z3zj@uy&-?}W@^c_ha5EHS|4|n)wD>N^b?;o6i^rGQW`t}9i!^c-(OHAA{EAnaFq0C zH>GM6100_Flb-TxH~J}s4;U2U>(18nHnnv-fq!7M`Rk^rQI{SEUgvsiW?0C({75Uw zZL&t$;d)o+Y+jrWQNMe`l-dmIrPMCg3)UgI+pAfvP+uu;)l%#@7Exo)2(ekKv9r#w)^ZrYOd{n1P6FD&3nFwr#SAps2v!PZh_+UR56&B zKQ+M`+4Qo$ROuy!4rD_CD0?vJSqzo_shuNxq|J97663KF_3dbIRJY_E{qJZ?E6FA=4yB zm00OFI#k|{hTxz=KjhB(9CNZtSp9Jh{;jt@>x97aZrdFOA4wX1nHrVTu;~?op^+C@ zwmCT#op~-hOGisAx$B+#FfMA4n$bqGRY)Ru{P~km#w} z%Iu&>5I;@dB{sF~55)o9>aj7&-ebBxqXwk2r<&>pwYhU#HPS18j7w<8goRth`nwdr zB$%a7oK6Z3Rjjlo^IG5YcM&&99XMl{{rOdaHOZwbo;GtUNgXDV=SWt=s{fz=NEM|< z>FgQP5;;3R^p`sN7l8>#8r_u`TI}qLC{ri+$2oVO){S%S)oc z@;@$t8WqI39O4fHbnuS}-Ratzx&L@kYun^(?AVv4-iPFvlJ9Wv#K);;Ho114;5cJ?JIRr(S)3XKNa~H~UMpVpxA5O>E}3RD+rVUp7t6v}*yH>%)tL&Faqw3mPf+ z74ak`z$oEwytGRC!NR6-+7=)8o$MBL>v3$u$vwC{V-M)zzDaUl!=|TN);XjhLc7kG zPEt{tmtJ1u=5fb9&#n2M+Icp8D9rO`=l{mY;5bpSe_oj&^SO-2vFMW(z zS0s&p=XzLsBdmQJK0dMdu@WMD7lIbn-U=UYp4-~O$Kfa`Q4}4l%?ckgE$*E1u@u(L zLh1kQ!Q2JJQN#Zhogynwi;$cnNTVrM=tSOZDR50Pef^J`2~q68$1LvuCJS=J_jFd4VjKKO$%&UTN&*@FyEZ zsS~hiJHyeP?>*qzPG4 zhi?|PB1J>eM~Yd_qGI1vYI%G?TK{V5jIV~ zt63yk#I*L6pyV0sp;KbA!#BrkWR8vH$&c%NNx3c@RyPV_PVOP>j z!S-NE@qIlF8o5*$bE!|>@e8P=J%I)9wKG@h7HZ=P&Kh-uw`E+A;6LbEv z$|P=I@m}Joh}l~Ik4#O+y@!J=6Zyu84S(h1Vm=!=qW44+1dRH5^QAw8o%`Y3c0XT3 zy|LORq`ov`sw23P=!@6$mt^@Uvg@_4RZksmCOOHkNQz&xajzna*^tv-odYh)m%~Zj+qXVJ$J{Z=Ev;&CXYDs4$-0p~ z2!ZO-iU>cT}ZZ@>gmWj&anT?qeFks>o~!J+uU>o&2Qz2m^i_*{9AWN z?#MfMr<{)DqxbA`pq(lmoGCoNLemB@#MdyI&wd-dnMYLbx4bq~%W*bdd|-0Q`%=Ze zbfG4OIQ(oaV&U=CwxAXS#U8jkp1_1Mm(w`$G`zOIqP>?FeOSO>PHy~Dis_X@WM+FA znVM}oeH*<(rxWBI_>26I4N}WRO(!r>OC{TgaAc#Wd$E~zefB0Pbi|yJ7A&~KO_$ls zgiC|hFKBNU-8ppIBxn1!>_RbQNbTMf!Q!eyy->!f zvu(ro0!^LiyPXrfyK=M;Q)+F`aAwODv0G1{(9rNrBvI&mQ~#wb{Z0)r#RHAgBhT|6 zw7Yn0DtVM@u8HC7hg2e%}fvXy{h!AlD>dgfv*Maz&#Q1%>)0LPF zn7mTOoVY39xD(js)6;3(^A_^wUmeXPl|5J!ZwB>SpNfF7n733s>ljrizgWxp zs`}pH7|6u`ZR93iikuzd#jNi6J67sBMDEwm_X@_`EIIh*@?RL~02G~*5kZ)C-qrkx zpv}Fn>^FNF^a(`SwTyGAhH_zqXB#7BvEg1U{Nbor#F~W4mAo-&?{m|k-99U%>e2ex zp6p8lDqdlsPWs8dgq%}Y#3Yy&U8>CdqzMMo#x8b(ljJHsQPA~>xWsI8U1$s z+7>rv5aIil=Ag@9pSh153Yq4}_p`_El;ce9hOhdcHaDhzb#1LVI2ITda##uR)Z%fI zP2!P7U;Ni+8WF}Yc}D8><=G6|;K~n={mZ|q_*EHn7YXJ2MQdT5M$0Tc={dBVLsRSg z;fUTmOoB}e6w+0=n9|M1A3tg1aLEs!pP&yZk-Y~L`9cZPCj&a?Owh$9qkOj0YSiB^ z#4SJZno`*1RJ@b{1I@W-zY%ezFm#CR4$Qi1(!b*icjC#5 z%gM(zTM@Kjvwc&aHE#}6A4!SkemVAu7+tVmCYEgp9WSJ&etS8c%|{ZMQFOU>RkYNH z{`M{iPhRJuCWkq`Rv-Tz32w${O8^PR3}2@yv4O%|^a_5~eq&QpRqIJY@?fbM@>7CVYg zXUyYyeh!bC$iO-x>l$jU1e15pwq8>_36weUZVbc3!I8TA?p$k=Y#ftk*6ZbtDLxuO z18c0QXep<#pU%m*1ihTN10sB;H5A!VCfbn{d~+K5*o*VoPw{gPT|<=NtLpWyuYkpE z@QcABAjDsrU7vcn7n^G;iyPE%`{0|$(4lGvpP}6TT0@;szHO$hi^#sH>$!e%ZF}6J zjX1~`3as^1=b#4Q5^$lpYpJpud~|#{`p(!vG+H$0jcxvu4hbn&s*6lf7c10Eeg^7| z3?`2co<}~7r9rN(Okt;9<~my9@o-@`2$f#bC^|j&-xy!GV2ZDXpf#@@@NUTsK+q1t zWy}L6E7{f>d&m`={EhD+xDLUtr?4~}Yj2}IhP~FL6ZEgL=lVxLuc)O6W5q*HH(>HI zY8uWt_1ktb4NQOeR5Vf@c9uVsGD(MMI|_4@8$ z@@PfsD#o{Bo3W1~IPlYn~Dwcm4nINfhIpq7T9JtOgls>g3`D2@@oLl{%$i zJhCtlw*lEHljx_#`gr@4QCkS^gIv$(t3pW}$CW9Q$lwLcYiG%`yX=|LC&NvW^BF%o z%JK2DDdimVa;HR=NyI!NzF7dy`VX9S*ji-XPO|ihdAjr2S+|wQ6QdHn>;K8fe5{Ey z=Ikc*a+dn3_Icn&*@w^gT7hMw0#qj+i&}QPC8`;0k91Y{l^Flpp&Sw{Za@6Cl|cNG zBy5g>Tm$M&%j*fI)1AjW21OGGF4>ZmWSRDd7Z}Dv zV~&7WVtS!7;A2k_-BtT=qi>9@PMp+XJ$d`B&-Q?g;LQF-Wkf?cTDR)keZGU{A?~OU z(?h!gHXxn1WG0p1^}x9S)UxJav*V^^ZhqOG&Zj}v`?djRg@%bjc~=p+SdngPx;d~=+JsUv2D*enXoYxMTs^`L?$18mgiwJvEw+A-yp##j%9`%QmTbP;%YruQ}un{zCN{X)tm>mg5PiUR6r z%1NN;SBl++pb>&uH$W?{{g_*ihqB)Gkc(x{b!94A1@*}P6`pt6`yg~6u`mePv~$0a zefw~SDYvI^ctGDlZ=>yQ^UB2GsYTbbyTSt+{DgZqjIrS#;`Pd`H_HxZ?xd)v z{9=)VbY>X__6x$56!D#k`InuiTw5MUl-E-x`|DSZ()%6sjrSW$4ieZxIU)8zl-T)aqqG9!7>{oTBKF}jG8fs)Es_(SD|XFw#+bH3G|!6TxvIzyrI+a5I0-#&c%g8*Jqy zZ!d2q?cmJ3lgEa8RsXJVg!6>`g}uBr=^k zAB#G&)+>RnoKFXZAC^6XRdVl?k!1Up``AxTuxnuP2JF6#{8zj&ar9lq_&2QznPTjCy+2brrmdACJKU4GjjGTdmc9z z*;U;k-3v8QRnPisY@;l8GtfHja@817qnY&vAR;PzwJ`Gs(eLy|&-G_d{1Hz6zu3Z4 zcxH$^mgXqry=JI9wag*B$^W8PxkqWDH;dTVRKrB;sFEC z5X=Q76kobb;D*IK+M!%FrD6`K} zJ)q@r8nLuHb>N_6a!o?P+su@vWOXLnCNURbM)Nn$8p)NvK{jzjGw*+G$1p;m3FI>gTMO8@dg0%+WQCm*0i@ z9N8_IJw4?!5qku=O>F2+OK!XIlhgR%+>uUEAhipZ3W5f3hlOnBGBTIxu$&*LandJ? z=3f?eIa&MY8ma0_MX%)4lqP~a9p(DU*>CnipR2Rc2v`5sCP|xd2q`O75>4jWH*Pwd zrt0pTBe;SLm9^FrOs=A>ftTX7OO3av^`e}$C*K75Rn&Z=cjOQ<*K?ayLB9z?*`a5#Sas3fn?m9R#ukP~BQZ9yf(u0sKM8VUwx91GX&^Xo&>+j(jj)$8rdM+o8x7g7@+0b>?uY}%QT z#GV7ef4-4A1m9()YX+Zh)!`1SXNifV0Y4_-U&2!m@&74k!yLehgk!T$I$qEanIW<~ zWxpDjH&wyGw-ZI@au!Q-e(ktN7ewkjMSNSZ5G2}%1&dtgQLt6DWqa@1X~4&O}1O{3mt>Js?W>oX(2UWvJ% zzk}d-eCSqTD?!CnxbfnpxvgWT$9gULt!)`7 zd6(9Eq|XzNFU(x(`Qqb6ryigiYIAO=??epZZpihk*m_sj5$$zd2B49&PTPZ?^IaME zD)BPVmQIr342+vT8J;w#3p3~%^ZkRqs$K22$GWTTGZ4u zs>b`_S)TvXl=KRW^ni2XNdNRCmL}Y4Rgh&cilT>mTxK($KF2pZ0?05h-Tvtk11#_W z7O>!L2Poqx#iB1&K}e;QWIODEZ%}~k)q~~*gW4<58h6|0K{ysMrc~V^CN}1#ME~0fvNEq>YT6?cP?KeO2k7G1 z5+IPB_a%M;WtW&wp#4E0UXN{_uxs4>eqD8{x=OyGBp8O6t4WgnD3}9 zpkcTZCESpdqd`%0>>AXE+%F9SAf#)cO7mw=(#T3YpB7D=>3bIpxVRrXWfM-n+k}Jj z)@s+u=2R~(r-@rFe>I6dvvbO~uNdcI4DJ`?7D%2%8Bp0GDf6`bDkiwOBXL@dfj{*6 zDNuVca=D}!FV6vXk(l8#N57}EK-Nm6>D)Xly@m(Y91q5|*I`y{KvN^z@ZCoM!5%|3 zM88f?+8!_|s~0lOb8k(r3Z)Lo3>4y*b43v@)RQh(8X}tq0K_zr#e(-nNyq<|fHl45 zN-&BOXa)}Jg0gRoN7*^Gnc@ayxA}a!r^S#VYH{L;wg1^p8j?=Lq?)oMP@#i6`(fwF>cU2R(lvI9KujFeO} zsqO_N?Yd=_r@I6o~(b+kLd|n_GhfJMnUoi=yD@im5wd!U@k`K9W&R> z&PCZJdb;aVNU{Xpm(^kN=#dT{EX`=NK!uS9`>rXcy%Kln?tYN_^%PBkOte`cGH{52 z)eh12Qa%^ybookp;WRSSaL5u_dQY5Z7+)BS%=|jU!8z4x=9PWW{n|q9TXV;245%+4 z1J8>10)oZF7_ar%?bl6YRJNPG#Ix@|=A~!W#=0F89i))piWit(GZo)#p9MC(%XLvo zr7w%;bb7fzO*}XFtngN09u6%_+J(gpX5HHuOJK0S@yFP+`GT6sicG$!ZPqyuvog(X zN_K@HxDTl#^qdyz4Bvj{1rKbiY>4AbT@O9m#H~5Dc@Fy(?({=2+o$P zZP!+Lc81Jof4prUE?rJ?`_#27G4~paOK<}+l1%n;S1RQJOM>GA*p}NPS@bfohS1JF zcW+Czgi7YvmzXxUz_vzaqa4=+?&62YyD}C0=QP+dVjc^9u2Xi|^$k^FdSKsh4K`4Y zQLf+jpl8CpS9~MEg{jDf_r#)%xdS z(=-$3VnL5v@YKY?xtf=Rm8nobD5Kd~Hk@sGXfE!=FYwfZbLw(W?%;4fZd6B2R?xY8 zu@Gs_9%?6{DXI9)Hm;DO*@KZ@88jMtZ-rvO3J*>ZpvW*@o@?|?9kTmIr%caiI#d`H zb7q#jTY8?6c98PjOr0GscDdV8m5v*Bxx;u?L|s!MTi3R@TVUT@?d_KqI(4LMMVWeD8I! z(Ai=}Gf#6KL1yawCm*xR;rx|KX}lLwL@l;Bh(EokCEp8jDlmSsX|Bev-IFKL@P|yj zeP;+%`OOAv+csG5?(63l1I-nXUJvP|Y&*Y)o^=B{6*d_zpJDo!=9$L=Q51AB8QuW( zy7=$oM4n?i_25mcbCrN6{_IZbdMjtzs%!n_9JOxnl{Ok?*+5l8a*j>$s3qm~NP@Br z6PUbr_d*TqTM3z5Xck?2{YVu}fQ8Lixjh;qJpXBbQg{}Zg(Nz6S&x+#a}u@DVgmnU zzIH!#$0lQ6#rfwYPem&a9J%_*aHgI`%Fq@H@yHfE&trroCs+hcY|-6KhT1_D{iGBX z=wj__>&~~d4XFo3NT_RTi8MM=%U6?B{`imDn~$KNxfKc+=eC_Hd86e&*~xlG{b1AA z%<@^&C&?Y&M@kG$TH|-aGojWGKbBR(Qr)R@*|9IMyR$32vm;l(50i&P@|!>OKt4x( zePOT3WshmI#?i}={wZ$__=qzjUftVt(p4o=g~PASudmu4jT}lTOvcSP?_wwyLKO7G z;0la*%+!)}J_A(w*f=TYb@Jh4tL};pUMbh&7^Yot?Z2y^%@jXLs7o6~x4GNNVA>s=d{y*zKA%E&G5O z=fl}hMa(5^@2+YDt}OMeYWKj+BF>M^27eTEfC&PQ0>D?@gj zo8)}OAAv%`UcWF$tPbzg(nqFPKu$O_>)wr?xq}6!d1gEZV_AzBfHhWjOy|bJ` z?Ft}Sy`0@QQjyjbX&Z+HI&Li$3_8ur6`7cTvg2M5(YNZ_QQ|#vbN+|n56$r(LUfXB zk83}PS)HIK%I|H$s*4_eTAlh=KSP1eP-0#2uMF4@x=FrQz_yi78mw%SgH4W z9=w)umNst)$0c_s;@uwCAELOtY>@~2!6^bMhwu|A(e|35?Ktl3+BBJ(i@JR-RMbv# zZFn~gfkIH9^|j<%5Y~i&J9SQ-+mA?fmRP0(C3F$udxp~rxpCIGtv23rj}ByWr6{(g zZx4|Uj~I2`@%Zm@+JEJa-!804bwUa33egDX{&$q!%H}->rvt|ajIZymf6AT1yu|k@ z18@jo5LB$!3r4fo_CY~LyW_l>T;@m4TiUILojjfP+qJ)*_ZaLx{=o|=CnLC9^00;d zUN+IBOW%Kh4b_!Xf~a-wkvEf$`3YoOmk+nh);VR_)ff=I74j=&Wv2YN@lHSgs+eN* zLcJ>Fq$y!TjyB>4V>zoT^YxRFV_miZG(}&0$VBxgWb%`Hp{?2SS-l&V%IGO5sj6x3 ziBfK1sBgXeQpeum%M>Y6R%3AJeF?8xf=-Q-I`~4A=jQf^qiDvcfCj}2(v-otm24{$ z+IS%f-<({-z~{eZgK3$MX?6AiiKKPr!V&Y->dJw;HM+m_4`0x3o7a6 zA!!PZMRBbX?roe@$B_>&+iN$N3;*fo0|h^;SJZ zj@d%y`$))0CEz;)T+vw5TEd~RfUNeD1gq%r+L zA8}O?z4`^;3?7SmPe1Db!JvVsJKzfo!3Q`v?u~rV6Sh8pc0vXkqnuXwe%Bx_)&QvJ z2DpFo<%7-fB_8XQsCc}zITLM?SAuufS(5Uy)xx)F>SOYBx#_j5_XE_U0wXHX^iX$W_@i#z1Uq=Yu#NBPsXx9f zkO($Sy4C?Pxa}f=ej=;h%t8icj+_&-7C=tFZwT6R)Vo`t>CX_P8Y@!Xn8BiDLk??9#LCX z$w1-36NwM-ik7cl!Ll!usS3DRj znNm0{ZnUN2lH@{}r6*SCp1O%JNvIsMdfr5Qt+0}X4%;-QV)8VRD#(XO5lVT3NDj(E zzwJk1H;x&%8JRAl1`Rv$NCR1g`%iad@TG`9=vVKM)nSpISP0MN40~ESu=>ZKxq4iR zPFo=!mz!S4fNOICD&8Brp5y2-$C8b%B;BKF?Q)7gvcTI`|f z8LSQqkzQ1F%g9X)&84E;8pAgd2~n)F-?!t8kq*Aaob>nGBpI;S@J|(6yXcnNA*W(R z$Y#Z8xyb#(AQVx&>A1Do_+N5f=0*SUdy$>rlto^>If9=SI=ek{p%ZKlbco9l<)KrE zS}4{9<+sSa6#Y@+6Sb8f=*NLt>jGH+#$%zH7&sY~EGxj~}gr+XyOpZG~ z{1o*nmZga(n#A0%YiqPLDNHVut^2?C$S0YGIx-j!_9QRnbcfS26TL#DMAkBf6$+wY zR)GUW7yWrrk5jIIA~S$INa-+U?0(RgfWJBW4R@K-VI)&&=qBzxCl`iMcDP*~EXiHQFq8CT&LGy-a2xw=Ci|QfxZYt(D+z zF{>{z@3P*7u33C@9?le`BOb|`5G%JGDykqboWmAMBDUyGM}U@ktiGbGoND??AXK!w zD8t03U~Xj!0<{N_EUO!>60$p&9%0m%IbAt^5-Im7JGTN*Ic_-Zr}zPrT%LI?wnC18 zIf7)WK}dJSvK}&9kU#O+v3J2uA=+?r)y1#0OY~58(e*V6cM=eo9GeIXSu_EDqxWh`-0oJD# zc`R5@zpZ?kWqw<75Yw2; zVC9a;Jypc&f!9mXAg&=UIBOcs!h{40(_+z&Ciwn1)ikHnIKU23m99AKxBmWleuz zSz^QP!c<5>hQ**>Qh%|mDFjyOf@~z*Jb0jr1hHNs@H-Act1wTz)zkgW-28N}pDjFY zfffY@SZKwRleJ22mGe8a?11J_XZL2_^IL3Xa38`VviU9S+U^w2;zkSPb}ZZPy#&-( zHHaZ?M$x|*<9XoL2xrVWRWwylh(>P0jQbf%Iqa1d8qJDE(dga8zpeakpp%}9Itu+A z^R;_R8nbz(F!Nps=h6N6=XVZ)-Gn=b)?#tCKn7afqdgGZK#}YmBDPrDP4VD+C}PCA zLX2KABWM$Up%cYWM9Zoo4tj|nU*;{^9kVJ(11@87bp`lpS<;xI@W7iD=MxYEnUP4r z{?WRCD7~%RQ^`%IHP0K1wad~2Otht*KUgwA$ixm0=Rhr{x@;(iedC`?psF~V6BIG? z{7OBLYG+Q?ghyF_FQa!2K$_t80RFn%{iKRZFQ*vB=Ew1hk!1`>keq!2d;&&kUJ_Tz z|4zb!l?mNe$g~nI%9Mdz`o8v zLjlA5;TD^Tz#8fe$U8i2pj%Y`gI3K(>VRvf!qPv)R?2gWInm$zYr3)!&k0AZDTDN$lm1X-{)1R&3pe-9FS>m{9mc zo=6*$;Qa|gE5owfy3oJBiCSEd8MtT^IVRDTAz^W*ZO2kWq1`dP9;l)0MbN7K%8>x1 z4|utk0#nv=Oy!z&34|QJuC=%6-qFJ;+HER0MY-30XS~{qckSwq7Qs|lHZe}A$mHjB zA_u$-55({yb?@(D*8s$~iK*PNk8x3-hlOQA(L}-F)Zp}YVASWUHYfx(^xi(jK>EQ;r&*@Nh3mniZRITGxTwQuZ`?-ayerEnS3&P!)OHc@xf6RO2+_pyQS|%r z=`(&K1eUq1$*!2_dJMmQXZKgx- zdq60W#wm1X{?&csz)q5c^4%CYt7wb^{|DL3n~Rp!AVw<#)Q*k9FCy-f{2d@G9oDIm zKSw^_>u4t1wF%(_HZ`M}PZDI>-{TaVc^{zFYM5f5c={S6yCu4QY}WoagVicfCRs1f zy(27`ek*${sw`+T4SkJ{qv>U?OYq{L&zjA<(GhxCtsTPVw>{vk_E3&bBOZ%5gIP!W zj0prI&9EF_AWVWcPnKoVN)du39Bm>{WnV*sp{_N0=zA_fR$9ZxGg22n+#??&3FXMs z8S(+SEFbx=ii@P&xUyGg*JDw6v-=1#l*+P-l`A|xd|CGLWzMa;0VqbtggRVnwRA0D zHh7g2HfoS-KUoC;(?f!IRG&a3I>I_<8yy9aohyHRe18X*a7Djl1)GC+$C#zTC|p6lsMq<8udm9KQ$7-)N_aZ2qc%|p{g{_(>)#2 zfvM=_^Xqch7BN&-=t}?3+4riS!q9{pnZ7E4eA=~>*FQfIil>&U2B_Z^vi zE`3@it+JyLQk8V+S%k2N!&*M=vXp&_rSJ(5hIDf}q2l$ls)K#cGj-XL`;j%31J>#H zHcl766{3413sWVG5q7l`k(AOJ+s6)tbTyDyzwSfrA~FWo}Hq8JF#hO76%`;j6E-1keocd?2du$cU_!t|H`IZoce zyhcV)9=%hWW^!Eq0;#_61gD3+s*L{0A}-_!EY89T#1-Gv^biT?0(Fo_l|f_Q1S7>0 zX%R8dQMZ3B`9sTdy~-~(d8b@IZ(14ws8xYyX743Jvc98=F?)aAJU+P6g zI8!+Bo9WCs?2kw8Q1qz}(6_j9s7_{BpCQEg@PqolyBDxoX^F(_*~XS{Sqetrln>`9 z@=~g4Z%nr^e7XA3^HTn`jo$UY_`3ySY>&xhS2J(;lk661-Q!rd3#&N7Dv0bo z7CHb$4DHrnn{t+6B4W%JFYR)pDJWEkvBHo4$=J4HoFYb& z6&m7W<)2LJ7GK-uqRm8;vyiq|@`DPQ1psuZQ#h7@njC;Ue`aAV_&d+UkAf{M=HZel zb6nulv9BQ7dob&uJN;)XU!{w1+Ko@$AlOA_Z792WSB6n;^wBhgqub`g(QtNq>>yP6 zs4g{sQ@mREo#C}-s)#dXTiG%X;6HRFcS+1U%kBJEW+J7NfR-%V;yK9WJNzg%hd)@(AG}#bOFOKP{YJHH6nmoMfEiHrc zfVvizI*2nofk{`JHv)RO5MxlhKGI6IRHB7FbP%3*nRYtN0L&t3okzyziR-du*r zfa6lMP~)+TZVSsT3-6>QIso-!9gkY)*P&@T3x5@L@}HD?I_|iT!hh z^Ryh79DDQ-s+dJX@H7GQPh1S2R#M+{eHFDmVR=x!@<|~Klny{Qx)rGSil_K}6KPNp zHx7a^IkCH_`GQsFEVw(wqP|j^^l^9^hWW5=8DhDDW_(3n?oi=E>6KlU5?OISON_N7 zK`XB1!)=j7e!9k2O%QLzZuupVvEpInC_0;Hl++lN2N$B5<(Dj{gMi*ycoN7yB_*=P z5w+m4Pkz;cD|xRTzT~jBQ21$x%;ntSY8L3U`u+KB^>hyl!6rnaq%iZ+vwGmIA$(639%3z~#A zIc&_Jpm~RvkL}a*;`(=Y@O-A=al-d&)EbGY-T4QWH2YF?w@B|hme@fs@20!TlV;i? zdAtZZc9!j;H~L@Xn|i4WHf^UATK%RVSX>FAb^Uz<^s|S+PJ$jsmnJQ^jr7 zK^xGwdwK-lEbF=6?l;$d#~S;x69US2tOL;evM_$+m&$x-8V*uIy#4!kq68I`US@cd z8~BfFo(^aW#gD^M*hACNnE!D?d;;nbnD`xQD}C4%Q$LP@c7&U*ncDTSmRba8z>=HN z=e!Ws|IE}^vDc4a)&c3~aKTl$`Z_P2pyU%KTChWJ|M-n=a)MSzk%f-haKai|`1=J! zf-yH8U(`ZJ`Knh)F@f{5cUI4u)j-Q}W+1*#W9_@$YBL%4R_G;*ppt@tV~WdR+0g2`iBY;a!& zpZW}4f>ym_dtm4l4^mU6*8Y0iL)@zb3&Y4@cBTvz{s_}+FB3L+T`Eq6@l$OHBmqkGVZth8=U0`nmH6xy8a;xhU3b+{MU z9j45-+x>t(+f}&YIU5wCiTL^Z;KSEbg$>y0SI=g?+t_Sx({(#%D2%$_=EvLmuHZ$c zHlk~26a9k>T)~~ud?sC!xrubb`**q89M?JmZvcgUT=&g~Q(XO=To zT@X|R<1`znD@N{Hh(!@MwQP=fL=47PO4T+f5=D+Zl!7*39S0+78?b3jm}QBm&tVClzBJ9~ zFw3d-pco1hVQXQcU^aoP;(7IlTrp+gX!bZ77$)bU1^uuY4t+3KPCNbI=XaZLQk!Vo z=8`Ax=KEn222gae-BZv?d=a{&w|PZ=XyC9{jWj{fGEk)Z4I?ej1yA?hfyf0dynxn~ z?WEAigBiM6vysEih1W@9kOF46HHm~$gt^};AtLd5MkDMCD$D8YqeB;W&ubq;Z%}*Q zdHZqapojkHKg^z2v#C{p=F_mU*+z4_?&nfFEqy1$F%b|!bgpIl6 zCT`|1^0V>6gI@B|shOTf)LYfSgsX+o8cRc%+zZ|u!bk_rYPNCXm@qouK5gby^b{8q z_~(g37dk~;Twqs@3+WLq`eE`O7sI`En1EXlG&&vLIu15ppf`YSiT z=Ljz4SOJjaNcYr*bVEazTqM?&*C7%8t~} z1+`63?)(-K6KlE5HTdRSs>NAFsV#aabM-yhYxu4d8}Rid>W%4;=W1kMG8RD2`Z{Gq zy~2_gu(~g47mlI~?pmyV9z>MjrE2!cLz$n+mWA$~)x-(5Zu1#hFBBu^#(DgVEiDUy)*k)~8_Jq#eE=vb zl$24%oy-M5iwVZ15SFqSM3OwU0~NxyDOJvKfM$w#P|zL37$=&Gr=TUiuzvI~T52mD(h^>g@r8L2XjNSHlP& z7S^+xCdrr)#-r4_`^gF-nzq3f<}qxWae-kL-?Gz0s5)X%aZvcv*A;q@X08ihq?v7C zD9TN?ub2HKp#SwL)tPqd&!UWHd7Ug=XOi0SB8QrPU)rNX6l!uiMq2%BsCdR#r;*Q_ zZ1gsHdH%8B(Ol|z)}IAaa6H3i+PF;9<1a)AJmZ_YDE9(ee$2W|-?=PL`4LPW0@^%^ zOZS&;SpheX%>xGdeNV{8pKek6a` zHlyG;r3XOLW*tO=5B~#c0Opb zD(wh1mwXu_Xxm>8RR6hmo5C5E^^~!LTnJk2h*X+a>?*1(-l`7bKrI*tHWv~`v-j|^ zN#R~6tL_|ykOcm32{z~o!0Qnn!r)g-?@Rw`yp&w7kxoNiiD236)MaZMX-RtXQ%M86 zE?=L&iK$H1%R##Ty=i#;Z!~9x^NA3n@LqYNx>J6N24>fu6u!#&@}k?Ds_lIS3_RCc zQA>hB8R%u9J280ypEVDwO)=B<2~?FB*XEsj^*#>2J4n$uGDP1Sut$pAhKs><)#}G` zb5vIO*5+mc*F8y342l-)sb&tD`wDOjK%5I4!-L6FRD@!!{ad?Y*c32giSIv0#D`SB zn$Z(Wp4tY}q$lUGR{-i8V)~G5=W!)8Urka!-)Y!iq49^&^nr5W05iopV~5c4Gt8np zeAo~ol$H;Z zBuq$5hmKMSbV#`Mh8PujTT(z2QGgI#_PJP}92y@8J?CWdF5wc% zz5TPJ+xVg{V)%b*`}TOI`~H7abfOf=VRV_wAt^>RITc1_5)tLlK{KL*jmW8}Fs7+g zYUY%YQ)Lc0Emx^Y%r;R&tDH77YNpwkeLpU)`?|mP{d@fW`8|IA=~3A}@6+dXc)p*n z=j+0|sfbN~E#*H^TepvjmW$gipOD@G|MA%F#&=k5mL=ijDRZ(|8o!zF4}5r|bNbDH zf7)9m#@u9ek(&x??Au`F#qqjk1Tj4IaHT)JQI)+2uF9}akuM+?jq3k0z*11`cfc34 zBYPniTLb2;2YY~rZ?b&i`>}4OVx;^vcm6b9J<%L_{6$Bznt8;j+SyNA zTlx3b{>a_G__)2{1D%V}$Ui3+I7=py`5S=jTTN8yI$m)cpze_l^5VS?hHcVcEYN)Y zV5F@mv$WT|G_Z>{O;MI+(T*a?m=kafs0Xsi;M$7!i84?_FyF?Mp zo2-VYRgAp-ge~2f53@^vE^gnq1YViqf9^gId%69kAu-#izjtPas}OO@q$$pM%JXm5 zp#M(ybuaMKm5b-^f5jSWk@zi5 z-KoWLxIXgcQ)3)ibW$!vE-0%SpQqq}I?;dq1rxls!Uu4q=bjAG162!O8~?_2?|X zOFGB~0KV(n6u{tmeA!3kqyO{9na@ey>7d@GjBKwm-}kSLqL<2i2UYvL3zllJ($*~` z?(JZa`f>%Bww8JtRv?Ax8IyPL?>W`Rm@YI^kV+kH@hOx>;M)Ec<>(|CxPk{po(^0h zvhX&5+DhEW3^V1`?APP13py;o=1zD$P-*uv{dwcp66hMz2)BA0(ert6Nj1*C0mSci zZO@FudBLFS=X%{{Md~Z>f(LTV)Uq7uQ`$P5w^O>D-5;9&*1BIfrf)y~da>( zhWKLe8R!I$A~(3m>HF*KyjxaNRlgW?EJ}0HjdVq~-c;b=5>o*9GLy+!uU<-ZsDGy%%Dnu^;O82(s=?G>iz0yue(xhZm)?A! zN)G5AAsBWNZ#A4uLyd(4HFuMkwEOe05o!r{J?h!+=Y+sP%85C>V>?bf&lQc1obF)G zVk+l<-8fS!*|tf?hE4hr0`s?DxtoGmho{Z-$BT-m`o8*~w_qX00IS+ibulPTb9z(9 z&C-3Bs|Us32dEc6-d~f3;P#XQ^m3W2M-3jP%B?=hz_7XVYqO%3K5qh? zBtW2gp8p&Dhr1S-fn(xLpI$BjVCEzV{7E^Hl78v;b@5IM>W^+>_yU}G5P%cY0m(o6 zb7k1HA!l8`-lWonZsCiy#YtZy#(kv`=ye{m;n=h?tm`OmqNT5)pEjPDodVTb&KJQNfK{HvUDZ0G9e(+a=;_IkZ6LX10LYJto6zrmxYJ1-zUEoL}NigdNQ9JQ$Ua~R4c zYaQndH;AHJt(LMgwbX1`j)r1oxr-PO_0DY z079HQ49?aq!7IS98HIuOg@5znzVDsxnp@G1$#8l239u5g#Zj$H^xCXHmtsUO5F+d) z{?oz{2drfJi_?JiCy;SG0BX6>fkT~?VVn^-5{62Oub1{vFaRPB$-sN89gReA3D za=?N7ZIX{!)Ko<;*4h6BL$Wt0cG+0-e*s0kTrfaSNfdZFVAEo@@E<03+OyxrJJe9~ zMy^m3u*x20^l^rP9HFP&pc@%+C+mfc^Aw4p+NgQjKS zEa)0~Dl7i!t4rqFE3z0`ud}iJN+jd0bm26sp%+O!WQZ z-ffqt$CbDzui7F!2->)e=bgZvr?b89 zoF2;yx#$gjZuTrlA%b|&#OP}kW1nXKd`i=K^N364krP1Vmn6emiiv^1F>n_!m&KI# zpY(P!V0R=xGrW|>3*)W|r!YEF&a)qaFOi#s#{P^`a<;^!$*cw@v*?xnNhlU3BLh@0 z{dF-{de>|ZZ7KZ!SNL^yKOhpE3s9u~J_-NTQDnC^Uo07iiq6by1?E@Rl^k-6pg<=yX2M4Jidx+SbP)3Y644imF|PTX zf0Z@+)hp)v8vme%wF_nTR7nF%GT|s#xW$n?@@o8{x(vBc)E_fqMnGN|j2}O%6Y^K{ zoqV3SyoUMg>Zm2_%jUk2+9Nij?gX)i;p({3>%O^4*=kQh0YDU>|6PVL^J)n z#P}sNe^J3eFcq>xSfVWB%l`~e{B11e*CZr@beSn#34bKa3n`QvlrOm3WAZYwlizVh zx>MOwrCe!~A@F6=kU6!4IAJcuO<8q?M&Gi8-AgJ>ev8Y!U`ZDQ?-MGnc?=Cv;OSDi zZ{G@V?z-6#inh{g_Cs|PS=?>*!Z)+Pw_(_zc}w7d{mC`ISbZ}lG7>m|c;VWV(|P~5 z7N)k_ArHP_bak*XJNYeuUZ>2v39_zKlXmlMM8=%{%W$9^o}oLyuLHW%5K>}H`R^$4 zj`hO^IH5hjz`HtBjl1noKl}(Mgb6@`7ZfA#sUTGeyTzCWy-Ko3Z8YX{C`4UA-UCWE zc1^pWOcQ?Ct}ES0Ggqo#h_Q9WezeFRrQnYQY+UYiF9ysk>WOyYcI<(Xc7ibP?Z0Z! zMw`;8a~pNb2_k7Ss@)x&ZDJLAxN#f3BXm`+lDLtqm(M#98$kli{!Z@68_1oW(9{(l z856NoqmC7xggkiFOc3Dnd8?WtL;X>h-)>)(Y2hI{!!F7^9a%VuMGmy@Th1IA*bfx= zhebu4D^1Rkf^iQVNd%fA(<|z9VcD+(zGl2(WnW7-s`PsDJ9!z8@rAm_Qr*@quWVkL zUI{x_W@(XtF@a?{3cmQLA%$N)6a;$O!_(5vA^eTT3HB41vUJk5us|2U71iKyUANjN zb=!3hO0=TqPwIl3j%G(3f+z3DU^f&Egd3HDy3ux60@D#Zx4gnccrjw%5u^L}O|$!ld`+2JDG+NUQRFSm~x1!@>i%8A$$Ilfm!?KctEKk=>s)k-<+Y)hIxrF9* z(@mzp7!ZsI+;4n>At#~U#(rliz(7XdvnGlkcaItdMxP^*I%aE=e^1>$DfQKH+ikNU zBjbrSZB{$j-{NF&NH8PXWC2@Qnmr1idg9wk}q+Z_eFc!lWhVT0kxyhmE5W9 z_-2yQL=u)n%e9a0?@;EkNpTbC^SBQ!q$d%75;m%1<=Lcyd)kpvLxH$C(X^5gXwn<@?)3qkfjM{ux7-isGHTkuPqxdpJQ>iu7SFgjdHcvy==Jg@!o9 zbA2MmC}$P(Ih4jccG_~qp{Juw=J{2t_|(FA-<#PzlgNFsBj_R4CBa_M=R4YaO!BKl zJ~mU)1dSa>+Bpp{g#20<>zW8LKTZJ}mewR$Pig3N+uHg}(4F>VoR_aJL2I6pR`6ma z#-oJDSS$(qSjD;B&`h^JeiyWKwBE_bkKh*ifLry@rnZNwF?YBf*;HnxajSrLJx0?` zT2YlulH9Q&*O+*GCJqZgxmEMG3R*g2SO;w|O23g3KiZchQOic?%r|f4+|`C;I?FG$ z<^?r*sIQLc2}lYkG&9|*af`s?V1|a@yin#1j#4Y9i_e=n zwX@NXZ`N9%wBpeakV(u1z1a60g190WWU_FPYDI2VU<}^#)fGh+$0>y(j-(X^3Z^EM zWsl0;U(R_`l+OQNWKHWlGQag#ALT@mvy2!o{}Qd|LKH-aPwU0LyO)RU>@9gO+LDc3 zt~EwF&_r<8MMi}#SFB!EA$vXM&0q*FFX;V&WvM{dL0zA=bt4S`dm|rzH;;`Pj=4S3 zxH3@o-I`bnsfKer*>D5e@RBv9%h@eg_DzhfEB8lUGeS@=68l;5vPTfRG-5s~KqE10 zt1Eit8?k`Od5Da=g4S9`*dL=TqipNb`i85~sN+6g1AHd-ra?AOHBJY}3!QC-7hzw&tdbl9T;8Ed&Y_PqJ) ztgPPXnBuA{Bv9_;Ez;a&-1lZLfG{vz4azsYysI*Gh-C!jo+*x1;h79AyACx<CQbHsmQBnnCK0G?DCQ9L?4)?bNjP$;*{03iA zh(`PUCIHAnjz4P+(i?BQs~>U9{o!ce3^`@3Beim}npF1t*KKC}dXUWhEH6@qTrP(6 z{mwdZya7Gsq9gsR;HsjB99ugSHJXUhn7atm>=P*^6|LhAd7R~49fEN|I>_4Z6`<)D zg#(%!4u*c)PR$sCDQc@AVX)Q1lLdDRl{)s)j8CcbhZm&KN2FJRu(o2ZZFpOCj8IU8fXe*=q@cx|US5Vnpy3 zYvVnZ^tPI$&bfB7JLKsB%WO+u_oBjK$ojpXb7Amdr#QXQ46v;UE#l?s7Qvope>QGp z!r>Wi%+U8^X8iX~K8}n$lxa}Kbal@p=W@0ZX<$w&-r42Wfqb<>S+f6)n=|2JB74WJ z2<|1t`mrE3=}FCT=l~2UX_gtwKNGgY;d;$7e(aAt`HpbIFZ8qbT8fk}SS8mokS!jb zh40zF5<$-k5>=;$Yf?~WfHJHnhnC@S-JM)ZESApb~%XL(xJKm#o(1! ze6Eig=Z%Cf>#_pw2}TdoT37Ya4(+i?cO#ld9D;2jzDNFa(G-+htF-O0M)_PK^4(XW zpIvDH=lW2$vClzOXr1oOw4xh@cdsAG-3dKlqj)9cj)~2WON9=0T=kVU$*5p5FpP*( zOy9YqYP&x6{(1uvJ$C0j1ZYglm>|`{(ZlWfc)yGGgxNWYiz4)rr)>1iqD>!q-;2-* z5GUdq2`5_Dunv_j>#~Z@K|#xxjerf>P|Gg@4x?Zo+7qTT!i{!O?k*w20M+=e&ta=| zeV;>&3wmi^*5JG1Qi`BQGLF-jZMNHvZjJ3>u!^xcy@*(Sre8-b4*lA-0UgLqc-2#h z|6mjDL#VYp=>=2evcHj@U=K~)E1Gr6VEj&&H$RDN{xx^afw3}TtQeS#l5JN%+ZOcB zM`V}e8jk%~;V$yL3K z6}#$UWc!o^eGu~$DmeX$FCdPbQ%H+k6VXcH|@Wd^lH+c z|2{aO>Nr;>p_9b!{sSjC;qr#wX6~6_I0UQZ+>679UW?NsL=^aX>ZB;xw?XBPh{(JGhuh;zw z>>}~#$Mc?OnVcd~cQuSuUa>4nDLqKKPc;=NJ5Tn>V!@V%K%PfPY+8D7dh|M@eB@jaMmaPMR zh2Z-Bb6fQ$8I-;a-`QUUtUlh`=^9-Iuk5+4`kKuNdJ@A_s{Z{3geLypZ(O<7eDtaj z=+`|R4kN7jt&Ia+0^RUpz*`|vATN5Zwk zcG6GHQ}a}U>2PbLc(&UHt@0fEvX^#^Y2YcC6gPOO6Dn_;ue;n}w_I^sQQ*A@ zE~6#L@Xg!{4L~I@KyRQDGzq6pZ1nXokB?K_QbSjYsrP@HT%^PGh`a8rsGl}8t5lph zUr{~(kt!FYCIN+Hzs#_|l|PM`ufsjz6l}0!RqWkMAH(2K#n$KEWPnxc~x=NrBQyq;@NcjAg60xy@R_ z_@lkkD}9Ei_Jt^Y!aV_STBqo;L#jnv>`Z95gpwKewj%eR3TE_t+o{B&!27)p(5D7> z_>&sG0i^pQvaZ6Ap89@huPd2y(LvyxwO2(^voG;*0r3}ghfJIQNZHw9@r$3ITB zwwwflSC*>i_l!5L2b^YZ`_M+{RH0aOWmo(-LbseRYjr;{+os9Aer3q~9IrWpUjm;@ zO)LD?w+Gnl$zzh{1i$>mXcTQg5c!Pf?P6RMdj9?mW{wx++rtk@B<&tyQU#-6*ry2) z9CQct!z`bgY4H=~n7-U2;hDtQ={Sg56cO^tC%TMYko*cj#xUJ4(%NUSWDth@*^dD?HtHb`@X*R9BSuX^Ss{ zy6$GtuAU|e=0xU)P__X^JRxyb)%WgSJEq|bj=CswD-Vwod_M;6sG?J-j#IVKsVa??k5HeW|;3AhZNzLzObW zB%O57EVCq?pPmr{#&JTZ;bbN5NNC|c55`&Z&1RWZ%E0TQHsKC2MsVT606u30B-D-D zy&r0{#lN>!^Pc9BIQh&4r*ev+{WS0%?#L95no;&=!GN8tsN0blzqzg3QH0v5@jRe@ zw_rng#*}Q7Di1%tkvq9pC#L($ZH<`^$VV~3=@Ez(YaJC38%jP%G`#LJQ zvhN}g8>;=5qX9IUv?+;K1X=cMS603)Ki0Qg3biVB9Sjw9+cF=7Nw8g!9lL>y#M0`G zU*C2Snv8Pedqy|SMH})uS8q=H8Xvd*ZK0EqRlIdI?z{&3Ud4AH-~nD*<=49)iU;Op zxq=%vpys=E`sc1Jw?&|maGrA_I2pT5qeJq z#d*@QJ6Vhb+wCM==45{mj65(K?XIWuvpd0YdyX>AO`UTY=9u7@wIhdehmCK^zn>N8z9J5lF7it6@AcKclG&^~G8F#v= z3)<@6@Z*>}m@}S3dNSfMkpw6qYCC5zgohhKS4qP7rXC80r~^~GM;UcbRAi&ulyL%K z$@GX*=exFyO+5+BHQ;^aHV8+$so{IeM{*u4OZ=&|CkXy7ky%*{d+BYFSkATT-C1Zg= zv#!?p+}>FS`dJcPXZ%MZ^N+TStS*Bjyw<9`1A11DmglvOr}^bm>sPnH3pgwIc97T+ z{%oIOCh(eARuoBy-d+FE@018jLxruQWsMn@`!deSOtpku`TASOS zqJ*?g&!eYT3Nq);^l8j+hw_`C4L^&307(Ap^s9?cF*l0>1^ar`Z@!^Dk`p2Pde^HT ziIc1YC3LmL{1D09I27*^-i@`Mg2Cu7@qS%FXPKPx)@t*Rx~UcCVdTy9I-Q!q=~HG9z~ePnPg@$uF)SUy`iZb z=Ij>xoo4>gjVuGYFtvGPK7Ak3mph&Rww%=24cQO;@A%ppX}Q_5PE29oaebx8tfsFF z4o zC|HBmyX@o1x{P0IA#>clB8W-A#ySf6}0$?)1-V$vF03`8H zGMkDsvEonYcXU>D&;35&GWl(?Qs?0e8tYZj3aul48-l&dFDhuF+|?fvkYyoqifBl4 z^G1HiDz@d|ZCnfcTjAWb;RMPhmXT<>6Y{MDQ&n3)D4CXCsmHt{LZd#*{Po z2I1B7n&Xrh%gX;Ym$`U6za~N`uTmhJT}uXn#$R&G1DtN zVX0x#xGyv7_$p}s!`mM^fLMK2AyHKgYP661`sG(l975($uF30|TsNbj&~t2$1J&2X z2_daHm38QB6fwL~u9FW&xIf8({GV)N)1HP#N~e?Xla3 zsg-8RbqTx0EkUU_?1_+3*r!6OWLi1)-8BAAeAu)%T|R0%@bW!cExN{-3W-*YtqdaC z3xSV?-h!KWJfli#a52$Uz?>}4mjl?t6$keU*``B;$HuxMI9hya%&dSDFHrv=olsRQ zLYtN+S9sm=e{Et_o8yI)d<=nq&)Zp)b9mB(YuKajSdtgAh`Z+Ls2gdXAuGr1{RFvI zK3i|N7K)Fu9lTQBtJ6J8m%Ylg=&eb}v5|UXF(@dd!fXvh;uc+&1EUgo%1zRU!JFAH zuNLQ<604SPgZU8rSvX)+@oW)8lG=GrJ2+;!?F052$MpQ>bCB9+)^~hx+%=A{Ut?^& zj>}GVGN`c9T7Irtw&H%-2*39=!D2OZAc}$vEVF~+ z&zIeQWf4rG_q=N!+4t$u#_Z?-lG>)8cS-OjPh@V@W|G@odF2s%R3T2@nd#Chm*%Ch zoZ%MDnX>No2;i}+%Bn&kHxp8w6*mPZ9H7DLisohl>=W-4NC0o@zyNXX1t@M~hgaGy z2RTPwwO6zGgp(HJQJdlvpdXG-X@?MP3D*>3PHE7vx3G;rZVYdB6?k9su;1E%?e0am zanAsF{6sga9=pE>B?lg;=m{^??`^f4?$@9kCPYaZ!Wg83KIchuRX}Z$pGVyy$bW%V8CSdy(zI*M?;!(M096Ik{8h^M6@}niFte*g!hEVgK{xxi z{feaKDnV1etcRjuaJ)S$q(Icq-5p)l89j5-zV$2<=JqI`^{Df;22Z$p*CwBJTrplW zFLW~P_eQ5QZ_iPosY~3!NPjP`jqi4Qg*C_zFCuzp#*%XUVbfeIWVRll6S&`%6a9$uAx6fK6ljXkGx4&yj8iGmLObC z=KuNUwNsNm|0bmqr(+-Y`C~h{pc>G#p?cQKR$oKtNo>CCHxnuVpgBm z$^h!uP|`eHJSGiRnZ5(tuVWzZ+(00^_rmB?bB=q~o*AU{Wk(%6*Enxo%4a)?W19^4 zKb?pk>kFmZr*x(2f8Q{j1B%wn@~?OIfu$fM`iP!lO<11hyS!0{2}~hja0++ZQM*4= z!NHMRpL}TL@UDSf$tDi;**XJ2v?}qMi=L!8%RXll=U(a!fpjPpC~IFDD(SOzru^y1 zUPWVw1NCL{1bLUKA2AFF0XU;&*DmGc2b_Bc^zB&sx%Kli#aCJJb!WzYLwfKrJwl1aw>lP?ZzbK=RGx!jg|*0bQO6$q1?^p~5CNXsl2Wz9V&RN9M#~Ku~{&M}6UF zaFgTpqIQv=z0f74P?%qG=XXZo`(Y6rPzlOw+`=!QJi?s>8=@!DPi+)k8>+&=+d)l! z(Y$q>TSZ=uN~s^Q{UC2`S_tycGf&r4wlL0qXdI9cmhNdXk<5DI2!KYWdS@*-+63lE zpA0!ZQyagkMHrqJ7*3nz!Al(xrvmmEUqi zv8w{4hm`SjuUd;H1Dfane=5HuK?z%yLTav%w>E!RUlDck!x+U{*+sMa(6R#2%Qp^; z?zGmUz4~3>o)<-Y@V{6WbhIA*?PQKR4FWAEWA5V~gQM|t))MmZKVAf$)As@bMJ#p> zxGTw!|8TrJXxB-pq~D1@lRV!^f0;T@M&9ws@LrFMC5tD`bTiUO1ueW7(f9lfg9f;M zrwX^9I=Fa80np3r_`b_$?a5w}ol~=r_0s#4qmW&Mtlr6kYA1SFSL$8jC&dE;%18-_ zBVuh-PqQ{7V@TF;o5Zh`MQ`Yn&C>|z?xh#%b&~vsS`6E1WIy!TH3$2|TS9x3-W9H? z#uL|N7zK-qA?Z7l`W2x!PELZGzWzyR2lH0T3>0CQV>=RoXvzC~$|6Rc-k zdOgG!^T)sg(=6^r$|W^3c;(cvlMmc|9Vd2l68v;vmeyLx2&l%6OV8O$Gy056N3X`| z>AVIQA$OcCqug?flSBs!2c~N0Ygd=-fuZD+a^jz}o_@XiA!AUNgNEEGH&gcE>?1Yq zJb=&ox3ABkjR=q{g@irIf;9LC$n7$DXznS||*O*wP z<$+y`sB(ff9H2pbdGef-7`gj0Z)j6N*+qhf;n}s-A0E(9u1iDg!fm&wSFb{mq8hFT zcf84w-@>ZUJX8p#O_jJq%nuW-fn3a8UbA0&(&73%VL0*2MleCs4h|GAcBYavS5h%I z7~n+EBWo?+J*TP36>IVe`ptPbY;&*r>L|W+^ns(iQPhW>tNB$f0&9XA=U&A+oOUUp zY5*q&_KEWtg7(XR>Jdpfv_g8~}RkHY*up9pUx3JZD%_R>qd?D+-(0<&7PuRwXZ1Ha_*yAAjk zw21?az~iDOVLwuE?YiaRGe(9h=rJNML>n=i4x}FIc=vo&=AfBbw40deD%4mLfR0GU z#nSF)(;6!t+e7orV(Ei|S&i`>nl8r$txjBX-D84gNGz^%y*T`wp2Nz;u_jzT68@9> zqk<=|)S%q4n`K;237hZd$)8jxc-1%6!jYzvfN{xzZdQPM7=x(TWQd70h~625xzS^A z6`Nk1ym=yRxy${#+U^Lq=O^`y%|3g}Cek2q&I`jU0Frn-!tD-Y@{}|l{Vus;_W5yp zwUxc*HuP`ShKQ8Rc@Z zK*UsF9Cm1t+S94egys7rmU+S_>jY!$NUL6_pxjgLpGaoF$Ur!h=lksBiha9PB#j&m zb%TR|#!X64n3Oa=3&xKX>fhn%BKHN7DMteze!v?6Uo9bD^(@4{OqGkZhBoC^vww+L z8P?pT!}jrlj&ovCc74(PqB+|Bk$UM3r-p)p_QLBubNb4yPj+oY})2*yjXH{^8%;qbCo8Y-Ufbdr)1_XX6*zMc4GBfzEjq>o=^0jC%x53@Y`%CW@+%z2S2|6975_C-EB*m*M0(Y`f#!prUH zEQ1fs>*YnjG!hOvU=Eo`E^vhCxWZ|%9DdUZzSzmni7k94e7a^6H&R?pxd_faZy=qu z+-~~$(T-!DZn?ePtS4VFUrUQN(!cB#KO(DtvEcJ4dCIX^zAz@@Tp2=tY5toL+B(j# z%PcMAo})lSrr{KGH>^0N6E$O}hnIBitY?pR;@}FFrTVKqP89ZTex~oFtxEegbuU?7 zJcThe(x~edSn$|E!tF3fLe2vtUVf%+B0C#H9?%lEok*3VNijMwh}F?~u|cUW%mH$= zLP(CZcc0hK9tkeax7F4b0Wrj5c4YZ6Bb~XP{d478DNewrX6db@<4%gd!}T@lye>Z# zLY_40cd-OPsd5rBQgX6aa0rB5(gi^v+BA**GJld`3Dy$)^o=3hN) zWq@Yi7w+~PTgyt#fJWHyebuIni=f*=i3lDL2fZJCb2)OY-s=7d~ literal 0 HcmV?d00001 diff --git a/docs/docs/pre_trained_models.md b/docs/docs/pre_trained_models.md new file mode 100644 index 0000000..28a349b --- /dev/null +++ b/docs/docs/pre_trained_models.md @@ -0,0 +1,23 @@ +# Pre-trained Models + +We maintain a list of pre-trained uncompressed models, so that the training process of model compression does not need to start from scratch. + +For the CIFAR-10 data set, we provide following pre-trained models: + +| Model name | Accuracy | URL | +|:----------:|:--------:|:---------------------------------------------------------------------------------:| +| LeNet | 81.79% | [Link](https://api.ai.tencent.com/pocketflow/models_lenet_at_cifar_10.tar.gz) | +| ResNet-20 | 91.93% | [Link](https://api.ai.tencent.com/pocketflow/models_resnet_20_at_cifar_10.tar.gz) | +| ResNet-32 | 92.59% | [Link](https://api.ai.tencent.com/pocketflow/models_resnet_32_at_cifar_10.tar.gz) | +| ResNet-44 | 92.76% | [Link](https://api.ai.tencent.com/pocketflow/models_resnet_44_at_cifar_10.tar.gz) | +| ResNet-56 | 93.23% | [Link](https://api.ai.tencent.com/pocketflow/models_resnet_56_at_cifar_10.tar.gz) | + +For the ImageNet (ILSVRC-12) data set, we provide following pre-trained models: + +| Model name | Top-1 Acc. | Top-5 Acc. | URL | +|:------------:|:----------:|:----------:|:-------------------------------------------------------------------------------------:| +| ResNet-18 | 70.28% | 89.38% | [Link](https://api.ai.tencent.com/pocketflow/models_resnet_18_at_ilsvrc_12.tar.gz) | +| ResNet-34 | 73.41% | 91.27% | [Link](https://api.ai.tencent.com/pocketflow/models_resnet_34_at_ilsvrc_12.tar.gz) | +| ResNet-50 | 75.97% | 92.88% | [Link](https://api.ai.tencent.com/pocketflow/models_resnet_50_at_ilsvrc_12.tar.gz) | +| MobileNet-v1 | 70.89% | 89.56% | [Link](https://api.ai.tencent.com/pocketflow/models_mobilenet_v1_at_ilsvrc_12.tar.gz) | +| MobileNet-v2 | 71.84% | 90.60% | [Link](https://api.ai.tencent.com/pocketflow/models_mobilenet_v2_at_ilsvrc_12.tar.gz) | diff --git a/docs/docs/reference.md b/docs/docs/reference.md new file mode 100644 index 0000000..a36950d --- /dev/null +++ b/docs/docs/reference.md @@ -0,0 +1,12 @@ +# Reference + +* [**Bergstra et al., 2013**] J. Bergstra, D. Yamins, and D. D. Cox. *Making a Science of Model Search: Hyperparameter Optimization in Hundreds of Dimensions for Vision Architectures*. In International Conference on Machine Learning (ICML), pages 115-123, Jun 2013. +* [**Han et al., 2016**] Song Han, Huizi Mao, and William J. Dally. *Deep Compression: Compressing Deep Neural Network with Pruning, Trained Quantization and Huffman Coding*. In International Conference on Learning Representations (ICLR), 2016. +* [**He et al., 2017**] Yihui He, Xiangyu Zhang, and Jian Sun. *Channel Pruning for Accelerating Very Deep Neural Networks*. In IEEE International Conference on Computer Vision (ICCV), pages 1389-1397, 2017. +* [**He et al., 2018**] Yihui He, Ji Lin, Zhijian Liu, Hanrui Wang, Li-Jia Li, and Song Han. *AMC: AutoML for Model Compression and Acceleration on Mobile Devices*. In European Conference on Computer Vision (ECCV), pages 784-800, 2018. +* [**Hinton et al., 2015**] Geoffrey Hinton, Oriol Vinyals, and Jeff Dean. *Distilling the Knowledge in a Neural Network*. CoRR, abs/1503.02531, 2015. +* [**Jacob et al., 2018**] Benoit Jacob, Skirmantas Kligys, Bo Chen, Menglong Zhu, Matthew Tang, Andrew Howard, Hartwig Adam, and Dmitry Kalenichenko. *Quantization and Training of Neural Networks for Efficient Integer-Arithmetic-Only Inference*. In IEEE Conference on Computer Vision and Pattern Recognition (CVPR), pages 2704-2713, 2018. +* [**Lillicrap et al., 2016**] Timothy P. Lillicrap, Jonathan J. Hunt, Alexander Pritzel, Nicolas Heess, Tom Erez, Yuval Tassa, David Silver, and Daan Wierstra. *Continuous Control with Deep Reinforcement Learning*. In International Conference on Learning Representations (ICLR), 2016. +* [**Mockus, 1975**] J. Mockus. *On Bayesian Methods for Seeking the Extremum*. In Optimization Techniques IFIP Technical Conference, pages 400-404, 1975. +* [**Zhu & Gupta, 2017**] Michael Zhu and Suyog Gupta. *To Prune, or Not to Prune: Exploring the Efficacy of Pruning for Model Compression*. CoRR, abs/1710.01878, 2017. +* [**Zhuang et al., 2018**] Zhuangwei Zhuang, Mingkui Tan, Bohan Zhuang, Jing Liu, Jiezhang Cao, Qingyao Wu, Junzhou Huang, and Jinhui Zhu. *Discrimination-aware Channel Pruning for Deep Neural Networks*. In Annual Conference on Neural Information Processing Systems (NIPS), 2018. diff --git a/docs/docs/reinforcement_learning.md b/docs/docs/reinforcement_learning.md new file mode 100644 index 0000000..2b9a0c1 --- /dev/null +++ b/docs/docs/reinforcement_learning.md @@ -0,0 +1,50 @@ +# Reinforcement Learning + +For most deep learning models, the parameter redundancy differs from one layer to another. +Some layers may be more robust to model compression algorithms due to larger redundancy, while others may be more sensitive. +Therefore, it is often sub-optimal to use a unified pruning ratio or number of quantization bits for all layers, which completely omits the redundancy difference. +However, it is also time-consuming or even impractical to manually setup the optimal value of such hyper-parameter for each layer, especially for deep networks with tens or hundreds of layers. + +To overcome this dilemma, in PocketFlow, we adopt reinforcement learning to automatically determine the optimal pruning ratio or number of quantization bits for each layer. +Our approach is innovated from (He et al., 2018), which automatically determines each layer's optimal pruning ratio, and generalize it to hyper-parameter optimization for more model compression methods. + +In this documentation, we take `UniformQuantLearner` as an example to explain how the reinforcement learning method is used to iteratively optimize the number of quantization bits for each layer. +It is worthy mentioning that this feature is also available for `ChannelPrunedLearner`, `WeightSparseLearner`, and `NonUniformQuantLearner`. + +## Algorithm Description + +Here, we assume the original model to be compressed consists of $T$ layers, and denote the $t$-th layer's weight tensor as $\mathbf{W}_{t}$ and its quantization bit-width as $b_{t}$. +In order to maximally exploit the parameter redundancy of each layer, we need to find the optimal combination of layer-wise quantization bit-width that achieves the highest accuracy after compression while satisfying: + +$$ +\sum_{t = 1}^{T} b_{t} \left| \mathbf{W}_{t} \right| \le b \cdot \sum_{t = 1}^{T} \left| \mathbf{W}_{t} \right| +$$ + +where $\left| \mathbf{W}_{t} \right|$ denotes the number of parameters in the weight tensor $\mathbf{W}_{t}$ and $b$ is the whole network's target quantization bit-width. + +Below, we present the overall workflow of adopting reinforcement learning, or more specifically, the DDPG algorithm (Lillicrap et al., 2016) to search for the optimal combination of layer-wise quantization bit-width: + +![RL Workflow](pics/rl_workflow.png) + +To start with, we initialize an DDPG agent and set the best reward $r_{best}$ to negative infinity to track the optimal combination of layer-wise quantization bit-width. +The search process consists of multiple roll-outs. +In each roll-out, we sequentially traverse each layer in the network to determine its quantization bit-width. +For the $t$-th layer, we construct its state vector with following information: + +* one-hot embedding of layer index +* weight tensor's shape +* number of parameters in the weight tensor +* number of quantization bits used by previous layers +* budget of quantization bits for remaining layers + +Afterwards, we feed this state vector into the DDPG agent to choose an action, which is then converted into the quantization bit-width under certain constraints. +A commonly-used constraint is that with the selected quantization bit-width, the budget of quantization bits for remaining layers should be sufficient, *e.g.* ensuring the minimal quantization bit-width can be satisfied. + +After obtaining all layer's quantization bit-width, we quantize each layer's weights with the corresponding quantization bit-width, and fine-tune the quantized network for a few iteration (as supported by each learner's "Fast Fine-tuning" mode). +We then evaluate the fine-tuned network' accuracy and use it as the reward signal $r_{n}$. +The reward signal is compared against the best reward discovered so far, and the optimal combination of layer-wise quantization bit-width is updated if the current reward is larger. + +Finally, we generate a list of transitions from all the $\left( \mathbf{s}_{t}, a_{t}, r_{t}, \mathbf{s}_{t + 1} \right)$ tuples in the roll-out, and store them in the DDPG agent's replay buffer. +The DDPG agent is then trained with one or more mini-batches of sampled transitions, so that it can choose better actions in the following roll-outs. + +After obtaining the optimal combination of layer-wise quantization bit-width, we can optionally use `UniformQuantLearner`'s "Re-training with Full Data" mode (also supported by others learners) for a complete quantization-aware training to further reduce the accuracy loss. diff --git a/docs/docs/test_cases.md b/docs/docs/test_cases.md new file mode 100644 index 0000000..991f4e7 --- /dev/null +++ b/docs/docs/test_cases.md @@ -0,0 +1,163 @@ +# Test Cases + +This document contains various test cases to cover different combinations of learners and hyper-parameter settings. Any merge request to the master branch should be able to pass all the test cases to be approved. + +## Full-Precision + +``` bash +# local mode +$ ./scripts/run_local.sh nets/resnet_at_cifar10_run.py +$ ./scripts/run_local.sh nets/resnet_at_cifar10_run.py \ + --enbl_dst +$ ./scripts/run_local.sh nets/resnet_at_cifar10_run.py \ + --data_disk hdfs +$ ./scripts/run_local.sh nets/resnet_at_cifar10_run.py \ + --data_disk hdfs \ + --enbl_dst + +# seven mode +$ ./scripts/run_seven.sh nets/resnet_at_ilsvrc12_run.py +$ ./scripts/run_seven.sh nets/resnet_at_ilsvrc12_run.py \ + --enbl_dst +$ ./scripts/run_seven.sh nets/resnet_at_ilsvrc12_run.py \ + --data_disk hdfs +$ ./scripts/run_seven.sh nets/resnet_at_ilsvrc12_run.py \ + --data_disk hdfs \ + --enbl_dst + +# docker mode +$ ./scripts/run_docker.sh nets/lenet_at_cifar10_run.py +$ ./scripts/run_docker.sh nets/lenet_at_cifar10_run.py \ + --enbl_dst +$ ./scripts/run_docker.sh nets/resnet_at_cifar10_run.py +$ ./scripts/run_docker.sh nets/resnet_at_cifar10_run.py \ + --enbl_dst +``` + +## Channel Pruning + +``` bash +# uniform preserve ratios for all layers +$ ./scripts/run_seven.sh nets/resnet_at_cifar10_run.py \ + --learner channel \ + --cp_prune_option uniform \ + --cp_uniform_preserve_ratio 0.5 + +# auto-tuned preserve ratios for each layer +$ ./scripts/run_seven.sh nets/resnet_at_cifar10_run.py \ + --cp_learner channel \ + --cp_prune_option auto \ + --cp_preserve_ratio 0.3 +``` + +## Discrimination-aware Channel Pruning + +``` bash +# no network distillation +$ ./scripts/run_local.sh nets/resnet_at_cifar10_run.py \ + --learner dis-chn-pruned \ + --dcp_nb_stages 3 \ + --data_disk hdfs + +# network distillation +$ ./scripts/run_seven.sh nets/mobilenet_at_ilsvrc12_run.py \ + --learner dis-chn-pruned \ + --enbl_dst \ + --dcp_nb_stages 4 +``` + +## Weight Sparsification + +``` bash +# uniform pruning ratios for all layers +$ ./scripts/run_local.sh nets/resnet_at_cifar10_run.py \ + --learner weight-sparse \ + --ws_prune_ratio_prtl uniform \ + --data_disk hdfs + +# optimal pruning ratios for each layer +$ ./scripts/run_local.sh nets/resnet_at_cifar10_run.py \ + --learner weight-sparse \ + --ws_prune_ratio_prtl optimal \ + --data_disk hdfs + +# heurist pruning ratios for each layer +$ ./scripts/run_seven.sh nets/resnet_at_ilsvrc12_run.py \ + --learner weight-sparse \ + --ws_prune_ratio_prtl heurist + +# optimal pruning ratios for each layer +$ ./scripts/run_seven.sh nets/resnet_at_ilsvrc12_run.py \ + --learner weight-sparse \ + --ws_prune_ratio_prtl optimal +``` + +## Uniform Quantization + +``` bash +# channel-based bucketing +$ ./scripts/run_local.sh nets/resnet_at_cifar10_run.py \ + --learner uniform \ + --uql_use_buckets \ + --uql_bucket_type channel \ + --data_disk hdfs + +# split-based bucketing +$ ./scripts/run_local.sh nets/resnet_at_cifar10_run.py \ + --learner uniform \ + --uql_use_buckets \ + --uql_bucket_type split \ + --data_disk hdfs + +# channel-based bucketing + RL +$ ./scripts/run_seven.sh nets/mobilenet_at_ilsvrc12_run.py -n=2 \ + --learner uniform \ + --uql_enbl_rl_agent \ + --uql_use_buckets \ + --uql_bucket_type channel + +# split-based bucketing + RL +$ ./scripts/run_seven.sh nets/resnet_at_ilsvrc12_run.py -n=2 \ + --learner uniform \ + --uql_enbl_rl_agent \ + --uql_use_buckets \ + --uql_bucket_type split +``` + +## Non-uniform Quantization + +``` bash +# channel-based bucketing + RL + optimize clusters +$ ./scripts/run_local.sh nets/resnet_at_cifar10_run.py \ + --learner non-uniform \ + --nuql_enbl_rl_agent \ + --nuql_use_buckets \ + --nuql_bucket_type channel \ + --nuql_opt_mode clusters \ + --data_disk hdfs + +# split-based bucketing + RL + optimize weights +$ ./scripts/run_local.sh nets/resnet_at_cifar10_run.py \ + --learner non-uniform \ + --nuql_enbl_rl_agent \ + --nuql_use_buckets \ + --nuql_bucket_type split \ + --nuql_opt_mode weights \ + --data_disk hdfs + +# channel-based bucketing + RL + optimize weights +$ ./scripts/run_seven.sh nets/mobilenet_at_ilsvrc12_run.py -n=2 \ + --learner non-uniform \ + --nuql_enbl_rl_agent \ + --nuql_use_buckets \ + --nuql_bucket_type channel \ + --nuql_opt_mode weights + +# split-based bucketing + RL + optimize clusters +$ ./scripts/run_seven.sh nets/resnet_at_ilsvrc12_run.py -n=2 \ + --learner non-uniform \ + --nuql_enbl_rl_agent \ + --nuql_use_buckets \ + --nuql_bucket_type split \ + --nuql_opt_mode clusters +``` diff --git a/docs/docs/tutorial.md b/docs/docs/tutorial.md new file mode 100644 index 0000000..f39a5d4 --- /dev/null +++ b/docs/docs/tutorial.md @@ -0,0 +1,241 @@ +# Tutorial + +In this tutorial, we demonstrate how to compress a convolutional neural network and export the compressed model into a \*.tflite file for deployment on mobile devices. The model we used here is a 18-layer residual network (denoted as "ResNet-18") trained for the ImageNet classification task. We will compress it with the discrimination-aware channel pruning algorithm (Zhuang et al., NIPS '18) to reduce the number of convolutional channels used in the network for speed-up. + +## Prepare the Data + +To start with, we need to convert the ImageNet data set (ILSVRC-12) into TensorFlow's native TFRecord file format. You may follow the data preparation guide [here](https://github.com/tensorflow/models/tree/master/research/inception#getting-started) to download the full data set and convert it into TFRecord files. After that, you should be able to find 4,096 training files and 128 validation files in the data directory, like this: + +``` bash +# training files +train-00000-of-04096 +train-00001-of-04096 +... +train-04095-of-04096 + +# validation files +validation-00000-of-00128 +validation-00001-of-00128 +... +validation-00127-of-00128 +``` + +## Prepare the Pre-trained Model + +The discrimination-aware channel pruning algorithm requires a pre-trained uncompressed model provided in advance, so that a channel-pruned model can be trained with warm-start. You can download a pre-trained model from [here](https://api.ai.tencent.com/pocketflow/list.html), and then unzip files into the `models` sub-directory. + +Alternatively, you can train an uncompressed full-precision model from scratch using `FullPrecLearner` with the following command (choose whatever mode that fits you): + +``` bash +# local mode with 1 GPU +$ ./scripts/run_local.sh nets/resnet_at_ilsvrc12_run.py + +# docker mode with 8 GPUs +$ ./scripts/run_docker.sh nets/resnet_at_ilsvrc12_run.py -n=8 + +# seven mode with 8 GPUs +$ ./scripts/run_seven.sh nets/resnet_at_ilsvrc12_run.py -n=8 +``` + +After the training process, you should be able to find the resulting model files located at the `models` sub-directory in PocketFlow's home directory. + +## Train the Compressed Model + +Now, we can train a compressed model with the discrimination-aware channel pruning algorithm, as implemented by `DisChnPrunedLearner`. Assuming you are now in PocketFlow's home directory, the training process of model compression can be started using the following command (choose whatever mode that fits you): + +``` bash +# local mode with 1 GPU +$ ./scripts/run_local.sh nets/resnet_at_ilsvrc12_run.py \ + --learner dis-chn-pruned + +# docker mode with 8 GPUs +$ ./scripts/run_docker.sh nets/resnet_at_ilsvrc12_run.py -n=8 \ + --learner dis-chn-pruned + +# seven mode with 8 GPUs +$ ./scripts/run_seven.sh nets/resnet_at_ilsvrc12_run.py -n=8 \ + --learner dis-chn-pruned +``` + +Let's take the execution command for the local mode as an example. In this command, `run_local.sh` is a shell script that executes the specified Python script with user-provided arguments. Here, we ask it to run the Python script named `nets/resnet_at_ilsvrc12_run.py`, which is the execution script for ResNet models on the ImageNet data set. After that, we use `--learner dis-chn-pruned` to specify that the `DisChnPrunedLearner` should be used for model compression. You may also use other learners by specifying the corresponding learner name. Below is a full list of available learners in PocketFlow: + +| Learner name | Learner class | Note | +|:-----------------|:-----------------------------|:---------------------------------------------| +| `full-prec` | `FullPrecLearner` | No model compression | +| `channel` | `ChannelPruningLearner` | Channel pruning [2] | +| `dis-chn-pruned` | `DisChnPrunedLearner` | Discrimination-aware channel pruning [1] | +| `weight-sparse` | `WeightSparseLearner` | Weight sparsification [3] | +| `uniform` | `UniformQuantizedLearner` | Uniform weight quantization [] | +| `tf-uniform` | `UniformQuantizedLearnerTF` | Uniform weight quantization in TensorFlow [] | +| `non-uniform` | `NonUniformQuantizedLearner` | Non-uniform weight quantization [] | + +The local mode only uses 1 GPU for the training process, which takes approximately 20-30 hours to complete. This can be accelerated by multi-GPU training in the docker and seven mode, which is enabled by adding `-n=x` right after the specified Python script, where `x` is the number of GPUs to be used. + +Optionally, you can pass some extra arguments to customize the training process. For the discrimination-aware channel pruning algorithm, some of key arguments are: + +| Name | Definition | Default Value | +|:------------------|:---------------------------------------|:--------------| +| `enbl_dst` | Enable training with distillation loss | False | +| `dcp_prune_ratio` | DCP algorithm's pruning ratio | 0.5 | + +You may override the default value by appending customized arguments at the end of the execution command. For instance, the following command: + +``` bash +$ ./scripts/run_local.sh nets/resnet_at_ilsvrc12_run.py \ + --learner dis-chn-pruned \ + --enbl_dst \ + --dcp_prune_ratio 0.75 +``` + +requires the `DisChnPrunedLearner` to achieve an overall pruning ratio of 0.75 and the training process will be carried out with the distillation loss. As a result, the number of channels in each convolutional layer of the compressed model will be one quarter of the original one. + +After the training process is completed, you should be able to find a sub-directory named `models_dcp_eval` created in the home directory of PocketFlow. This sub-directory contains all the files that define the compressed model, and we will export them to a TensorFlow Lite formatted model file for deployment in the next section. + +## Export to TensorFlow Lite + +TensorFlow's checkpoint files cannot be directly used for deployment on mobile devices. Instead, we need to firstly convert them into a single \*.tflite file that is supported by the TensorFlow Lite Interpreter. For model compressed with channel-pruning based algorithms, *e.g.* `ChannelPruningLearner` and `DisChnPrunedLearner`, we have prepared a model conversion script, `tools/conversion/export_pb_tflite_models.py`, to generate a TF-Lite model from TensorFlow's checkpoint files. + +To convert checkpoint files into a \*.tflite file, use the following command: + +``` bash +# convert checkpoint files into a *.tflite model +$ python tools/conversion/export_pb_tflite_models.py \ + --model_dir models_dcp_eval +``` + +In the above command, we specify the model directory containing checkpoint files generated in the previous training process. The conversion script automatically detects which channels can be safely pruned, and then produces a light-weighted compressed model. The resulting TensorFlow Lite file is also placed at the `models_dcp_eval` directory, named as `model_transformed.tflite`. + +## Deploy on Mobile Devices + +After exporting the compressed model to the TensorFlow Lite file format, you may follow the official [guide](https://www.tensorflow.org/lite/demo_android) for creating an Android demo App from it. Basically, this demo App uses a TensorFlow Lite model to continuously classifies images captured by the camera, and all the computation are performed on mobile devices in real time. + +To use the `model_transformed.tflite` model file, you need to place it in the `asserts` directory and create a Java class named `ImageClassifierFloatResNet` to use this model for classification. Below is the example code, which is modified from `ImageClassifierFloatInception.java` used in the official demo project: + +``` Java +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package com.example.android.tflitecamerademo; + +import android.app.Activity; + +import java.io.IOException; + +/** + * This classifier works with the ResNet-18 model. + * It applies floating point inference rather than using a quantized model. + */ +public class ImageClassifierFloatResNet extends ImageClassifier { + + /** + * The ResNet requires additional normalization of the used input. + */ + private static final float IMAGE_MEAN_RED = 123.58f; + private static final float IMAGE_MEAN_GREEN = 116.779f; + private static final float IMAGE_MEAN_BLUE = 103.939f; + + /** + * An array to hold inference results, to be feed into Tensorflow Lite as outputs. + * This isn't part of the super class, because we need a primitive array here. + */ + private float[][] labelProbArray = null; + + /** + * Initializes an {@code ImageClassifier}. + * + * @param activity + */ + ImageClassifierFloatResNet(Activity activity) throws IOException { + super(activity); + labelProbArray = new float[1][getNumLabels()]; + } + + @Override + protected String getModelPath() { + return "model_transformed.tflite"; + } + + @Override + protected String getLabelPath() { + return "labels_imagenet_slim.txt"; + } + + @Override + protected int getImageSizeX() { + return 224; + } + + @Override + protected int getImageSizeY() { + return 224; + } + + @Override + protected int getNumBytesPerChannel() { + // a 32bit float value requires 4 bytes + return 4; + } + + @Override + protected void addPixelValue(int pixelValue) { + imgData.putFloat(((pixelValue >> 16) & 0xFF) - IMAGE_MEAN_RED); + imgData.putFloat(((pixelValue >> 8) & 0xFF) - IMAGE_MEAN_GREEN); + imgData.putFloat((pixelValue & 0xFF) - IMAGE_MEAN_BLUE); + } + + @Override + protected float getProbability(int labelIndex) { + return labelProbArray[0][labelIndex]; + } + + @Override + protected void setProbability(int labelIndex, Number value) { + labelProbArray[0][labelIndex] = value.floatValue(); + } + + @Override + protected float getNormalizedProbability(int labelIndex) { + // TODO the following value isn't in [0,1] yet, but may be greater. Why? + return getProbability(labelIndex); + } + + @Override + protected void runInference() { + tflite.run(imgData, labelProbArray); + } +} +``` + +After that, you need to change the image classifier class used in `Camera2BasicFragment.java`. Locate the function named `onActivityCreated` and change its content as below. Now you will be able to use the compressed ResNet-18 model to classify objects on your mobile phone in real time. + +``` Java +/** Load the model and labels. */ +@Override +public void onActivityCreated(Bundle savedInstanceState) { + super.onActivityCreated(savedInstanceState); + try { + classifier = new ImageClassifierFloatResNet(getActivity()); + } catch (IOException e) { + Log.e(TAG, "Failed to initialize an image classifier.", e); + } + startBackgroundThread(); +} +``` + +## Reference + +* [1] Zhuangwei Zhuang, Mingkui Tan, Bohan Zhuang, Jing Liu, Jiezhang Cao, Qingyao Wu, Junzhou Huang, Jinhui Zhu, *Discrimination-aware Channel Pruning for Deep Neural Networks*, In Proc. of the Annual Conference on Neural Information Processing Systems (NIPS), 2018. +* [2] Yihui He, Xiangyu Zhang, Jian Sun, *Channel Pruning for Accelerating Very Deep Neural Networks*, In Proc. of the IEEE International Conference on Computer Vision (ICCV), 2017. +* [3] Michael Zhu, Suyog Gupta, *To Prune, or Not to Prune: Exploring the Efficacy of Pruning for Model Compression*, CoRR, abs/1710.01878, 2017. diff --git a/docs/docs/uq_learner.md b/docs/docs/uq_learner.md new file mode 100644 index 0000000..0eebe77 --- /dev/null +++ b/docs/docs/uq_learner.md @@ -0,0 +1,167 @@ + +# Uniform Quantization + +This document describes how to set up uniform quantization with PocketFlow. Uniform quantization is widely used for model compression and acceleration. Originally the weights in the network are represented by 32-bit float numbers. With uniform quantization, low-precision (e.g., 4 bit, 8 bit) and evenly distributed float numbers are used to approximate the full precision networks. For 8 bit quantization, the network size can be reduced by 4 folds with little drop of performance. + + Currently PocketFlow supports two types of uniform quantization: + +* Uniform Quantization Learner: the self-developed learner. Aside from uniform quantization, the learner is carefully optimized with various extensions supported. The detailed algorithm of the Uniform Quantized Learner will be introduced at the end of the algorithm. The algorithm details are presented at the end of the document. + +* TensorFlow Quantization Wrapper: a wrapper based on the [post training quantization](https://www.tensorflow.org/performance/post_training_quantization) in TensorFlow. The wrapper currently only supports 8-bit quantization, enjoying 4x reduction of memory and nearly 4x times speed up of inference. + +A comparison of the two learners are shown below: + +| Features| Uniform Quantization Learner | TensorFlow Quantization Wrapper | +| :--------: | :--------:| :--: | +| Compression | yes | Yes | +| Acceleration | | Yes | +| Fine-tuning | Yes | | +| Bucketing | Yes | | +| Hyper-param Searching| Yes | | + + +## Uniform Quantization Learner +The uniform quantization learner supports both weight quantization and activation quantization, where users can manually set up the bits for quantization. The uniform quantization learner also supports bucketing, which leads to more fine-grained quantization and better performance. The users can also turn on the hyper parameter optimizer with reinforcement learning to search for the optimal bit allocation for the learner. + +### Prepare the Model +To quantize the network, users should first get the model prepared. Users can either use the pre-built models in PocketFlow, or develop their custom models according to [TODO](???). + +### Configure the Learner + +To configure the learner, users can pass the options via the TensorFlow flag interface. The available options are as follows: + +| Options | Default Value | Description | +| :-------- | :--------:| :-- | +| `--uql_weight_bits` | 4 | the number of bits for weight | +| `--uql_activation_bits` | 32 | the number of bits for activation， by default it remains full precision | +| `--uql_save_quant_mode_path` | *TODO* | the save path for quantized models | +| `--uql_use_buckets` | False | use bucketing or not | +| `--uql_bucket_type` | channel | two bucket type available: ['split', 'channel'] | +| `--uql_bucket_size` | 256 | quantize the first and last layers of the network or not | +| `--uql_enbl_rl_agent` | False | enable reinforcement learning to learn the optimal bit allocation or not | +| `--uql_quantize_all_layers` | False | quantize the first and last layers of the network or not | +| `--uql_quant_epoch` | 60 | the number of epochs for fine-tuning | + +### Examples +Once the model is built, the quantization can be easily triggered by directly passing the Uniform Quantization Learner in the command line as follows: +```bash +# quantize resnet-20 on CIFAR-10 +# you can also configure the +sh ./scripts/run_local.sh nets/resnet_at_cifar10_run.py \ +--data_disk local \ +--data_dir_local ${PF_CIFAR10_LOCAL} \ +--learner=uniform \ +--uql_weight_bits=4 \ +--uql_activation_bits=4 \ + +# quantize the resnet-18 on ILSVRC-12 +sh ./scripts/run_local.sh nets/resnet_at_ilsvrc12_run.py \ +--learner=uniform \ +--data_disk local \ +--data_dir_local ${PF_ILSVRC12_LOCAL} \ +--uql_weight_bits=8 \ +--uql_activation_bits=8 \ +--uql_use_buckets=True \ +--uql_bucket_type=channel +``` + +### Configure the Hyper Parameter Optimizer +Once the hyper parameter optimizer is turned on, i.e., `uql_enbl_rl_agent==True` , the reinforcement learning agents will search for the optimal allocation of bits to each layers. Before the search, users are supposed set up the bit constraints via `--uql_evquivalent_bits`, so that the optimal bits searched by the RL agent will not exceed the bit number without RL agent. +*For example, TODO* + +Users can also configure other options in the RL agent, such as the number of roll-outs, the fine-tuning steps to get the reward, e.t.c.. Full list of options are listed as follows: + +| Options | Default Value | Description | +| :-------- | :-------:| :-- | +| `--uql_evquivalent_bits` | 4 | the number of re-allocated bits that is equivalent to uniform quantization without RL agent | +| `--uql_nb_rlouts` | 200 | the number of roll outs for training the RL agent | +| `--uql_w_bit_min` | 2 | the minimal number of bits for each layer | +| `--uql_w_bit_max` | 8| the maximal number of bits for each layer | +| `--uql_enbl_rl_global_tune` | True | enable fine-tuning all layers of the network or not | +| `--uql_enbl_rl_layerwise_tune` | False | enable fine-tuning the network layer by layer or not | +| `--uql_tune_layerwise_steps` | 100 | the number of steps for layerwise fine-tuning | +| `--uql_tune_global_steps` | 2000 | the number of steps for global fine-tuning | +| `--uql_tune_disp_steps` | 300 | the display steps to show the fine-tuning progress | +| `--uql_enbl_random_layers` | True | randomly permute the layers during RL agent training | + +### Examples +```bash +# quantize mobilenet-v1 on ILSVRC-12 +sh ./scripts/run_local.sh nets/mobilnet_at_ilsvrc12_run.py \ +--data_disk local \ +--data_dir_local ${PF_CIFAR10_LOCAL} \ +--learner=uniform \ +--uql_enbl_rl_agent=True \ +--uql_equivalent_bits=4 \ +--uql_tune_global_steps=1200 +``` + +### Performance +Here we list some of the performance on Cifar-10 using the Uniform Quantization Learner and the built-in models in PocketFlow. The options not displayed remain the default values. + +| Model | Weight Bit| Activation Bit | Acc | +| :--------: |:--------:| :--: | :--:| +| ResNet-20 | 32 | 32 | 91.96 | +| ResNet-20 | 4 | 4 | 90.73 | +| ResNet-20 | 8 | 8 | 92.25 | + +| Model | Weight Bit| Bucketing | Acc | +| :--------: | :--: |:--------:| :--: | +| ResNet-20 | 2 | channel | 89.67 | +| ResNet-20 | 4 | channel | 92.02 | +| ResNet-20 | 2 | split | 91.15 | +| ResNet-20 | 4 | split | 91.98 | + +| Model | Weight Bit| RL search | Acc | +| :--------: | :--: |:--------:| :--: | +| ResNet-20 | 2 | FALSE | 86.17 | +| ResNet-20 | 4 | FALSE | 91.76 | +| ResNet-20 | 2 | TRUE| 90.30 | +| ResNet-20 | 4 | TRUE | 91.88 | + +## TensorFlow Quantization Wrapper +PocketFlow wraps the post training quantization in Tensorflow, and include all the necessary steps to convert the model to the .tflite format, which can be deployed on Andriod devices. To run the wrapper, users only need to get the checkpoint files ready, and then run the script. + +### Prepare the Checkpoint Files +Generally in TensorFlow, the checkpoints of a model include three files: .data, .index, .meta. Users are also supposed to add the input and output to collections, and configure the wrapper to acquire the corresponding collections. An example is as follows: + +*TODO* add quantization to the conversion tools +```bash +# load the checkpoints in ./models, and read the collections of 'inputs' and 'outputs' +python export_pb_tflite_models.py \ +--model_dir ./models +--input_coll inputs +--output_coll outputs +--quantize True +``` +If successfully transformed, the `.pb` and `.tflite` files will be saved in `./models`. + +### Deploy on Mobile Devices +*TODO* + + +## Algorithms +Now we introduce the detailed algorithm in the Uniform Quantization Learner. As is shown in the following graph, given a full precision model, the Uniform Quantization Learner inserts quantization nodes into the computation graph of the model. To enable activation quantization, the quantization nodes shall also be inserted after the activation function. +In the training phase, both full-precision and quantized weights are stored. During the forward pass, quantized weights are obtained by applying the quantization functions on the full precision weights. For the backward propagation of gradients, since the gradients w.r.t. the quantized weights are 0 almost everywhere, we use the straight-through estimator (STE) ([Hinton et.al 2012](https://www.coursera.org/learn/neural-networks), [Bengio et.al 2013](https://arxiv.org/abs/1308.3432)) to pass the gradient of quantized weights directly to the full precision weights for update. + + ![Train_n_Inference](pics/train_n_inference.png) + +### Uniform Quantization Function +Uniform quantization distributed the quantization points evenly across the distribution of weights, and the full precision numbers are then assigned to the closest quantization point. To achieve this, we first normalize the full precision weights $x$ of one layer to $[0, 1]$, i.e., +$$ +sc(x) = \frac{x-\beta}{\alpha}, +$$ +where $\alpha=\max{x}-\min{x}$ and $\beta = \min{x}$ are the scaling factors. Then we assign $sc(x)$ to the discrete value by +$$ +\hat{x}=\frac{1}{2^k-1}\mathrm{round}((2^k-1)\cdot sc(x)), +$$ +and finally we do the inverse linear transformation to recover the quantized weights to the original scale, +$$ +Q(x)=\alpha\hat{x}+\beta. +$$ + + +## References +Bengio Y, Léonard N, Courville A. Estimating or propagating gradients through stochastic neurons for conditional computation. arXiv preprint [arXiv:1308.3432, 2013](https://arxiv.org/abs/1308.3432) + +Geoffrey Hinton, Nitish Srivastava, Kevin Swersky, Tijmen Tieleman and Abdelrahman Mohamed. Neural Networks for Machine Learning. [Coursera, video lectures, 2012](https://www.coursera.org/learn/neural-networks) diff --git a/docs/docs/ws_learner.md b/docs/docs/ws_learner.md new file mode 100644 index 0000000..34f6e89 --- /dev/null +++ b/docs/docs/ws_learner.md @@ -0,0 +1,96 @@ +# Weight Sparsification + +## Introduction + +By imposing sparsity constraints on convolutional and fully-connected layers, the number of non-zero weights can be dramatically reduced, which leads to smaller model size and lower FLOPS for inference (actual acceleration depends on efficient implementation for sparse operations). Directly training a network with fixed sparsity degree may encounter some optimization difficulties and takes longer time to converge. To overcome this, Zhu & Gupta proposed a dynamic pruning schedule to gradually remove network weights to simplify the optimization process (Zhu & Gupta, 2017). + +Note: in this documentation, we will use both "sparsity" and "pruning ratio" to denote the ratio of zero-valued weights over all weights. + +## Algorithm Description + +For each convolutional kernel (for convolutional layer) or weighting matrix (for fully-connected layer), we create a binary mask of the same size to impose the sparsity constraint. During the forward pass, the convolutional kernel (or weighting matrix) is multiplied with the binary mask, so that some weights will not participate in the computation and also will not be updated via gradients. The binary mask is computed based on absolute values of weights: weight with the smallest absolute value will be masked-out until the desired sparsity is reached. + +During the training process, the sparsity is gradually increased to improve the overall optimization behaviour. The dynamic pruning schedule is defined as: + +$$ +s_{t} = s_{f} - s_{f} \cdot \left( 1 - \frac{t - t_{b}}{t_{e} - t_{b}} \right)^{\alpha}, t \in \left[ t_{b}, t_{e} \right] +$$ + +where $s_{t}$ is the sparsity at iteration \#$t$, $s_{f}$ is the target sparsity, $t_{b}$ and $t_{e}$ are the iteration indices where the sparsity begins and stops increasing, and $\alpha$ is the exponent term. In the actual implementation, the binary mask is not updated at each iteration. Instead, it is updated every $\Delta t$ iterations so as to stabilize the training process. We visualize the dynamic pruning schedule in the figure below. + +![WSL PR Schedule](pics/wsl_pr_schedule.png) + +Most networks consist of multiple layers, and the weight redundancy may differ from one layer to another. In order to maximally exploit the weight redundancy, we incorporate a reinforcement learning controller to automatically determine the optimal sparsity (or pruning ratio) for each layer. In each roll-out, the RL agent sequentially determine the sparsity for each layer, and then the network is pruned and re-trained for a few iterations using layer-wise regression & global fine-tuning. The reward function's value is computed based on the re-trained network's accuracy (and computation efficiency), and then used update model parameters of RL agent. For more details, please refer to the documentation named "Hyper-parameter Optimizer - Reinforcement Learning". + +## Hyper-parameters + +Below is the full list of hyper-parameters used in the weight sparsification learner: + +| Name | Description | +|:-----|:------------| +| `ws_save_path` | model's save path | +| `ws_prune_ratio` | target pruning ratio | +| `ws_prune_ratio_prtl` | pruning ratio protocol: 'uniform' / 'heurist' / 'optimal' | +| `ws_nb_rlouts` | number of roll-outs for the RL agent | +| `ws_nb_rlouts_min` | minimal number of roll-outs for the RL agent to start training | +| `ws_reward_type` | reward type: 'single-obj' / 'multi-obj' | +| `ws_lrn_rate_rg` | learning rate for layer-wise regression | +| `ws_nb_iters_rg` | number of iterations for layer-wise regression | +| `ws_lrn_rate_ft` | learning rate for global fine-tuning | +| `ws_nb_iters_ft` | number of iterations for global fine-tuning | +| `ws_nb_iters_feval` | number of iterations for fast evaluation | +| `ws_prune_ratio_exp` | pruning ratio's exponent term | +| `ws_iter_ratio_beg` | iteration ratio at which the pruning ratio begins increasing | +| `ws_iter_ratio_end` | iteration ratio at which the pruning ratio stops increasing | +| `ws_mask_update_step` | step size for updating the pruning mask | + +Here, we provide detailed description (and some analysis) for above hyper-parameters: + +* `ws_save_path`: save path for model created in the training graph. The resulting checkpoint files can be used to resume training from a previous run and compute model's loss function's value and some other evaluation metrics. +* `ws_prune_ratio`: target pruning ratio for convolutional & fully-connected layers. The larger `ws_prune_ratio` is, the more weights will be pruned. If `ws_prune_ratio` equals 0, then no weights will be pruned and model remains the same; if `ws_prune_ratio` equals 1, then all weights are pruned. +* `ws_prune_ratio_prtl`: pruning ratio protocol. Possible options include: 1) uniform: all layers use the same pruning ratio; 2) heurist: the more weights in one layer, the higher pruning ratio will be; 3) optimal: each layer's pruning ratio is determined by reinforcement learning. +* `ws_nb_rlouts`: number of roll-outs for training the reinforcement learning agent. A roll-out refers to: use the RL agent to determine the pruning ratio for each layer; fine-tune the weight sparsified network; evaluate the fine-tuned network to obtain the reward value. +* `ws_nb_rlouts_min`: minimal number of roll-outs for the RL agent to start training. The RL agent requires a few roll-outs for random exploration before actual training starts. We recommend to set this to be a quarter of `ws_nb_rlouts`. +* `ws_reward_type`: reward function's type for the RL agent. Possible options include: 1) single-obj: the reward function only depends on the compressed model's accuracy (the sparsity constraint is imposed during roll-out); 2) multi-obj: the reward function depends on both the compressed model's accuracy and the actual sparsity. +* `ws_lrn_rate_rg`: learning rate for layer-wise regression. +* `ws_nb_iters_rg`: number of iterations for layer-wise regression. This should be set to some value that the layer-wise regression can almost converge and the loss function's value does not decrease much even if more iterations are used. +* `ws_lrn_rate_ft`: learning rate for global fine-tuning. +* `ws_nb_iters_ft`: number of iterations for global fine-tuning. This should be set to some value that the global fine-tuning can almost converge and the loss function's value does not decrease much even if more iterations are used. +* `ws_nb_iters_feval`: number of iterations for fast evaluation. In each roll-out, the re-trained network is evaluated on a subset of evaluation data to save time. +* `ws_prune_ratio_exp`: pruning ratio's exponent term as defined in the dynamic pruning schedule above. +* `ws_iter_ratio_beg`: iteration ratio at which the pruning ratio begins increasing. In the dynamic pruning schedule defined above, $t_{b}$ equals to the total number of training iterations multiplied with `ws_iter_ratio_beg`. +* `ws_iter_ratio_end`: iteration ratio at which the pruning ratio stops increasing. In the dynamic pruning schedule defined above, $t_{e}$ equals to the total number of training iterations multiplied with `ws_iter_ratio_end`. +* `ws_mask_update_step`: step size for updating the pruning mask. By increasing `ws_mask_update_step`, binary masks for weight pruning are less frequently updated, which will speed-up the training but the difference between pre-update and post-update sparsity will be larger. + +## Usage Examples + +In this section, we provide some usage examples to demonstrate how to use `WeightSparseLearner` under different execution modes and hyper-parameter combinations: + +To compress a ResNet-20 model for CIFAR-10 classification task in the local mode, use: + +``` bash +# set the target pruning ratio to 0.75 +./scripts/run_local.sh nets/resnet_at_cifar10_run.py \ + --learner weight-sparse \ + --ws_prune_ratio 0.75 +``` + +To compress a ResNet-34 model for ILSVRC-12 classification task in the docker mode with 4 GPUs, use: + +``` bash +# set the pruning ratio protocol to "heurist" +./scripts/run_docker.sh nets/resnet_at_ilsvrc12_run.py -n=4 \ + --learner weight-sparse \ + --resnet_size 34 \ + --ws_prune_ratio_prtl heurist +``` + +To compress a MobileNet-v2 model for ILSVRC-12 classification task in the seven mode with 8 GPUs, use: + +``` bash +# enable training with distillation loss +./scripts/run_seven.sh nets/mobilenet_at_ilsvrc12_run.py -n=8 \ + --learner weight-sparse \ + --mobilenet_version 2 \ + --enbl_dst +``` diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml new file mode 100644 index 0000000..9a00639 --- /dev/null +++ b/docs/mkdocs.yml @@ -0,0 +1,30 @@ +site_name: PocketFlow Docs +nav: +- Home: index.md +- Installation: installation.md +- Tutorial: tutorial.md +- Learners - Algorithms: + - Channel Pruning: cp_learner.md + - Discrimination-aware Channel Pruning: dcp_learner.md + - Weight Sparsification: ws_learner.md + - Uniform Quantization: uq_learner.md + - Non-uniform Quantization: nuq_learner.md +- Learners - Misc.: + - Distillation: distillation.md + - Multi-GPU Training: multi_gpu_training.md +- Hyper-parameter Optimizers: + - Reinforcement Learning: reinforcement_learning.md + - AutoML-based Methods: automl_based_methods.md +- Performance: performance.md +- Frequently Asked Questions: faq.md +- Appendix: + - Pre-trained Models: pre_trained_models.md + - Test Cases: test_cases.md + - Reference: reference.md +theme: readthedocs + +markdown_extensions: + - pymdownx.arithmatex +extra_javascript: + - mathjax-config.js + - MathJax.js?config=TeX-AMS-MML_HTMLorMML From b66cb3ff5809248741429c789fc43d4b12a0e536 Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Thu, 15 Nov 2018 17:17:49 +0800 Subject: [PATCH 006/173] [Issue#52]Fix an bug of channel pruning (#65) * fix cp rl saver not found issue * make the number of batches smaller --- learners/channel_pruning/channel_pruner.py | 2 +- learners/channel_pruning/learner.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/learners/channel_pruning/channel_pruner.py b/learners/channel_pruning/channel_pruner.py index 522e7eb..b29dc10 100644 --- a/learners/channel_pruning/channel_pruner.py +++ b/learners/channel_pruning/channel_pruner.py @@ -45,7 +45,7 @@ achieve low flops with guaranted accuracy.''') tf.app.flags.DEFINE_integer('cp_nb_points_per_layer', 10, 'Sample how many point for each layer') -tf.app.flags.DEFINE_integer('cp_nb_batches', 60, +tf.app.flags.DEFINE_integer('cp_nb_batches', 30, 'Input how many bathes data into a model') diff --git a/learners/channel_pruning/learner.py b/learners/channel_pruning/learner.py index f1e7d6b..2b2b98d 100644 --- a/learners/channel_pruning/learner.py +++ b/learners/channel_pruning/learner.py @@ -693,9 +693,6 @@ def __prune_rl(self): # pylint: disable=too-many-locals strategy: {}, accuracy: {} and pruned ratio: {}""".format(self.bestinfo[0], self.bestinfo[1], self.bestinfo[2])) - with self.pruner.model.g.as_default(): - self.__save_in_progress_pruned_model() - #self.__save_best_pruned_model() tf.logging.info('automatic channl pruning time cost: {}s'.format(timer() - start)) From 766877dc60c51944a03bcf0f8220a9ebd8b583b8 Mon Sep 17 00:00:00 2001 From: Jiaxiang Wu Date: Thu, 15 Nov 2018 23:22:34 +0800 Subject: [PATCH 007/173] Fix a few documentation issues (#66) * add source files for documentation * add Python package dependencies * add README.md * issue #48 fixed * update learners' description; issue #50 fixed * update UQ-learner's hyper-param description; issue #42 fixed * update docs for UniformQuantLearner and NonUniformQuantLearner * typo fixed; remove trailing whitespaces --- docs/docs/nuq_learner.md | 178 +++++++++++++------------- docs/docs/tutorial.md | 32 ++--- docs/docs/uq_learner.md | 264 ++++++++++++++++++++++++--------------- 3 files changed, 268 insertions(+), 206 deletions(-) diff --git a/docs/docs/nuq_learner.md b/docs/docs/nuq_learner.md index 8f22cfc..98da143 100644 --- a/docs/docs/nuq_learner.md +++ b/docs/docs/nuq_learner.md @@ -1,109 +1,115 @@ # Non-Uniform Quantization Learner -This document describes how to set up the Non-Uniform Quantization Learner in PocketFlow. In non-uniform quantization, the quantization points are not distributed evenly, and can be optimized via the back-propagation of the network gradients. -Consequently, with the same number of bits, non-uniform quantization is more expressive to approximate the original full-precision network comparing to uniform quantization. - -Following a similar pattern in the previous sections, we first show how to configure the Non-Uniform Quantization Learner, followed by the algorithms used in the learner. - -### Prepare the Model -Again, users should first get the model prepared. Users can either use the pre-built models in PocketFlow, or develop their custom models according to [TODO](???). - -### Configure the Learner -To configure the learner, users can pass the options via the TensorFlow flag interface. The available options are as follows: - -| Options | Default Value | Description | -| :-------- | :--------:| :-- | -| `--nuql_opt_mode` | weight| variables to optimize: ['weights', 'clusters', 'both'] | -| `--nuql_init_style` | quantile | the initialization of quantization points: ['quantile', 'uniform'] | -| `--nuql_weight_bits` | 4 | the number of bits for weight | -| `--nuql_activation_bits` | 32 | the number of bits for activation， by default it remains full precision | -| `--nuql_save_quant_mode_path` | *TODO* | the save path for quantized models | -| `--nuql_use_buckets` | False | use bucketing or not | -| `--nuql_bucket_type` | channel | two bucket type available: ['split', 'channel'] | -| `--nuql_bucket_size` | 256 | quantize the first and last layers of the network or not | -| `--nuql_enbl_rl_agent` | False | enable reinforcement learning to learn the optimal bit allocation or not | -| `--nuql_quantize_all_layers` | False | quantize the first and last layers of the network or not | -| `--nuql_quant_epoch` | 60 | the number of epochs for fine-tuning | - -Note that since non-uniform quantization cannot be accelerated directly, by default we do not quantize the activations. - -### Examples -Once the model is built, the Non-Uniform Quantization Learner can be easily triggered by passing the Uniform Quantization Learner in the command line as follows: +Non-uniform quantization is a generalization to uniform quantization. In non-uniform quantization, the quantization points are not distributed evenly, and can be optimized via the back-propagation of the network gradients. Consequently, with the same number of bits, non-uniform quantization is more expressive to approximate the original full-precision network comparing to uniform quantization. Nevertheless, the non-uniform quantized model cannot be accelerated directly based on current deep learning frameworks, since the low-precision multiplication requires the intervals among quantization points to be equal. Therefore, the `NonUniformQuantLearner` can only help better compress the model. + +## Algorithm + +`NonUniformQuantLearner` adopts a similar training and evaluation procedure to the `UniformQuantLearner`. In the training process, the quantized weights are forwarded, while in the backward pass, full precision weights are updated via the STE estimator. The major difference from uniform quantization is that the locations of quantization points are not evenly distributed, but can be optimized and initialized differently. In the following, we introduce the scheme to the update and initialization of quantization points. + +### Optimization the quantization points + +Unlike uniform quantization, non-uniform quantization can optimize the location of quantization points dynamically during the training of the network, and thereon leads to less quantization loss. The location of quantization points can be updated by summing the gradients of weights that fall into the point ([Han et.al 2015](https://arxiv.org/abs/1510.00149)), i.e.: +$$ +\frac{\partial \mathcal{L}}{\partial c_k} = \sum_{i,j}\frac{\partial\mathcal{L}}{\partial w_{ij}}\frac{\partial{w_{ij}}}{\partial c_k}=\sum_{ij}\frac{\partial\mathcal{L}}{\partial{w_{ij}}}1(I_{ij}=k) +$$ +The following figure taken from [Han et.al 2015](https://arxiv.org/abs/1510.00149) shows the above process of updating the clusters: + +![Deep Compression Algor](D:/OneDrive%20-%20The%20Chinese%20University%20of%20Hong%20Kong/Research/MyWorks/automc/doc/pocketflow-docs/docs/pics/deep_compression_algor.png) + +### Initialization of quantization points + +Aside from optimizing the quantization points, another helpful strategy is to properly initialize the quantization points according to the distribution of weights. PocketFlow currently supports two kinds of initialization: + +- Uniform initialization: The quantization points are initialized to be evenly distributed along the range $[w_{min}, w_{max}]$ of that layer/bucket. +- Quantile initialization: The quantization points are initialized to be the quantiles of full-precision weights. Comparing to uniform initialization, quantile initialization can generally lead to better performance. + +## Hyper-parameters + +To configure `NonUniformQuantLearner`, users can pass the options via the TensorFlow flag interface. The available options are as follows: + +| Options | Description | +| :-------------------------- | :----------------------------------------------------------- | +| `nuql_opt_mode` | the fine-tuning mode: [`weights`, `clusters`, `both`]. Default: `weight` | +| `nuql_init_style` | the initialization of quantization point: [`quantile`, `uniform`]. Default: `quantile`. | +| `nuql_weight_bits` | the number of bits for weight. Default: `4`. | +| `nuql_activation_bits` | the number of bits for activation. Default: `32`. | +| `nuql_save_quant_mode_path` | the save path for quantized models. Default: `./nuql_quant_models/model.ckpt` | +| `nuql_use_buckets` | the switch to quantize first and last layers of network. Default: `False`. | +| `nuql_bucket_type` | two bucket type available: ['split', 'channel']. Default: `channel`. | +| `nuql_bucket_size` | the number of bucket size for bucket type 'split'. Default: `256. | +| `nuql_enbl_rl_agent` | the switch to enable RL to learn optimal bit strategy. Default:`False`. | +| `nuql_quantize_all_layers` | the switch to quantize first and last layers of network. Default: `False`. | +| `nuql_quant_epoch` | the number of epochs for fine-tuning. Default: `60`. | + +Here, we provide detailed description (and some analysis) for some of the above hyper-parameters: + +- `nuql_opt_mode`: the mode for fine-tuning the non-uniformly quantized network, choose among [`weights`, `clusters`, `both`]. `weight` refers to only updating the network weights, while `clusters` refers to only updating the quantization points, and `both` means updating weights and quantization points simultaneously. Experimentally, we found that `weight` and `both` achieve similar performance, both of which outperform `clusters`. +- `nuql_init_style`: the style of initialization of quantization points, currently supports [`quantile`, `uniform`]. The differences between the two strategies have been discussed earlier. +- `nuql_weight_bits`: The number of bits for weight quantization. Generally, for lower bit quantization (e.g., 2 bit on CIFAR10 and 4 bit on ILSVRC_12), `NonUniformQuantLearner` performs much better than `UniformQuantLearner`. The gap becomes less when using higher bits. +- `nuql_activation_bits`: The number of bits for activation quantization. Since non-uniform quantized models cannot be accelerated directly, by default we leave it as 32 bit. +- `nuql_save_quant_mode_path`: the path to save the quantized model. Quantization nodes have already been inserted into the graph. +- `nuql_use_buckets`: the switch to turn on the bucket. With bucketing, weights are split into multiple pieces, while the $\alpha$ and $\beta$ are calculated individually for each piece. Therefore, turning on the bucketing can lead to more fine-grained quantization. +- `nuql_bucket_type`: the type of bucketing. Currently two types are supported: [`split`, `channel`]. `split` refers to that the weights of a layer are first concatenated into a long vector, and then cut it into short pieces according to `uql_bucket_size`. The remaining last piece is still regarded as a new piece. After quantization for each piece, the vectors are then folded back to the original shape as the quantized weights. `channel` refers to that weights with shape `[k, k, cin, cout]` in a convolutional layer are cut into `cout` buckets, where each bucket has the size of `k * k * cin`. For weights with shape `[m, n]` in fully connected layers, they are cut into `n` buckets, each of size `m`. In practice, bucketing with type `channel` can be calculated more quickly comparing to type `split` since there are less buckets and less computation to iterate through all buckets. +- `nuql_bucket_size`: the size of buckets when using bucket type `split`. Generally, smaller bucket size can lead to more fine grained quantization, while more storage are required since full precision statistics ($\alpha$ and $\beta$) of each bucket need to be kept. +- `nuql_quantize_all_layers`: the switch to quantize the first and last layers. The first and last layers of the network are connected directly with the input and output, and are arguably more sensitive to quantization. Keeping them un-quantized can slightly increase the performance, nevertheless, if you want to accelerate the inference speed, all layers are supposed to be quantized. +- `nuql_quant_epoch`: the epochs for fine-tuning a quantized network. +- `nuql_enbl_rl_agent`: the switch to turn on the RL agent as hyper parameter optimizer. Details about the RL agent and its configurations are described below. + +### Configure the RL Agent + +Similar to uniform quantization, once `nuql_enbl_rl_agent==True` , the RL agent will automatically search for the optimal bit allocation strategy for each layer. In order to search efficiently, the agent need to be configured properly. While here we list all the configurable hyper parameters for the agent, users can just keep the default value for most parameters, while modify only a few of them if necessary. + +| Options | Description | +| :---------------------------- | :----------------------------------------------------------- | +| `nuql_evquivalent_bits` | the number of re-allocated bits that is equivalent to non-uniform quantization without RL agent. Default: `4`. | +| `nuql_nb_rlouts` | the number of roll outs for training the RL agent. Default: `200`. | +| `nuql_w_bit_min` | the minimal number of bits for each layer. Default: `2`. | +| `nuql_w_bit_max` | the maximal number of bits for each layer. Default: `8`. | +| `nuql_enbl_rl_global_tune` | the switch to fine-tune all layers of the network. Default: `True`. | +| `nuql_enbl_rl_layerwise_tune` | the switch to fine-tune the network layer by layer. Default: `False`. | +| `nuql_tune_layerwise_steps` | the number of steps for layer-wise fine-tuning. Default: `300`. | +| `nuql_tune_global_steps` | the number of steps for global fine-tuning. Default: `2000`. | +| `nuql_tune_disp_steps` | the display steps to show the fine-tuning progress. Default: `100`. | +| `nuql_enbl_random_layers` | the switch to randomly permute layers during RL agent training. Default: `True`. | + +Detailed description can be found in [Uniform Quantization](https://pocketflow.github.io/uq_learner/), with the only difference that the prefix is changed to `nuql_`. + +## Usage Examples + +Again, users should first get the model prepared. Users can either use the pre-built models in PocketFlow, or develop their customized nets following the model definition in PocketFlow (for example, [resnet_at_cifar10.py](https://github.com/Tencent/PocketFlow/blob/master/nets/resnet_at_cifar10.py)) Once the model is built, the Non-Uniform Quantization Learner can be easily triggered as follows: + +To quantize a ResNet-20 model for CIFAR-10 classification task with 4 bits in the local mode, use: + ```bash # quantize resnet-20 on CIFAR-10 -# you can also configure the sh ./scripts/run_local.sh nets/resnet_at_cifar10_run.py \ ---data_disk local \ ---data_dir_local ${PF_CIFAR10_LOCAL} \ --learner=non-uniform \ --nuql_weight_bits=4 \ --nuql_activation_bits=4 \ +``` +To quantize a ResNet-18 model for ILSVRC_12 classification task with 8 bits in the docker mode with 4 GPUs, and allow to use the channel-wise bucketing, use: + +``` bash # quantize the resnet-18 on ILSVRC-12 -sh ./scripts/run_local.sh nets/resnet_at_ilsvrc12_run.py \ ---learner=uniform \ ---data_disk local \ ---data_dir_local ${PF_ILSVRC12_LOCAL} \ +sh ./scripts/run_docker.sh nets/resnet_at_ilsvrc12_run.py \ +-n=4 \ +--learner=non-uniform \ --nuql_weight_bits=8 \ --nuql_activation_bits=8 \ --nuql_use_buckets=True \ --nuql_bucket_type=channel ``` -To enable the RL agent, one can follow similar patterns as those in the Uniform Quantization Learner: +To quantize a MobileNet-v1 model for ILSVRC_12 classification task with 4 bits in the seven mode with 8 GPUs, and allow the RL agent to search for the optimal bit strategy, use: + ```bash # quantize mobilenet-v1 on ILSVRC-12 -sh ./scripts/run_local.sh nets/mobilnet_at_ilsvrc12_run.py \ ---data_disk local \ ---data_dir_local ${PF_CIFAR10_LOCAL} \ ---learner=uniform \ +sh ./scripts/run_seven.sh nets/mobilnet_at_ilsvrc12_run.py \ +-n=8 \ +--learner=non-uniform \ --nuql_enbl_rl_agent=True \ --nuql_equivalent_bits=4 \ ---nuql_tune_global_steps=1200 ``` -### Performance -Here we list some of the performance on Cifar-10 using the Non-Uniform Quantization Learner and the built-in models in PocketFlow. The options not displayed remain the default values. - - -| Model | Weight Bit| Activation Bit | Acc | -| :--------: |:--------:| :--: | :--:| -| ResNet-20 | 32 | 32 | 91.96 | -| ResNet-20 | 2 | 4 | 90.31 | -| ResNet-20 | 4 | 8 | 91.70 | - - -| Model | Weight Bit| Bucketing | Acc | -| :--------: | :--: |:--------:| :--: | -| ResNet-20 | 2 | channel | 90.90 | -| ResNet-20 | 4 | channel | 91.97 | -| ResNet-20 | 2 | split | 90.02 | -| ResNet-20 | 4 | split | 91.56 | - - -| Model | Weight Bit| RL search | Acc | -| :--------: | :--: |:--------:| :--: | -| ResNet-20 | 2 | FALSE | 90.31 | -| ResNet-20 | 4 | FALSE | 91.70 | -| ResNet-20 | 2 | TRUE| 90.60 | -| ResNet-20 | 4 | TRUE | 91.79 | - -## Algorithm -Non-Uniform Quantization Learner adopts a similar training and evaluation procedure to the Uniform Quantization. In the training process, the quantized weights are forwarded. In the backward pass, the full precision weights are updated via the STE estimator. The major difference from uniform quantization is that, the location of quantization points are not evenly distributed, but can be optimized and initialized differently. In the following, we introduce the scheme to update and initialize the quantization points. - -### Optimization the quantization points -Unlike uniform quantization, non-uniform quantization can optimize the location of quantization points dynamically during the training of the network, and thereon leads to less quantization loss. The location of quantization points can be updated by summing the gradients of weights that fall into the point ([Han et.al 2015](https://arxiv.org/abs/1510.00149)), i.e.,: -$$ -\frac{\partial \mathcal{L}}{\partial c_k} = \sum_{i,j}\frac{\partial\mathcal{L}}{\partial w_{ij}}\frac{\partial{w_{ij}}}{\partial c_k}=\sum_{ij}\frac{\partial\mathcal{L}}{\partial{w_{ij}}}1(I_{ij}=k) -$$ - -The following figure taken from [Han et.al 2015](https://arxiv.org/abs/1510.00149) shows the process of updating the clusters: - -![Deep Compression Algor](pics/deep_compression_algor.png) - -### Initialization of quantization points -Aside from optimizing the quantization points, another helpful strategy is to properly initialize the quantization points according to the distribution of weights. PocketFlow currently supports two kinds of initialization: uniform initialization and quantile initialization. Comparing to uniform initialization, quantile initialization uses the quantiles of weights as the initial locations of quantization points. Quantile initialization considers the distribution of weights and can usually lead to better performance. - - - ## References Han S, Mao H, and Dally W J. Deep compression: Compressing deep neural networks with pruning, trained quantization and huffman coding. [arXiv:1510.00149, 2015](https://arxiv.org/abs/1510.00149) diff --git a/docs/docs/tutorial.md b/docs/docs/tutorial.md index f39a5d4..f776d20 100644 --- a/docs/docs/tutorial.md +++ b/docs/docs/tutorial.md @@ -4,14 +4,14 @@ In this tutorial, we demonstrate how to compress a convolutional neural network ## Prepare the Data -To start with, we need to convert the ImageNet data set (ILSVRC-12) into TensorFlow's native TFRecord file format. You may follow the data preparation guide [here](https://github.com/tensorflow/models/tree/master/research/inception#getting-started) to download the full data set and convert it into TFRecord files. After that, you should be able to find 4,096 training files and 128 validation files in the data directory, like this: +To start with, we need to convert the ImageNet data set (ILSVRC-12) into TensorFlow's native TFRecord file format. You may follow the data preparation guide [here](https://github.com/tensorflow/models/tree/master/research/inception#getting-started) to download the full data set and convert it into TFRecord files. After that, you should be able to find 1,024 training files and 128 validation files in the data directory, like this: ``` bash # training files -train-00000-of-04096 -train-00001-of-04096 +train-00000-of-01024 +train-00001-of-01024 ... -train-04095-of-04096 +train-01023-of-01024 # validation files validation-00000-of-00128 @@ -59,15 +59,15 @@ $ ./scripts/run_seven.sh nets/resnet_at_ilsvrc12_run.py -n=8 \ Let's take the execution command for the local mode as an example. In this command, `run_local.sh` is a shell script that executes the specified Python script with user-provided arguments. Here, we ask it to run the Python script named `nets/resnet_at_ilsvrc12_run.py`, which is the execution script for ResNet models on the ImageNet data set. After that, we use `--learner dis-chn-pruned` to specify that the `DisChnPrunedLearner` should be used for model compression. You may also use other learners by specifying the corresponding learner name. Below is a full list of available learners in PocketFlow: -| Learner name | Learner class | Note | -|:-----------------|:-----------------------------|:---------------------------------------------| -| `full-prec` | `FullPrecLearner` | No model compression | -| `channel` | `ChannelPruningLearner` | Channel pruning [2] | -| `dis-chn-pruned` | `DisChnPrunedLearner` | Discrimination-aware channel pruning [1] | -| `weight-sparse` | `WeightSparseLearner` | Weight sparsification [3] | -| `uniform` | `UniformQuantizedLearner` | Uniform weight quantization [] | -| `tf-uniform` | `UniformQuantizedLearnerTF` | Uniform weight quantization in TensorFlow [] | -| `non-uniform` | `NonUniformQuantizedLearner` | Non-uniform weight quantization [] | +| Learner name | Learner class | Note | +|:-----------------|:-------------------------|:------------------------------------------------------------------------------| +| `full-prec` | `FullPrecLearner` | No model compression | +| `channel` | `ChannelPrunedLearner` | Channel pruning with LASSO-based channel selection (He et al., 2017) | +| `dis-chn-pruned` | `DisChnPrunedLearner` | Discrimination-aware channel pruning (Zhuang et al., 2018) | +| `weight-sparse` | `WeightSparseLearner` | Weight sparsification with dynamic pruning schedule (Zhu & Gupta, 2017) | +| `uniform` | `UniformQuantLearner` | Weight quantization with uniform reconstruction levels (Jacob et al., 2018) | +| `uniform-tf` | `UniformQuantTFLearner` | Weight quantization with uniform reconstruction levels and TensorFlow APIs | +| `non-uniform` | `NonUniformQuantLearner` | Weight quantization with non-uniform reconstruction levels (Han et al., 2016) | The local mode only uses 1 GPU for the training process, which takes approximately 20-30 hours to complete. This can be accelerated by multi-GPU training in the docker and seven mode, which is enabled by adding `-n=x` right after the specified Python script, where `x` is the number of GPUs to be used. @@ -233,9 +233,3 @@ public void onActivityCreated(Bundle savedInstanceState) { startBackgroundThread(); } ``` - -## Reference - -* [1] Zhuangwei Zhuang, Mingkui Tan, Bohan Zhuang, Jing Liu, Jiezhang Cao, Qingyao Wu, Junzhou Huang, Jinhui Zhu, *Discrimination-aware Channel Pruning for Deep Neural Networks*, In Proc. of the Annual Conference on Neural Information Processing Systems (NIPS), 2018. -* [2] Yihui He, Xiangyu Zhang, Jian Sun, *Channel Pruning for Accelerating Very Deep Neural Networks*, In Proc. of the IEEE International Conference on Computer Vision (ICCV), 2017. -* [3] Michael Zhu, Suyog Gupta, *To Prune, or Not to Prune: Exploring the Efficacy of Pruning for Model Compression*, CoRR, abs/1710.01878, 2017. diff --git a/docs/docs/uq_learner.md b/docs/docs/uq_learner.md index 0eebe77..3159bd5 100644 --- a/docs/docs/uq_learner.md +++ b/docs/docs/uq_learner.md @@ -1,13 +1,15 @@ # Uniform Quantization -This document describes how to set up uniform quantization with PocketFlow. Uniform quantization is widely used for model compression and acceleration. Originally the weights in the network are represented by 32-bit float numbers. With uniform quantization, low-precision (e.g., 4 bit, 8 bit) and evenly distributed float numbers are used to approximate the full precision networks. For 8 bit quantization, the network size can be reduced by 4 folds with little drop of performance. +## Introduction - Currently PocketFlow supports two types of uniform quantization: +Uniform quantization is widely used for model compression and acceleration. Originally the weights in the network are represented by 32-bit float numbers. With uniform quantization, low-precision (e.g., 4 bit, 8 bit) and evenly distributed float numbers are used to approximate the full precision networks. For $k$-bit quantization, the memory saving can be up to $32/k$. For example, 8 bit quantization reduce the network size by 4 folds with little drop of performance. -* Uniform Quantization Learner: the self-developed learner. Aside from uniform quantization, the learner is carefully optimized with various extensions supported. The detailed algorithm of the Uniform Quantized Learner will be introduced at the end of the algorithm. The algorithm details are presented at the end of the document. + Currently PocketFlow supports two types of uniform quantization learner: -* TensorFlow Quantization Wrapper: a wrapper based on the [post training quantization](https://www.tensorflow.org/performance/post_training_quantization) in TensorFlow. The wrapper currently only supports 8-bit quantization, enjoying 4x reduction of memory and nearly 4x times speed up of inference. +* `UniformQuantLearner`: the self-developed learner for uniform quantization. The learner is carefully optimized with various extensions and variations supported. + +* `UniformQuantTFLearner`: a wrapper based on the [quantization aware training](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/quantize) in TensorFlow. The wrapper currently only supports 8-bit quantization, enjoying 4x reduction of memory and approximately 3x times speed up of inference. A comparison of the two learners are shown below: @@ -19,146 +21,206 @@ A comparison of the two learners are shown below: | Bucketing | Yes | | | Hyper-param Searching| Yes | | +## Algorithm + +Both uniform quantization learners generally obey the following procedures: + +Given a pre-defined full precision model, the learners insert quantization nodes into the computation graph of the model. To enable activation quantization, quantization nodes will also be placed after the activation operations (e.g., ReLu). + +In the training phase, both full-precision and quantized weights are kept. In the forward pass, quantized weights are obtained by applying the quantization functions on the full precision weights. To update the weights in the backward pass, since the gradients w.r.t. the quantized weights are 0 almost everywhere, we use the straight-through estimator (STE) ([Hinton et.al 2012](https://www.coursera.org/learn/neural-networks), [Bengio et.al 2013](https://arxiv.org/abs/1308.3432)) to pass the gradient of quantized weights directly to the full precision weights for update. + + ![Train_n_Inference](D:/OneDrive%20-%20The%20Chinese%20University%20of%20Hong%20Kong/Research/MyWorks/automc/doc/pocketflow-docs/docs/pics/train_n_inference.png) + +### Uniform Quantization Function + +Uniform quantization distribute the quantization points evenly across the range $[w_{min}, w_{max}]$, where $w_{max}, w_{min}$ are the maximum and minimum value of weights in the layer/bucket. Then the original full precision weights are then assigned to the closest quantization point. To achieve this, we first normalize the full precision weights $x$ of one layer to $[0, 1]$, i.e., +$$ +sc(x) = \frac{x-\beta}{\alpha}, +$$ +where $\alpha=w_{max}-w_{min}$ and $\beta = w_{min}$ are the scaling factors. Then we assign $sc(x)$ to the discrete value by +$$ +\hat{x}=\frac{1}{2^k-1}\mathrm{round}((2^k-1)\cdot sc(x)), +$$ +and finally we do the inverse linear transformation to recover the quantized weights to the original scale, +$$ +Q(x)=\alpha\hat{x}+\beta. +$$ + + +Next we introduce usages for the two uniform quantization learners. + +## 1. UniformQuantLearner + +`UniformQuantLearner` is the self-developed learner, which allows a number of customized configurations for uniform quantization. For example, the learner supports bucketing, leading to more fine-grained quantization and better performance. The learner also allows to allocate different bits across layers, in which users can turn on the hyper parameter optimizer with reinforcement learning to search for the optimal bit allocation for the quantization. + +### Hyper-parameters + +To configure `UniformQuantLearner`, users can pass the options via the TensorFlow flag interface. The available options are as follows: + +| Options | Description | +| :-------- | :-- | +| `uql_weight_bits` | the number of bits for weights. Default: `4`. | +| `uql_activation_bits` | the number of bits for activation. Default: `32.` | +| `uql_save_quant_model_path` | quantized model's save path. Default: `./uql_quant_models/model.ckpt` | +| `uql_use_buckets` | the switch to use bucketing. Default: `False.` | +| `uql_bucket_type` | two bucket type available: [`split`, `channel`]. Default: `channel.` | +| `uql_bucket_size` | the number of bucket size for bucket type `split`. Default: `256`. | +| `uql_quantize_all_layers` | the switch to quantize first and last layers of network. Default: `False.` | +| `uql_quant_epoch` | the number of epochs for fine-tuning. Default: `60`. | +| `uql_enbl_rl_agent` | the switch to enable RL to learn optimal bit strategy. Default:`False`. | + +Here, we provide detailed description (and some analysis) for above hyper-parameters: + +- `uql_weight_bits`: The number of bits for weight quantization. Generally, 8 bit does not hurt the model performance while it can compress the model size by 4 folds. While 2 bit and 4 bit could lead to drop of performance on large datasets such as Imagenet. +- `uql_activation_bits`: The number of bits for activation quantization. When both weights and activations are quantized, 8 bit does not lead to apparent drop of performance, and sometimes can even increase the classification accuracy, which is probably due to better generalization ability. Nevertheless, the performance will be more challenged when both weights and activations are quantized to lower bits, comparing to weight-only quantization. +- `uql_save_quant_mode_path`: the path to save the quantized model. Quantization nodes have already been inserted into the graph. +- `uql_use_buckets`: the switch to turn on the bucket. With bucketing, weights are split into multiple pieces, while the $\alpha$ and $\beta$ are calculated individually for each piece. Therefore, turning on the bucketing can lead to more fine-grained quantization. +- `uql_bucket_type`: the type of bucketing. Currently two types are supported: [`split`, `channel`]. `split` refers to that the weights of a layer are first concatenated into a long vector, and then cut it into pieces according to `uql_bucket_size`. The remaining last piece will be padded and taken as a new piece. After quantization of each piece, the vectors are then folded back to the original shape as the quantized weights. `channel` refers to that weights with shape `[k, k, cin, cout]` in a convolutional layer are cut into `cout` buckets, where each bucket has the size of `k * k * cin`. For weights with shape `[m, n]` in fully connected layers, they are cut into `n` buckets, each of size `m`. In practice, bucketing with type `channel` can be calculated more efficiently comparing to type `split` since there are less buckets and less computation to iterate through all of them. +- `uql_bucket_size`: the size of buckets when using bucket type `split`. Generally, smaller bucket size can lead to more fine grained quantization, while more storage are required since full precision statistics ($\alpha$ and $\beta$) of each bucket need to be kept. +- `uql_quantize_all_layers`: the switch to quantize the first and last layers. The first and last layers of the network are connected directly with the input and output, and are arguably more sensitive to quantization. Keeping them un-quantized can slightly increase the performance, nevertheless, if you want to accelerate the inference speed, all layers are supposed to be quantized. +- `uql_quant_epoch`: the epochs for fine-tuning a quantized network. +- `uql_enbl_rl_agent`: the switch to turn on the RL agent as hyper parameter optimizer. Details about the RL agent and its configurations are described below. + +### Configure the RL Agent + +Once the hyper parameter optimizer is turned on, i.e., `uql_enbl_rl_agent==True` , the RL agent will automatically search for the optimal bit allocation strategy for each layer. In order to search efficiently, the agent need to be configured properly. While here we list all the configurable hyper parameters for the agent, users can just keep the default value for most parameters, while modify only a few of them if necessary. -## Uniform Quantization Learner -The uniform quantization learner supports both weight quantization and activation quantization, where users can manually set up the bits for quantization. The uniform quantization learner also supports bucketing, which leads to more fine-grained quantization and better performance. The users can also turn on the hyper parameter optimizer with reinforcement learning to search for the optimal bit allocation for the learner. +| Options | Description | +| :--------------------------- | :----------------------------------------------------------- | +| `uql_evquivalent_bits` | the number of re-allocated bits that is equivalent to uniform allocation of bits. Default: `4`. | +| `uql_nb_rlouts` | the number of roll outs for training the RL agent. Default: `200`. | +| `uql_w_bit_min` | the minimal number of bits for each layer. Default: `2`. | +| `uql_w_bit_max` | the maximal number of bits for each layer. Default: `8`. | +| `uql_enbl_rl_global_tune` | the switch to fine-tune all layers of the network. Default: `True`. | +| `uql_enbl_rl_layerwise_tune` | the switch to fine-tune the network layer by layer. Default: `False`. | +| `uql_tune_layerwise_steps` | the number of steps for layer-wise fine-tuning. Default: `300`. | +| `uql_tune_global_steps` | the number of steps for global fine-tuning. Default: `2000`. | +| `uql_tune_disp_steps` | the display steps to show the fine-tuning progress. Default: `100`. | +| `uql_enbl_random_layers` | the switch to randomly permute layers during RL agent training. Default: `True`. | -### Prepare the Model -To quantize the network, users should first get the model prepared. Users can either use the pre-built models in PocketFlow, or develop their custom models according to [TODO](???). +Detailed description and usages for above hyper-parameters are listed below: -### Configure the Learner +- `uql_equivalent_bits`: the total number of bits used in the optimal strategy will not exceed $n_{param}*$`uql_equivalent_bits` . For example, by setting `uql_equivalent_bits`=4, the RL agent will try to find the best quantization strategy with the same compression ratio to that each layer is quantized by 4 bits. -To configure the learner, users can pass the options via the TensorFlow flag interface. The available options are as follows: +The following parameters can be kept in default value in most cases. Users can also modify them when using their customized models if necessary. -| Options | Default Value | Description | -| :-------- | :--------:| :-- | -| `--uql_weight_bits` | 4 | the number of bits for weight | -| `--uql_activation_bits` | 32 | the number of bits for activation， by default it remains full precision | -| `--uql_save_quant_mode_path` | *TODO* | the save path for quantized models | -| `--uql_use_buckets` | False | use bucketing or not | -| `--uql_bucket_type` | channel | two bucket type available: ['split', 'channel'] | -| `--uql_bucket_size` | 256 | quantize the first and last layers of the network or not | -| `--uql_enbl_rl_agent` | False | enable reinforcement learning to learn the optimal bit allocation or not | -| `--uql_quantize_all_layers` | False | quantize the first and last layers of the network or not | -| `--uql_quant_epoch` | 60 | the number of epochs for fine-tuning | +- `uql_nb_rlouts`: the number of roll-out for training the RL agent. Generally we will use the first quarter of `uql_nb_rlouts` for collection of the training buffer, and last three quarters for the training of the agent. The larger the `uql_nb_rlouts`, the slower the search for the hyper-parameter optimizer. +- `uql_w_bit_min`: the minimum number of quantization bit for a layer. This is used to constrain the searching space and avoid extreme strategies that crash the entire performance of the compressed model. +- `uql_w_bit_max`: the maximum number of quantization bit for a layer. This is used to constrain the searching space and avoid that one layer may use too much unnecessary bits. +- `uql_enbl_rl_global_tune`: the switch to globally fine-tune the network in each roll-out, which is done by updating the full-precision weights for all layers via the STE estimator. The aim of the fine-tune is to obtain effective reward from the current strategy. +- `uql_enbl_rl_layerwise_tune`: the switch to layer-wise fine-tune the network in each roll-out, which is done by minimizing the l2-norm between the quantized layer and full-precision layer. +- `uql_tune_layerwise_steps`: the number of steps for layer-wise fine-tuning. Generally, the larger the value, the more precise the reward and thereon the better the strategy. +- `uql_tune_global_steps`: the number of steps for global fine-tuning. Generally, the larger the value, the more precise the reward and thereon the better the strategy. +- `uql_tune_disp_steps`: the intervals to display the global training process in each roll-out. +- `uql_enbl_random_layers` : the switch to randomly permute layers of the network when searching the optimal strategy. This could be helpful since the bit budget used in previous layers may affect the searching space for following layers, while randomly shuffling all layers makes sure that all layers have equal probability of all strategies. + +### Usage Examples + +In this section, we provide some usage examples to demonstrate how to use `UniformQuantLearner`under different execution modes and hyper-parameter combinations. + +To quantize the network, users should first get the model prepared. Users can either use the pre-built models in PocketFlow, or develop their customized nets following the model definition in PocketFlow (for example, [resnet_at_cifar10.py](https://github.com/Tencent/PocketFlow/blob/master/nets/resnet_at_cifar10.py)). Once the model is built, the quantization can be easily triggered by directly as follows: + +To quantize a ResNet-20 model for CIFAR-10 classification task with 4 bits in the local mode, use: -### Examples -Once the model is built, the quantization can be easily triggered by directly passing the Uniform Quantization Learner in the command line as follows: ```bash # quantize resnet-20 on CIFAR-10 -# you can also configure the sh ./scripts/run_local.sh nets/resnet_at_cifar10_run.py \ ---data_disk local \ ---data_dir_local ${PF_CIFAR10_LOCAL} \ --learner=uniform \ --uql_weight_bits=4 \ --uql_activation_bits=4 \ +``` + +To quantize a ResNet-18 model for ILSVRC_12 classification task with 8 bits in the docker mode with 4 GPUs, and allow to use the channel-wise bucketing, use: +``` bash # quantize the resnet-18 on ILSVRC-12 -sh ./scripts/run_local.sh nets/resnet_at_ilsvrc12_run.py \ +sh ./scripts/run_docker.sh nets/resnet_at_ilsvrc12_run.py \ +-n=4 \ --learner=uniform \ ---data_disk local \ ---data_dir_local ${PF_ILSVRC12_LOCAL} \ --uql_weight_bits=8 \ --uql_activation_bits=8 \ --uql_use_buckets=True \ --uql_bucket_type=channel ``` -### Configure the Hyper Parameter Optimizer -Once the hyper parameter optimizer is turned on, i.e., `uql_enbl_rl_agent==True` , the reinforcement learning agents will search for the optimal allocation of bits to each layers. Before the search, users are supposed set up the bit constraints via `--uql_evquivalent_bits`, so that the optimal bits searched by the RL agent will not exceed the bit number without RL agent. -*For example, TODO* - -Users can also configure other options in the RL agent, such as the number of roll-outs, the fine-tuning steps to get the reward, e.t.c.. Full list of options are listed as follows: - -| Options | Default Value | Description | -| :-------- | :-------:| :-- | -| `--uql_evquivalent_bits` | 4 | the number of re-allocated bits that is equivalent to uniform quantization without RL agent | -| `--uql_nb_rlouts` | 200 | the number of roll outs for training the RL agent | -| `--uql_w_bit_min` | 2 | the minimal number of bits for each layer | -| `--uql_w_bit_max` | 8| the maximal number of bits for each layer | -| `--uql_enbl_rl_global_tune` | True | enable fine-tuning all layers of the network or not | -| `--uql_enbl_rl_layerwise_tune` | False | enable fine-tuning the network layer by layer or not | -| `--uql_tune_layerwise_steps` | 100 | the number of steps for layerwise fine-tuning | -| `--uql_tune_global_steps` | 2000 | the number of steps for global fine-tuning | -| `--uql_tune_disp_steps` | 300 | the display steps to show the fine-tuning progress | -| `--uql_enbl_random_layers` | True | randomly permute the layers during RL agent training | - -### Examples +To quantize a MobileNet-v1 model for ILSVRC_12 classification task with 4 bits in the seven mode with 8 GPUs, and allow the RL agent to search for the optimal bit strategy, use: + ```bash # quantize mobilenet-v1 on ILSVRC-12 -sh ./scripts/run_local.sh nets/mobilnet_at_ilsvrc12_run.py \ ---data_disk local \ ---data_dir_local ${PF_CIFAR10_LOCAL} \ +sh ./scripts/run_seven.sh nets/mobilnet_at_ilsvrc12_run.py \ +-n=8 \ --learner=uniform \ --uql_enbl_rl_agent=True \ --uql_equivalent_bits=4 \ ---uql_tune_global_steps=1200 ``` -### Performance -Here we list some of the performance on Cifar-10 using the Uniform Quantization Learner and the built-in models in PocketFlow. The options not displayed remain the default values. -| Model | Weight Bit| Activation Bit | Acc | -| :--------: |:--------:| :--: | :--:| -| ResNet-20 | 32 | 32 | 91.96 | -| ResNet-20 | 4 | 4 | 90.73 | -| ResNet-20 | 8 | 8 | 92.25 | -| Model | Weight Bit| Bucketing | Acc | -| :--------: | :--: |:--------:| :--: | -| ResNet-20 | 2 | channel | 89.67 | -| ResNet-20 | 4 | channel | 92.02 | -| ResNet-20 | 2 | split | 91.15 | -| ResNet-20 | 4 | split | 91.98 | +## 2. UniformQuantTFLearner + +PocketFlow also wraps the quantization aware training in TensorFlow. The quantized model can be directly exported to `.tflite` format via [export_quant_tflite_model.py](https://github.com/haolibai/PocketFlow/blob/master/tools/conversion/export_quant_tflite_model.py) in PocketFlow, and then be easily deployed on Andriod devices. -| Model | Weight Bit| RL search | Acc | -| :--------: | :--: |:--------:| :--: | -| ResNet-20 | 2 | FALSE | 86.17 | -| ResNet-20 | 4 | FALSE | 91.76 | -| ResNet-20 | 2 | TRUE| 90.30 | -| ResNet-20 | 4 | TRUE | 91.88 | +To configure `UniformQuantTFLearner`, the hyper-parameters are as follows: -## TensorFlow Quantization Wrapper -PocketFlow wraps the post training quantization in Tensorflow, and include all the necessary steps to convert the model to the .tflite format, which can be deployed on Andriod devices. To run the wrapper, users only need to get the checkpoint files ready, and then run the script. +| Options | Description | +| ---------------------- | ------------------------------------------------------------ | +| `uqtf_save_path` | UQ-TF: model\'s save path. Default: `./models_uqtf/model.ckpt`. | +| `uqtf_save_path_eval` | UQ-TF: model\'s save path for evaluation. Default: `./models_uqtf_eval/model.ckpt`. | +| `uqtf_weight_bits` | UQ-TF: # of bits for weight quantization. Default: `8`. | +| `uqtf_activation_bits` | UQ-TF: # of bits for activation quantization. Default: `8`. | +| `uqtf_quant_delay` | UQ-TF: # of steps after which weights and activations are quantized. Default: `0`. | +| `uqtf_freeze_bn_delay` | UT-TF: # of steps after which moving mean and variance are frozen. Default: `None`. | +| `uqtf_lrn_rate_dcy` | UQ-TF: learning rate\'s decaying factor. Default: `1e-2`. | -### Prepare the Checkpoint Files -Generally in TensorFlow, the checkpoints of a model include three files: .data, .index, .meta. Users are also supposed to add the input and output to collections, and configure the wrapper to acquire the corresponding collections. An example is as follows: +Here, the detailed description (and some analysis) for some above hyper-parameters are listed as follows: + +- `uqtf_quant_delay`: The number of steps to start fine-tuning on the quantized network. Before the training step reaches `uqtf_quant_delay`, only full precision weights of the model are updated. +- `uqtf_freeze_bn_delay`: The number of steps after which the moving mean and variance of batch normalization layers are frozen and used, instead of the batch statistics during training. +- `uqtf_lrn_rate_dcy` : The decay of learning rate for the quantized model. Generally the quantized network needs smaller learning rate comparing to that for the full-precision model. + +### Usage Examples + +To deploy a quantized network on Andriod devices, there are generally 3 steps: + +#### Quantize the pre-trained network + +To quantize a MobileNet-v1 model for ILSVRC_12 classification task with 8 bits in the seven mode, use: + +``` bash +# quantize MobileNet-v1 on ILSVRC_12 +sh ./scripts/run_seven.sh nets/mobilnet_at_ilsvrc12_run.py \ +-n=8 \ +--learner=uniform-tf \ +--uqtf_weight_bits=8 \ +--uqtf_activation_bits=8 \ +--uqtf_quant_delay=10000 +``` + +#### Export to .tflite format -*TODO* add quantization to the conversion tools ```bash # load the checkpoints in ./models, and read the collections of 'inputs' and 'outputs' -python export_pb_tflite_models.py \ ---model_dir ./models ---input_coll inputs ---output_coll outputs ---quantize True +python export_quant_tflite_models.py \ +--model_dir ./models \ +--input_coll inputs \ +--output_coll outputs \ +--enbl_post_quant True ``` +Note that we set `enbl_post_quant`to`True` to ensure all operations being quantized. On the one hand, some operations may not be successfully quantized via [tf.contrib.quantize.experimental_create_training_graph](https://www.tensorflow.org/api_docs/python/tf/contrib/quantize/experimental_create_training_graph) in `UniformQuantTFLearner`, post quantization can help remedy this, possibly at the cost of slight decrease of the quantized performance. On the other hand, users can directly export a quantized model to `.tflite` format without going through the `UniformQuantTFLearner`. This could be helpful when users want to quickly test the inference speed, or there is more tolerance for the performance of quantized model. + If successfully transformed, the `.pb` and `.tflite` files will be saved in `./models`. -### Deploy on Mobile Devices -*TODO* +#### Deploy on Mobile Devices +The Deployment of a quantized model is very similar to that of a full-precision model, as is shown in the [tutorial page](https://pocketflow.github.io/tutorial/). Specifically, users need to do the following modifications: -## Algorithms -Now we introduce the detailed algorithm in the Uniform Quantization Learner. As is shown in the following graph, given a full precision model, the Uniform Quantization Learner inserts quantization nodes into the computation graph of the model. To enable activation quantization, the quantization nodes shall also be inserted after the activation function. -In the training phase, both full-precision and quantized weights are stored. During the forward pass, quantized weights are obtained by applying the quantization functions on the full precision weights. For the backward propagation of gradients, since the gradients w.r.t. the quantized weights are 0 almost everywhere, we use the straight-through estimator (STE) ([Hinton et.al 2012](https://www.coursera.org/learn/neural-networks), [Bengio et.al 2013](https://arxiv.org/abs/1308.3432)) to pass the gradient of quantized weights directly to the full precision weights for update. +1. In [ImageClassifierQuantizedMobileNet.java](https://github.com/tensorflow/tensorflow/blob/r1.12/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifierQuantizedMobileNet.java) L24: rename the class w.r.t. your model. +2. In [ImageClassifierQuantizedMobileNet.java](https://github.com/tensorflow/tensorflow/blob/r1.12/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifierQuantizedMobileNet.java) L46: replace the model input "mobilenet_quant_v1_224.tflite" to your "*.tflite" file. +3. In [ImageClassifierQuantizedMobileNet.java](https://github.com/tensorflow/tensorflow/blob/r1.12/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifierQuantizedMobileNet.java) L51: replace the label file "labels_mobilenet_quant_v1_224.txt" to your label files. + +4. In [Camera2BasicFragment.java](https://github.com/tensorflow/tensorflow/blob/r1.10/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java) L332: change the name of the class accordingly. - ![Train_n_Inference](pics/train_n_inference.png) -### Uniform Quantization Function -Uniform quantization distributed the quantization points evenly across the distribution of weights, and the full precision numbers are then assigned to the closest quantization point. To achieve this, we first normalize the full precision weights $x$ of one layer to $[0, 1]$, i.e., -$$ -sc(x) = \frac{x-\beta}{\alpha}, -$$ -where $\alpha=\max{x}-\min{x}$ and $\beta = \min{x}$ are the scaling factors. Then we assign $sc(x)$ to the discrete value by -$$ -\hat{x}=\frac{1}{2^k-1}\mathrm{round}((2^k-1)\cdot sc(x)), -$$ -and finally we do the inverse linear transformation to recover the quantized weights to the original scale, -$$ -Q(x)=\alpha\hat{x}+\beta. -$$ ## References From 384f07dbbaa7e6e93748248ddbb38957864096e2 Mon Sep 17 00:00:00 2001 From: Jiaxiang Wu Date: Fri, 16 Nov 2018 19:17:39 +0800 Subject: [PATCH 008/173] Update the command for model conversion (#70) * add source files for documentation * add Python package dependencies * add README.md * issue #48 fixed * update learners' description; issue #50 fixed * update UQ-learner's hyper-param description; issue #42 fixed * update docs for UniformQuantLearner and NonUniformQuantLearner * typo fixed; remove trailing whitespaces * UniformQuant(TF)Learner's doc revised; issue #61 fixed --- docs/docs/reference.md | 1 + docs/docs/uq_learner.md | 122 +++++++++++++++++++--------------------- 2 files changed, 58 insertions(+), 65 deletions(-) diff --git a/docs/docs/reference.md b/docs/docs/reference.md index a36950d..8865321 100644 --- a/docs/docs/reference.md +++ b/docs/docs/reference.md @@ -1,5 +1,6 @@ # Reference +* [**Bengio et al., 2015**] Yoshua Bengio, Nicholas Leonard, and Aaron Courville. *Estimating or Propagating Gradients Through Stochastic Neurons for Conditional Computation*. CoRR, abs/1308.3432, 2013. * [**Bergstra et al., 2013**] J. Bergstra, D. Yamins, and D. D. Cox. *Making a Science of Model Search: Hyperparameter Optimization in Hundreds of Dimensions for Vision Architectures*. In International Conference on Machine Learning (ICML), pages 115-123, Jun 2013. * [**Han et al., 2016**] Song Han, Huizi Mao, and William J. Dally. *Deep Compression: Compressing Deep Neural Network with Pruning, Trained Quantization and Huffman Coding*. In International Conference on Learning Representations (ICLR), 2016. * [**He et al., 2017**] Yihui He, Xiangyu Zhang, and Jian Sun. *Channel Pruning for Accelerating Very Deep Neural Networks*. In IEEE International Conference on Computer Vision (ICCV), pages 1389-1397, 2017. diff --git a/docs/docs/uq_learner.md b/docs/docs/uq_learner.md index 3159bd5..10c093e 100644 --- a/docs/docs/uq_learner.md +++ b/docs/docs/uq_learner.md @@ -3,62 +3,66 @@ ## Introduction -Uniform quantization is widely used for model compression and acceleration. Originally the weights in the network are represented by 32-bit float numbers. With uniform quantization, low-precision (e.g., 4 bit, 8 bit) and evenly distributed float numbers are used to approximate the full precision networks. For $k$-bit quantization, the memory saving can be up to $32/k$. For example, 8 bit quantization reduce the network size by 4 folds with little drop of performance. +Uniform quantization is widely used for model compression and acceleration. Originally the weights in the network are represented by 32-bit floating-point numbers. With uniform quantization, low-precision (*e.g.* 4-bit or 8-bit) fixed-point numbers are used to approximate the full-precision network. For $k$-bit quantization, the memory saving can be up to $32 / k​$. For example, 8-bit quantization can reduce the network size by 4 folds with negligible drop of performance. - Currently PocketFlow supports two types of uniform quantization learner: +Currently, PocketFlow supports two types of uniform quantization learners: -* `UniformQuantLearner`: the self-developed learner for uniform quantization. The learner is carefully optimized with various extensions and variations supported. +* `UniformQuantLearner`: a self-developed learner for uniform quantization. The learner is carefully optimized with various extensions and variations supported. -* `UniformQuantTFLearner`: a wrapper based on the [quantization aware training](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/quantize) in TensorFlow. The wrapper currently only supports 8-bit quantization, enjoying 4x reduction of memory and approximately 3x times speed up of inference. +* `UniformQuantTFLearner`: a wrapper based on TensorFlow's [quantization-aware training](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/quantize) training APIs. For now, this wrapper only supports 8-bit quantization, which leads to approximately 4x memory reduction and 3x inference speed-up. -A comparison of the two learners are shown below: +A comparison of these two learners are shown below: -| Features| Uniform Quantization Learner | TensorFlow Quantization Wrapper | -| :--------: | :--------:| :--: | -| Compression | yes | Yes | -| Acceleration | | Yes | -| Fine-tuning | Yes | | -| Bucketing | Yes | | -| Hyper-param Searching| Yes | | +| Features | `UniformQuantLearner` | `UniformQuantTFLearner` | +|:--------:|:---------------------:|:-----------------------:| +| Compression | Yes | Yes | +| Acceleration | | Yes | +| Fine-tuning | Yes | | +| Bucketing | Yes | | +| Hyper-param Optimization | Yes | | ## Algorithm -Both uniform quantization learners generally obey the following procedures: +### Training Workflow -Given a pre-defined full precision model, the learners insert quantization nodes into the computation graph of the model. To enable activation quantization, quantization nodes will also be placed after the activation operations (e.g., ReLu). +Both two uniform quantization learners generally follow the training workflow below: -In the training phase, both full-precision and quantized weights are kept. In the forward pass, quantized weights are obtained by applying the quantization functions on the full precision weights. To update the weights in the backward pass, since the gradients w.r.t. the quantized weights are 0 almost everywhere, we use the straight-through estimator (STE) ([Hinton et.al 2012](https://www.coursera.org/learn/neural-networks), [Bengio et.al 2013](https://arxiv.org/abs/1308.3432)) to pass the gradient of quantized weights directly to the full precision weights for update. +Given a pre-defined full-precision model, the learner inserts quantization nodes and operations into the computation graph of the model. With activation quantization enabled, quantization nodes will also be placed after activation operations (*e.g.* ReLU). - ![Train_n_Inference](D:/OneDrive%20-%20The%20Chinese%20University%20of%20Hong%20Kong/Research/MyWorks/automc/doc/pocketflow-docs/docs/pics/train_n_inference.png) +In the training phase, both full-precision and quantized weights are kept. In the forward pass, quantized weights are obtained by applying the quantization function on full-precision weights. To update full-precision weights in the backward pass, since gradients w.r.t. quantized weights are zeros almost everywhere, we use the straight-through estimator (STE, Bengio et al., 2015) to pass gradients of quantized weights directly to full-precision weights for update. -### Uniform Quantization Function +![train_n_inference](pics/train_n_inference.png) + +### Quantization Function + +Uniform quantization distributes all the quantization points evenly across the range $\left[ w_{min}, w_{max} \right]$, where $w_{max}$ and $w_{min}$ are the maximum and minimum values of weights in each layer (or bucket). The original full-precision weights are then assigned to their closest quantization points. To achieve this, we first normalize the full-precision weights $x$ to $\left[ 0, 1 \right]$: -Uniform quantization distribute the quantization points evenly across the range $[w_{min}, w_{max}]$, where $w_{max}, w_{min}$ are the maximum and minimum value of weights in the layer/bucket. Then the original full precision weights are then assigned to the closest quantization point. To achieve this, we first normalize the full precision weights $x$ of one layer to $[0, 1]$, i.e., -$$ -sc(x) = \frac{x-\beta}{\alpha}, -$$ -where $\alpha=w_{max}-w_{min}$ and $\beta = w_{min}$ are the scaling factors. Then we assign $sc(x)$ to the discrete value by $$ -\hat{x}=\frac{1}{2^k-1}\mathrm{round}((2^k-1)\cdot sc(x)), +\text{sc} \left( x \right) = \frac{ x - \beta}{\alpha}, $$ -and finally we do the inverse linear transformation to recover the quantized weights to the original scale, + +where $\alpha = w_{max} - w_{min}$ and $\beta = w_{min}$. Then, we assign $\text{sc} \left( x \right)$ to its closest quantization point (assuming $k$-bit quantization is used): + $$ -Q(x)=\alpha\hat{x}+\beta. +\hat{x} = \frac{1}{2^{k} - 1} \text{round} \left( \left( 2^{k} - 1 \right) \cdot \text{sc} \left( x \right) \right), $$ +and finally we use inverse linear transformation to recover the quantized weights to the original scale: -Next we introduce usages for the two uniform quantization learners. +$$ +Q \left( x \right) = \alpha \cdot \hat{x} + \beta. +$$ -## 1. UniformQuantLearner +## UniformQuantLearner -`UniformQuantLearner` is the self-developed learner, which allows a number of customized configurations for uniform quantization. For example, the learner supports bucketing, leading to more fine-grained quantization and better performance. The learner also allows to allocate different bits across layers, in which users can turn on the hyper parameter optimizer with reinforcement learning to search for the optimal bit allocation for the quantization. +`UniformQuantLearner` is a self-developed learner, which allows a number of customized configurations for uniform quantization. For example, the learner supports bucketing, leading to more fine-grained quantization and better performance. The learner also allows to allocate different bits across layers, in which users can turn on the hyper-parameter optimizer with reinforcement learning to search for the optimal bit allocation strategy. ### Hyper-parameters -To configure `UniformQuantLearner`, users can pass the options via the TensorFlow flag interface. The available options are as follows: +To configure `UniformQuantLearner`, users can pass options via the TensorFlow flag interface. The available options are listed as follows: -| Options | Description | -| :-------- | :-- | +| Option | Description | +|:-------|:------------| | `uql_weight_bits` | the number of bits for weights. Default: `4`. | | `uql_activation_bits` | the number of bits for activation. Default: `32.` | | `uql_save_quant_model_path` | quantized model's save path. Default: `./uql_quant_models/model.ckpt` | @@ -85,8 +89,8 @@ Here, we provide detailed description (and some analysis) for above hyper-parame Once the hyper parameter optimizer is turned on, i.e., `uql_enbl_rl_agent==True` , the RL agent will automatically search for the optimal bit allocation strategy for each layer. In order to search efficiently, the agent need to be configured properly. While here we list all the configurable hyper parameters for the agent, users can just keep the default value for most parameters, while modify only a few of them if necessary. -| Options | Description | -| :--------------------------- | :----------------------------------------------------------- | +| Option | Description | +|:-------|:------------| | `uql_evquivalent_bits` | the number of re-allocated bits that is equivalent to uniform allocation of bits. Default: `4`. | | `uql_nb_rlouts` | the number of roll outs for training the RL agent. Default: `200`. | | `uql_w_bit_min` | the minimal number of bits for each layer. Default: `2`. | @@ -154,16 +158,14 @@ sh ./scripts/run_seven.sh nets/mobilnet_at_ilsvrc12_run.py \ --uql_equivalent_bits=4 \ ``` - - -## 2. UniformQuantTFLearner +## UniformQuantTFLearner PocketFlow also wraps the quantization aware training in TensorFlow. The quantized model can be directly exported to `.tflite` format via [export_quant_tflite_model.py](https://github.com/haolibai/PocketFlow/blob/master/tools/conversion/export_quant_tflite_model.py) in PocketFlow, and then be easily deployed on Andriod devices. To configure `UniformQuantTFLearner`, the hyper-parameters are as follows: -| Options | Description | -| ---------------------- | ------------------------------------------------------------ | +| Option | Description | +|:-------|:------------| | `uqtf_save_path` | UQ-TF: model\'s save path. Default: `./models_uqtf/model.ckpt`. | | `uqtf_save_path_eval` | UQ-TF: model\'s save path for evaluation. Default: `./models_uqtf_eval/model.ckpt`. | | `uqtf_weight_bits` | UQ-TF: # of bits for weight quantization. Default: `8`. | @@ -182,35 +184,33 @@ Here, the detailed description (and some analysis) for some above hyper-paramete To deploy a quantized network on Andriod devices, there are generally 3 steps: -#### Quantize the pre-trained network +### Quantize the pre-trained network -To quantize a MobileNet-v1 model for ILSVRC_12 classification task with 8 bits in the seven mode, use: +To quantize a MobileNet-v1 model for ILSVRC-12 classification task with 8 bits in the seven mode, use: ``` bash -# quantize MobileNet-v1 on ILSVRC_12 -sh ./scripts/run_seven.sh nets/mobilnet_at_ilsvrc12_run.py \ --n=8 \ ---learner=uniform-tf \ ---uqtf_weight_bits=8 \ ---uqtf_activation_bits=8 \ ---uqtf_quant_delay=10000 +# quantize MobileNet-v1 on ILSVRC-12 +$ ./scripts/run_seven.sh nets/mobilnet_at_ilsvrc12_run.py -n=8 \ + --learner uniform-tf \ + --nb_epochs_rat 0.2 ``` -#### Export to .tflite format +where `--nb_epochs_rat 0.2` specifies that only 20% training epochs to be used, which usually should be enough. + +### Export to .tflite format ```bash -# load the checkpoints in ./models, and read the collections of 'inputs' and 'outputs' -python export_quant_tflite_models.py \ ---model_dir ./models \ ---input_coll inputs \ ---output_coll outputs \ ---enbl_post_quant True +# load the checkpoints in ./models_uqtf_eval +$ python export_quant_tflite_models.py \ + --model_dir ./models_uqtf_eval \ + --enbl_post_quant ``` -Note that we set `enbl_post_quant`to`True` to ensure all operations being quantized. On the one hand, some operations may not be successfully quantized via [tf.contrib.quantize.experimental_create_training_graph](https://www.tensorflow.org/api_docs/python/tf/contrib/quantize/experimental_create_training_graph) in `UniformQuantTFLearner`, post quantization can help remedy this, possibly at the cost of slight decrease of the quantized performance. On the other hand, users can directly export a quantized model to `.tflite` format without going through the `UniformQuantTFLearner`. This could be helpful when users want to quickly test the inference speed, or there is more tolerance for the performance of quantized model. -If successfully transformed, the `.pb` and `.tflite` files will be saved in `./models`. +Note that we enable the `enbl_post_quant` option to ensure all operations being quantized. On one hand, some operations may not be successfully quantized via TensorFlow's quantization-aware training APIs, so post-training quantization can help remedy this, possibly at the cost of slightly reduced accuracy of the quantized model. On the other hand, users can directly export a full-precision model to its quantized counterpart without going through the `UniformQuantTFLearner`. This could be helpful when users want to quickly evaluate the inference speed, or there is more tolerance for the performance degradation of quantized model. -#### Deploy on Mobile Devices +If the conversion completes without error, then `.pb` and `.tflite` files will be saved in `./models_uqtf_eval`. + +### Deploy on Mobile Devices The Deployment of a quantized model is very similar to that of a full-precision model, as is shown in the [tutorial page](https://pocketflow.github.io/tutorial/). Specifically, users need to do the following modifications: @@ -219,11 +219,3 @@ The Deployment of a quantized model is very similar to that of a full-precision 3. In [ImageClassifierQuantizedMobileNet.java](https://github.com/tensorflow/tensorflow/blob/r1.12/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifierQuantizedMobileNet.java) L51: replace the label file "labels_mobilenet_quant_v1_224.txt" to your label files. 4. In [Camera2BasicFragment.java](https://github.com/tensorflow/tensorflow/blob/r1.10/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java) L332: change the name of the class accordingly. - - - - -## References -Bengio Y, Léonard N, Courville A. Estimating or propagating gradients through stochastic neurons for conditional computation. arXiv preprint [arXiv:1308.3432, 2013](https://arxiv.org/abs/1308.3432) - -Geoffrey Hinton, Nitish Srivastava, Kevin Swersky, Tijmen Tieleman and Abdelrahman Mohamed. Neural Networks for Machine Learning. [Coursera, video lectures, 2012](https://www.coursera.org/learn/neural-networks) From def4738fb9b2bceb45fb8e2b86e60f659db268e3 Mon Sep 17 00:00:00 2001 From: ylfzr Date: Mon, 19 Nov 2018 08:54:28 +0800 Subject: [PATCH 009/173] numerical stability for 1 bit quantization (#74) --- learners/nonuniform_quantization/utils.py | 5 +++-- learners/uniform_quantization/utils.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/learners/nonuniform_quantization/utils.py b/learners/nonuniform_quantization/utils.py index ea7963e..e76cb01 100644 --- a/learners/nonuniform_quantization/utils.py +++ b/learners/nonuniform_quantization/utils.py @@ -405,8 +405,9 @@ def __scale(self, w, mode): w_max = tf.stop_gradient(tf.reduce_max(w, axis=axis)) w_min = tf.stop_gradient(tf.reduce_min(w, axis=axis)) - - alpha = w_max - w_min + eps = tf.constant(value=1e-10, dtype=tf.float32) + + alpha = w_max - w_min + eps beta = w_min w = (w - beta) / alpha return w, alpha, beta diff --git a/learners/uniform_quantization/utils.py b/learners/uniform_quantization/utils.py index c51fbc2..fc41bda 100644 --- a/learners/uniform_quantization/utils.py +++ b/learners/uniform_quantization/utils.py @@ -223,8 +223,9 @@ def __scale(self, w, mode): w_max = tf.stop_gradient(tf.reduce_max(w, axis=axis)) w_min = tf.stop_gradient(tf.reduce_min(w, axis=axis)) - - alpha = w_max - w_min + eps = tf.constant(value=1e-10, dtype=tf.float32) + + alpha = w_max - w_min + eps beta = w_min w = (w - beta) / alpha return w, alpha, beta From 84f6c00c28b914245a5c0bdff4b1ef12a206f682 Mon Sep 17 00:00:00 2001 From: ylfzr Date: Mon, 19 Nov 2018 12:56:23 +0800 Subject: [PATCH 010/173] fix Shape Error #76 (#81) * fix Shape Error #76 * Update learner.py * Update learner.py * Update bit_optimizer.py * Update learner.py --- .../nonuniform_quantization/bit_optimizer.py | 2 +- learners/nonuniform_quantization/learner.py | 20 ++++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/learners/nonuniform_quantization/bit_optimizer.py b/learners/nonuniform_quantization/bit_optimizer.py index 5cdc406..dd5337b 100644 --- a/learners/nonuniform_quantization/bit_optimizer.py +++ b/learners/nonuniform_quantization/bit_optimizer.py @@ -278,7 +278,7 @@ def __layerwise_finetune(self, feed_dict_train, layer_bits): def __global_finetune(self, feed_dict_train): time_prev = timer() for t_step in range(self.tune_global_steps): - _ = self.sess_train.run(self.ops['train'], feed_dict=feed_dict_train) + self.sess_train.run(self.ops['rl_fintune'], feed_dict=feed_dict_train) if (t_step+1) % self.tune_global_disp_steps == 0: log_rslt = self.sess_train.run(self.ops['log'], feed_dict=feed_dict_train) time_prev = self.__monitor_progress(t_step, log_rslt, time_prev) diff --git a/learners/nonuniform_quantization/learner.py b/learners/nonuniform_quantization/learner.py index e5cbcf7..db0968b 100644 --- a/learners/nonuniform_quantization/learner.py +++ b/learners/nonuniform_quantization/learner.py @@ -255,10 +255,17 @@ def __build_train(self): if v not in clusters] # determine the var_list optimize - if FLAGS.nuql_opt_mode == 'both': - optimizable_vars = self.trainable_vars - elif FLAGS.nuql_opt_mode == 'clusters': - optimizable_vars = clusters + if FLAGS.nuql_opt_mode in ['cluster', 'both']: + if FLAGS.nuql_opt_mode == 'both': + optimizable_vars = self.trainable_vars + else: + optimizable_vars = clusters + if FLAGS.nuql_enbl_rl_agent: + optimizer_fintune = tf.train.GradientDescentOptimizer(lrn_rate) + if FLAGS.enbl_multi_gpu: + optimizer_fintune = mgw.DistributedOptimizer(optimizer_fintune) + grads_fintune = optimizer_fintune.compute_gradients(loss, var_list=optimizable_vars) + elif FLAGS.nuql_opt_mode == 'weights': optimizable_vars = rest_trainable_vars else: @@ -272,7 +279,10 @@ def __build_train(self): # define the ops with tf.control_dependencies(self.update_ops): self.ops['train'] = optimizer.apply_gradients(grads, global_step=self.ft_step) - + if FLAGS.nuql_opt_mode in ['both', 'cluster'] and FLAGS.nuql_enbl_rl_agent: + self.ops['rl_fintune'] = optimizer_fintune.apply_gradients(grads_fintune, global_step=self.ft_step) + else: + self.ops['rl_fintune'] = self.ops['train'] self.ops['summary'] = tf.summary.merge_all() if FLAGS.enbl_dst: self.ops['log'] = [lrn_rate, dst_loss, model_loss, loss, acc_top1, acc_top5] From cdfec38f33abbb58e235257bb9c6c8e5fefdff16 Mon Sep 17 00:00:00 2001 From: KranthiGV Date: Tue, 20 Nov 2018 06:20:48 +0530 Subject: [PATCH 011/173] Fix for using single GPU in local mode; Change of definition of idle GPU (#75) * Fix for allowing training on single local gpu. Change of idle gpu definition * Fix to allow single local GPU training * Fix naming and code conventions --- utils/get_idle_gpus.py | 54 ++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/utils/get_idle_gpus.py b/utils/get_idle_gpus.py index c9e5dcf..458818c 100644 --- a/utils/get_idle_gpus.py +++ b/utils/get_idle_gpus.py @@ -24,37 +24,29 @@ assert len(sys.argv) == 2 nb_idle_gpus = int(sys.argv[1]) -# dump the output of "nvidia-smi" command to file -dump_file = './nvidia-smi-dump' -with open(dump_file, 'w') as o_file: - subprocess.call(['nvidia-smi'], stdout=o_file) +# assume: idle gpu has no more than 50% of total card memory used +max_gpu_usage_threshold = .5 -# parse the output of "nvidia-smi" command -with open(dump_file, 'r') as i_file: - # obtain list of all & busy GPUs - parse_procs = False - all_gpus, busy_gpus = [], [] - for i_line in i_file: - if 'Processes' in i_line: - parse_procs = True - sub_strs = i_line.split() - if len(sub_strs) < 2: - continue - if not parse_procs: - if sub_strs[1].isdigit(): - all_gpus.append(sub_strs[1]) - else: - if sub_strs[1].isdigit(): - busy_gpus.append(sub_strs[1]) +# command to execute to get gpu id and corresponding memory used +# and total memory. It gives output in the format +# gpu id, memory used, total memory +cmd = 'nvidia-smi --query-gpu=index,memory.used,memory.total ' \ + '--format=csv,noheader,nounits' +gpu_smi_output = subprocess.check_output(cmd, shell=True) +gpu_smi_output = gpu_smi_output.decode('utf-8') - # obtain list of idle GPUs - idle_gpus = list(set(all_gpus) - set(busy_gpus)) - idle_gpus.sort() - if len(idle_gpus) < nb_idle_gpus: - raise ValueError('not enough idle GPUs; idle GPUs are: {}'.format(idle_gpus)) - idle_gpus = idle_gpus[:nb_idle_gpus] - idle_gpus_str = ','.join([str(idle_gpu) for idle_gpu in idle_gpus]) - print(idle_gpus_str) +idle_gpus = [] -# remove the dump file -os.remove(dump_file) +for gpu in gpu_smi_output.split(sep='\n')[:-1]: + (gpu_id, used, total) = [int(value) for value in gpu.split(sep=',')] + memory_usage_percentage = (used/total) + if memory_usage_percentage < max_gpu_usage_threshold: + # GPU usage is less than the threshold + idle_gpus.append(gpu_id) + +if len(idle_gpus) < nb_idle_gpus: + raise ValueError('not enough idle GPUs;' + ' idle GPUs are: {}'.format(idle_gpus)) +idle_gpus = idle_gpus[:nb_idle_gpus] +idle_gpus_str = ','.join([str(idle_gpu) for idle_gpu in idle_gpus]) +print(idle_gpus_str) From 4775d4c378f9cee0de830825cb4fe4745b74a25a Mon Sep 17 00:00:00 2001 From: Jiaxiang Wu Date: Wed, 21 Nov 2018 13:54:16 +0800 Subject: [PATCH 012/173] GPU-based Channel Selection (#84) * add source files for documentation * add Python package dependencies * add README.md * initial implementation for GPU-optimized channel pruning * use adpative learning rate and hard thresholding * allow specify each layer's pruning ratio with a list * use to skip first & last layers for pruning * adaptively adjust the L2,1-norm's regularization strength * use truncated pruning percentile for channel selection * use Adam for layer-wise regression after channel selection * remove hyper-parameters for group-lasso loss term's coefficient * remove debug-only code * adjust the default value of learning rates for PGD & Adam --- learners/channel_pruning_gpu/__init__.py | 0 learners/channel_pruning_gpu/learner.py | 561 +++++++++++++++++++++++ learners/learner_utils.py | 3 + 3 files changed, 564 insertions(+) create mode 100644 learners/channel_pruning_gpu/__init__.py create mode 100644 learners/channel_pruning_gpu/learner.py diff --git a/learners/channel_pruning_gpu/__init__.py b/learners/channel_pruning_gpu/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/learners/channel_pruning_gpu/learner.py b/learners/channel_pruning_gpu/learner.py new file mode 100644 index 0000000..8579f9c --- /dev/null +++ b/learners/channel_pruning_gpu/learner.py @@ -0,0 +1,561 @@ +# Tencent is pleased to support the open source community by making PocketFlow available. +# +# Copyright (C) 2018 THL A29 Limited, a Tencent company. All rights reserved. +# +# Licensed under the BSD 3-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Channel pruning learner with GPU-based optimization.""" + +import os +import re +import math +from timeit import default_timer as timer +import numpy as np +import tensorflow as tf + +from learners.abstract_learner import AbstractLearner +from learners.distillation_helper import DistillationHelper +from utils.lrn_rate_utils import setup_lrn_rate +from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw + +FLAGS = tf.app.flags.FLAGS + +tf.app.flags.DEFINE_string('cpg_save_path', './models_cpg/model.ckpt', 'CPG: model\'s save path') +tf.app.flags.DEFINE_string('cpg_save_path_eval', './models_cpg_eval/model.ckpt', + 'CPG: model\'s save path for evaluation') +tf.app.flags.DEFINE_string('cpg_prune_ratio_type', 'uniform', + 'CPG: pruning ratio type (\'uniform\' OR \'list\')') +tf.app.flags.DEFINE_float('cpg_prune_ratio', 0.5, 'CPG: uniform pruning ratio') +tf.app.flags.DEFINE_boolean('cpg_skip_ht_layers', True, 'CPG: skip head & tail layers for pruning') +tf.app.flags.DEFINE_string('cpg_prune_ratio_file', None, + 'CPG: file path to the list of pruning ratios') +tf.app.flags.DEFINE_float('cpg_lrn_rate_pgd_init', 1e-10, + 'CPG: proximal gradient descent\'s initial learning rate') +tf.app.flags.DEFINE_float('cpg_lrn_rate_pgd_incr', 1.4, + 'CPG: proximal gradient descent\'s learning rate\'s increase ratio') +tf.app.flags.DEFINE_float('cpg_lrn_rate_pgd_decr', 0.7, + 'CPG: proximal gradient descent\'s learning rate\'s decrease ratio') +tf.app.flags.DEFINE_float('cpg_lrn_rate_adam', 1e-2, 'CPG: Adam\'s initial learning rate') +tf.app.flags.DEFINE_integer('cpg_nb_iters_layer', 1000, 'CPG: # of iterations for layer-wise FT') + +def get_vars_by_scope(scope): + """Get list of variables within certain name scope. + + Args: + * scope: name scope + + Returns: + * vars_dict: dictionary of list of all, trainable, and maskable variables + """ + + vars_dict = {} + vars_dict['all'] = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=scope) + vars_dict['trainable'] = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope) + vars_dict['maskable'] = [] + conv2d_pattern = re.compile(r'/Conv2D$') + conv2d_ops = get_ops_by_scope_n_pattern(scope, conv2d_pattern) + for var in vars_dict['trainable']: + for op in conv2d_ops: + for op_input in op.inputs: + if op_input.name == var.name.replace(':0', '/read:0'): + vars_dict['maskable'] += [var] + + return vars_dict + +def get_ops_by_scope_n_pattern(scope, pattern): + """Get list of operations within certain name scope and also matches the pattern. + + Args: + * scope: name scope + * pattern: name pattern to be matched + + Returns: + * ops: list of operations + """ + + ops = [] + for op in tf.get_default_graph().get_operations(): + if op.name.startswith(scope) and re.search(pattern, op.name) is not None: + ops += [op] + + return ops + +def calc_prune_ratio(vars_list): + """Calculate the overall pruning ratio for the given list of variables. + + Args: + * vars_list: list of variables + + Returns: + * prune_ratio: overall pruning ratio of the given list of variables + """ + + nb_params_nnz = tf.add_n([tf.count_nonzero(var) for var in vars_list]) + nb_params_all = tf.add_n([tf.size(var) for var in vars_list]) + prune_ratio = 1.0 - tf.cast(nb_params_nnz, tf.float32) / tf.cast(nb_params_all, tf.float32) + + return prune_ratio + +class ChannelPrunedGpuLearner(AbstractLearner): # pylint: disable=too-many-instance-attributes + """Channel pruning learner with GPU-based optimization.""" + + def __init__(self, sm_writer, model_helper): + """Constructor function. + + Args: + * sm_writer: TensorFlow's summary writer + * model_helper: model helper with definitions of model & dataset + """ + + # class-independent initialization + super(ChannelPrunedGpuLearner, self).__init__(sm_writer, model_helper) + + # define scopes for full & channel-pruned models + self.model_scope_full = 'model' + self.model_scope_prnd = 'pruned_model' + + # download the pre-trained model + if self.is_primary_worker('local'): + self.download_model() # pre-trained model is required + self.auto_barrier() + tf.logging.info('model files: ' + ', '.join(os.listdir('./models'))) + + # class-dependent initialization + if FLAGS.enbl_dst: + self.helper_dst = DistillationHelper(sm_writer, model_helper, self.mpi_comm) + self.__build_train() + self.__build_eval() + + def train(self): + """Train a model and periodically produce checkpoint files.""" + + # restore the full model from pre-trained checkpoints + save_path = tf.train.latest_checkpoint(os.path.dirname(self.save_path_full)) + self.saver_full.restore(self.sess_train, save_path) + + # initialization + self.sess_train.run([self.init_op, self.init_opt_op]) + self.sess_train.run([layer_op['init_opt'] for layer_op in self.layer_ops]) + if FLAGS.enbl_multi_gpu: + self.sess_train.run(self.bcast_op) + + # choose channels and evaluate the model before re-training + self.__choose_channels() + if self.is_primary_worker('global'): + self.__save_model(is_train=True) + self.evaluate() + self.auto_barrier() + + # fine-tune the model with chosen channels only + time_prev = timer() + for idx_iter in range(self.nb_iters_train): + # train the model + if (idx_iter + 1) % FLAGS.summ_step != 0: + self.sess_train.run(self.train_op) + else: + __, summary, log_rslt = self.sess_train.run([self.train_op, self.summary_op, self.log_op]) + if self.is_primary_worker('global'): + time_step = timer() - time_prev + self.__monitor_progress(summary, log_rslt, idx_iter, time_step) + time_prev = timer() + + # save the model at certain steps + if self.is_primary_worker('global') and (idx_iter + 1) % FLAGS.save_step == 0: + self.__save_model(is_train=True) + self.evaluate() + self.auto_barrier() + + # save the final model + if self.is_primary_worker('global'): + self.__save_model(is_train=True) + self.__restore_model(is_train=False) + self.__save_model(is_train=False) + self.evaluate() + + def evaluate(self): + """Restore a model from the latest checkpoint files and then evaluate it.""" + + self.__restore_model(is_train=False) + nb_iters = int(np.ceil(float(FLAGS.nb_smpls_eval) / FLAGS.batch_size_eval)) + eval_rslts = np.zeros((nb_iters, len(self.eval_op))) + for idx_iter in range(nb_iters): + eval_rslts[idx_iter] = self.sess_eval.run(self.eval_op) + for idx, name in enumerate(self.eval_op_names): + tf.logging.info('%s = %.4e' % (name, np.mean(eval_rslts[:, idx]))) + + def __build_train(self): # pylint: disable=too-many-locals,too-many-statements + """Build the training graph.""" + + with tf.Graph().as_default(): + # create a TF session for the current graph + config = tf.ConfigProto() + config.gpu_options.visible_device_list = str(mgw.local_rank() if FLAGS.enbl_multi_gpu else 0) # pylint: disable=no-member + sess = tf.Session(config=config) + + # data input pipeline + with tf.variable_scope(self.data_scope): + iterator = self.build_dataset_train() + images, labels = iterator.get_next() + + # model definition - distilled model + if FLAGS.enbl_dst: + logits_dst = self.helper_dst.calc_logits(sess, images) + + # model definition - full model + with tf.variable_scope(self.model_scope_full): + __ = self.forward_train(images) + self.vars_full = get_vars_by_scope(self.model_scope_full) + self.saver_full = tf.train.Saver(self.vars_full['all']) + self.save_path_full = FLAGS.save_path + + # model definition - channel-pruned model + with tf.variable_scope(self.model_scope_prnd): + logits_prnd = self.forward_train(images) + self.vars_prnd = get_vars_by_scope(self.model_scope_prnd) + self.maskable_var_names = [var.name for var in self.vars_prnd['maskable']] + self.saver_prnd_train = tf.train.Saver(self.vars_prnd['all']) + + # loss & extra evaluation metrics + loss, metrics = self.calc_loss(labels, logits_prnd, self.vars_prnd['trainable']) + if FLAGS.enbl_dst: + loss += self.helper_dst.calc_loss(logits_prnd, logits_dst) + tf.summary.scalar('loss', loss) + for key, value in metrics.items(): + tf.summary.scalar(key, value) + + # learning rate schedule + self.global_step = tf.train.get_or_create_global_step() + lrn_rate, self.nb_iters_train = setup_lrn_rate( + self.global_step, self.model_name, self.dataset_name) + + # overall pruning ratios of trainable & maskable variables + pr_trainable = calc_prune_ratio(self.vars_prnd['trainable']) + pr_maskable = calc_prune_ratio(self.vars_prnd['maskable']) + tf.summary.scalar('pr_trainable', pr_trainable) + tf.summary.scalar('pr_maskable', pr_maskable) + + # create masks and corresponding operations for channel pruning + self.masks = [] + self.mask_updt_ops = [] + for idx, var in enumerate(self.vars_prnd['maskable']): + tf.logging.info('creating a pruning mask for {} of size {}'.format(var.name, var.shape)) + name = '/'.join(var.name.split('/')[1:]).replace(':0', '_mask') + self.masks += [tf.get_variable(name, initializer=tf.ones(var.shape), trainable=False)] + var_norm = tf.sqrt(tf.reduce_sum(tf.square(var), axis=[0, 1, 3], keepdims=True)) + mask_vec = tf.cast(var_norm > 0.0, tf.float32) + mask_new = tf.tile(mask_vec, [var.shape[0], var.shape[1], 1, var.shape[3]]) + self.mask_updt_ops += [self.masks[-1].assign(mask_new)] + + # build extra losses for regression & discrimination + self.reg_losses = self.__build_extra_losses() + self.nb_layers = len(self.reg_losses) + for idx, reg_loss in enumerate(self.reg_losses): + tf.summary.scalar('reg_loss_%d' % idx, reg_loss) + + # obtain the full list of trainable variables & update operations + self.vars_all = tf.get_collection( + tf.GraphKeys.GLOBAL_VARIABLES, scope=self.model_scope_prnd) + self.trainable_vars_all = tf.get_collection( + tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.model_scope_prnd) + self.update_ops_all = tf.get_collection( + tf.GraphKeys.UPDATE_OPS, scope=self.model_scope_prnd) + + # TF operations for initializing the channel-pruned model + init_ops = [] + with tf.control_dependencies([tf.variables_initializer(self.vars_all)]): + for var_full, var_prnd in zip(self.vars_full['all'], self.vars_prnd['all']): + init_ops += [var_prnd.assign(var_full)] + self.init_op = tf.group(init_ops) + + # TF operations for layer-wise, block-wise, and whole-network fine-tuning + self.layer_ops, self.lrn_rates_pgd, self.prune_perctls = self.__build_layer_ops() + self.train_op, self.init_opt_op = self.__build_network_ops(loss, lrn_rate) + + # TF operations for logging & summarizing + self.sess_train = sess + self.summary_op = tf.summary.merge_all() + self.log_op = [lrn_rate, loss, pr_trainable, pr_maskable] + list(metrics.values()) + self.log_op_names = ['lr', 'loss', 'pr_trn', 'pr_msk'] + list(metrics.keys()) + if FLAGS.enbl_multi_gpu: + self.bcast_op = mgw.broadcast_global_variables(0) + + def __build_eval(self): + """Build the evaluation graph.""" + + with tf.Graph().as_default(): + # create a TF session for the current graph + config = tf.ConfigProto() + config.gpu_options.visible_device_list = str(mgw.local_rank() if FLAGS.enbl_multi_gpu else 0) # pylint: disable=no-member + self.sess_eval = tf.Session(config=config) + + # data input pipeline + with tf.variable_scope(self.data_scope): + iterator = self.build_dataset_eval() + images, labels = iterator.get_next() + + # model definition - distilled model + if FLAGS.enbl_dst: + logits_dst = self.helper_dst.calc_logits(self.sess_eval, images) + + # model definition - channel-pruned model + with tf.variable_scope(self.model_scope_prnd): + # loss & extra evaluation metrics + logits = self.forward_eval(images) + vars_prnd = get_vars_by_scope(self.model_scope_prnd) + loss, metrics = self.calc_loss(labels, logits, vars_prnd['trainable']) + if FLAGS.enbl_dst: + loss += self.helper_dst.calc_loss(logits, logits_dst) + + # overall pruning ratios of trainable & maskable variables + pr_trainable = calc_prune_ratio(vars_prnd['trainable']) + pr_maskable = calc_prune_ratio(vars_prnd['maskable']) + + # TF operations for evaluation + self.eval_op = [loss, pr_trainable, pr_maskable] + list(metrics.values()) + self.eval_op_names = ['loss', 'pr_trn', 'pr_msk'] + list(metrics.keys()) + self.saver_prnd_eval = tf.train.Saver(vars_prnd['all']) + + # add input & output tensors to certain collections + tf.add_to_collection('images_final', images) + tf.add_to_collection('logits_final', logits) + + def __build_extra_losses(self): + """Build extra losses for regression. + + Returns: + * reg_losses: list of regression losses (one per layer) + """ + + # insert additional losses to intermediate layers + pattern = re.compile('Conv2D$') + core_ops_full = get_ops_by_scope_n_pattern(self.model_scope_full, pattern) + core_ops_prnd = get_ops_by_scope_n_pattern(self.model_scope_prnd, pattern) + reg_losses = [] + for core_op_full, core_op_prnd in zip(core_ops_full, core_ops_prnd): + reg_losses += [tf.nn.l2_loss(core_op_full.outputs[0] - core_op_prnd.outputs[0])] + + return reg_losses + + def __build_layer_ops(self): + """Build layer-wise fine-tuning operations. + + Returns: + * layer_ops: list of training and initialization operations for each layer + * lrn_rates_pgd: list of layer-wise learning rate + * prune_perctls: list of layer-wise pruning percentiles + """ + + layer_ops = [] + lrn_rates_pgd = [] # list of layer-wise learning rate + prune_perctls = [] # list of layer-wise pruning percentiles + for idx, var_prnd in enumerate(self.vars_prnd['maskable']): + # create placeholders + lrn_rate_pgd = tf.placeholder(tf.float32, shape=[], name='lrn_rate_pgd_%d' % idx) + prune_perctl = tf.placeholder(tf.float32, shape=[], name='prune_perctl_%d' % idx) + + # select channels for the current convolutional layer + optimizer = tf.train.GradientDescentOptimizer(lrn_rate_pgd) + if FLAGS.enbl_multi_gpu: + optimizer = mgw.DistributedOptimizer(optimizer) + grads = optimizer.compute_gradients(self.reg_losses[idx], [var_prnd]) + with tf.control_dependencies(self.update_ops_all): + var_prnd_new = var_prnd - lrn_rate_pgd * grads[0][0] + var_norm = tf.sqrt(tf.reduce_sum(tf.square(var_prnd_new), axis=[0, 1, 3], keepdims=True)) + threshold = tf.contrib.distributions.percentile(var_norm, prune_perctl) + shrk_vec = tf.maximum(1.0 - threshold / var_norm, 0.0) + prune_op = var_prnd.assign(var_prnd_new * shrk_vec) + + # fine-tune with selected channels only + optimizer_base = tf.train.AdamOptimizer(FLAGS.cpg_lrn_rate_adam) + if not FLAGS.enbl_multi_gpu: + optimizer = optimizer_base + else: + optimizer = mgw.DistributedOptimizer(optimizer_base) + grads_origin = optimizer.compute_gradients(self.reg_losses[idx], [var_prnd]) + grads_pruned = self.__calc_grads_pruned(grads_origin) + with tf.control_dependencies(self.update_ops_all): + finetune_op = optimizer.apply_gradients(grads_pruned) + init_opt_op = tf.variables_initializer(optimizer_base.variables()) + + # append layer-wise operations & variables + layer_ops += [{'prune': prune_op, 'finetune': finetune_op, 'init_opt': init_opt_op}] + lrn_rates_pgd += [lrn_rate_pgd] + prune_perctls += [prune_perctl] + + return layer_ops, lrn_rates_pgd, prune_perctls + + def __build_network_ops(self, loss, lrn_rate): + """Build network training operations. + + Returns: + * train_op: training operation of the whole network + * init_opt_op: initialization operation of the whole network's optimizer + """ + + optimizer_base = tf.train.MomentumOptimizer(lrn_rate, FLAGS.momentum) + if not FLAGS.enbl_multi_gpu: + optimizer = optimizer_base + else: + optimizer = mgw.DistributedOptimizer(optimizer_base) + grads_origin = optimizer.compute_gradients(loss, self.trainable_vars_all) + grads_pruned = self.__calc_grads_pruned(grads_origin) + with tf.control_dependencies(self.update_ops_all): + train_op = optimizer.apply_gradients(grads_pruned, global_step=self.global_step) + init_opt_op = tf.variables_initializer(optimizer_base.variables()) + + return train_op, init_opt_op + + def __calc_grads_pruned(self, grads_origin): + """Calculate the mask-pruned gradients. + + Args: + * grads_origin: list of original gradients + + Returns: + * grads_pruned: list of mask-pruned gradients + """ + + grads_pruned = [] + for grad in grads_origin: + if grad[1].name not in self.maskable_var_names: + grads_pruned += [grad] + else: + idx_mask = self.maskable_var_names.index(grad[1].name) + grads_pruned += [(grad[0] * self.masks[idx_mask], grad[1])] + + return grads_pruned + + def __choose_channels(self): # pylint: disable=too-many-locals + """Choose channels for all convolutional layers.""" + + # obtain each layer's pruning ratio + if FLAGS.cpg_prune_ratio_type == 'uniform': + ratio_list = [FLAGS.cpg_prune_ratio] * self.nb_layers + if FLAGS.cpg_skip_ht_layers: + ratio_list[0] = 0.0 + ratio_list[-1] = 0.0 + elif FLAGS.cpg_prune_ratio_type == 'list': + with open(FLAGS.cpg_prune_ratio_file, 'r') as i_file: + i_line = i_file.readline().strip() + ratio_list = [float(sub_str) for sub_str in i_line.split(',')] + else: + raise ValueError('unrecognized pruning ratio type: ' + FLAGS.cpg_prune_ratio_type) + + # select channels for all convolutional layers + nb_workers = mgw.size() if FLAGS.enbl_multi_gpu else 1 + nb_iters_layer = int(FLAGS.cpg_nb_iters_layer / nb_workers) + for idx_layer in range(self.nb_layers): + # skip if no pruning is required + if ratio_list[idx_layer] == 0.0: + continue + if self.is_primary_worker('global'): + tf.logging.info('layer #%d: pr = %.2f (target)' % (idx_layer, ratio_list[idx_layer])) + tf.logging.info('mask.shape = {}'.format(self.masks[idx_layer].shape)) + + # select channels for the current convolutional layer + time_prev = timer() + reg_loss_prev = 0.0 + lrn_rate_pgd = FLAGS.cpg_lrn_rate_pgd_init + for idx_iter in range(nb_iters_layer): + # take a stochastic proximal gradient descent step + prune_perctl = ratio_list[idx_layer] * 100.0 * (idx_iter + 1) / nb_iters_layer + __, reg_loss = self.sess_train.run( + [self.layer_ops[idx_layer]['prune'], self.reg_losses[idx_layer]], + feed_dict={self.lrn_rates_pgd[idx_layer]: lrn_rate_pgd, + self.prune_perctls[idx_layer]: prune_perctl}) + mask = self.sess_train.run(self.masks[idx_layer]) + if self.is_primary_worker('global'): + nb_chns_nnz = np.count_nonzero(np.sum(mask, axis=(0, 1, 3))) + tf.logging.info('iter %d: nnz-chns = %d | loss = %.2e | lr = %.2e | percentile = %.2f' + % (idx_iter + 1, nb_chns_nnz, reg_loss, lrn_rate_pgd, prune_perctl)) + + # adjust the learning rate + if reg_loss < reg_loss_prev: + lrn_rate_pgd *= FLAGS.cpg_lrn_rate_pgd_incr + else: + lrn_rate_pgd *= FLAGS.cpg_lrn_rate_pgd_decr + reg_loss_prev = reg_loss + + # fine-tune with selected channels only + self.sess_train.run(self.mask_updt_ops[idx_layer]) + for idx_iter in range(nb_iters_layer): + __, reg_loss = self.sess_train.run( + [self.layer_ops[idx_layer]['finetune'], self.reg_losses[idx_layer]]) + mask = self.sess_train.run(self.masks[idx_layer]) + if self.is_primary_worker('global'): + nb_chns_nnz = np.count_nonzero(np.sum(mask, axis=(0, 1, 3))) + tf.logging.info('iter %d: nnz-chns = %d | loss = %.2e' + % (idx_iter + 1, nb_chns_nnz, reg_loss)) + + # re-compute the pruning ratio + mask_vec = np.mean(np.square(self.sess_train.run(self.masks[idx_layer])), axis=(0, 1, 3)) + prune_ratio = 1.0 - float(np.count_nonzero(mask_vec)) / mask_vec.size + if self.is_primary_worker('global'): + tf.logging.info('layer #%d: pr = %.2f (actual) | time = %.2f' + % (idx_layer, prune_ratio, timer() - time_prev)) + + # compute overall pruning ratios + if self.is_primary_worker('global'): + log_rslt = self.sess_train.run(self.log_op) + log_str = ' | '.join(['%s = %.4e' % (name, value) + for name, value in zip(self.log_op_names, log_rslt)]) + + def __save_model(self, is_train): + """Save the current model for training or evaluation. + + Args: + * is_train: whether to save a model for training + """ + + if is_train: + save_path = self.saver_prnd_train.save(self.sess_train, FLAGS.cpg_save_path, self.global_step) + else: + save_path = self.saver_prnd_eval.save(self.sess_eval, FLAGS.cpg_save_path_eval) + tf.logging.info('model saved to ' + save_path) + + def __restore_model(self, is_train): + """Restore a model from the latest checkpoint files. + + Args: + * is_train: whether to restore a model for training + """ + + save_path = tf.train.latest_checkpoint(os.path.dirname(FLAGS.cpg_save_path)) + if is_train: + self.saver_prnd_train.restore(self.sess_train, save_path) + else: + self.saver_prnd_eval.restore(self.sess_eval, save_path) + tf.logging.info('model restored from ' + save_path) + + def __monitor_progress(self, summary, log_rslt, idx_iter, time_step): + """Monitor the training progress. + + Args: + * summary: summary protocol buffer + * log_rslt: logging operations' results + * idx_iter: index of the training iteration + * time_step: time step between two summary operations + """ + + # write summaries for TensorBoard visualization + self.sm_writer.add_summary(summary, idx_iter) + + # compute the training speed + speed = FLAGS.batch_size * FLAGS.summ_step / time_step + if FLAGS.enbl_multi_gpu: + speed *= mgw.size() + + # display monitored statistics + log_str = ' | '.join(['%s = %.4e' % (name, value) + for name, value in zip(self.log_op_names, log_rslt)]) + tf.logging.info('iter #%d: %s | speed = %.2f pics / sec' % (idx_iter + 1, log_str, speed)) diff --git a/learners/learner_utils.py b/learners/learner_utils.py index 6d57627..7fe6351 100644 --- a/learners/learner_utils.py +++ b/learners/learner_utils.py @@ -21,6 +21,7 @@ from learners.full_precision.learner import FullPrecLearner from learners.weight_sparsification.learner import WeightSparseLearner from learners.channel_pruning.learner import ChannelPrunedLearner +from learners.channel_pruning_gpu.learner import ChannelPrunedGpuLearner from learners.discr_channel_pruning.learner import DisChnPrunedLearner from learners.uniform_quantization.learner import UniformQuantLearner from learners.uniform_quantization_tf.learner import UniformQuantTFLearner @@ -46,6 +47,8 @@ def create_learner(sm_writer, model_helper): learner = WeightSparseLearner(sm_writer, model_helper) elif FLAGS.learner == 'channel': learner = ChannelPrunedLearner(sm_writer, model_helper) + elif FLAGS.learner == 'chn-pruned-gpu': + learner = ChannelPrunedGpuLearner(sm_writer, model_helper) elif FLAGS.learner == 'dis-chn-pruned': learner = DisChnPrunedLearner(sm_writer, model_helper) elif FLAGS.learner == 'uniform': From 8033dd4e93443faa0ab1b60da7f9682648cb9a02 Mon Sep 17 00:00:00 2001 From: Jiaxiang Wu Date: Thu, 22 Nov 2018 11:05:58 +0800 Subject: [PATCH 013/173] Setup learning rate in each ModelHelper (#90) * add abstract method: setup_lrn_rate * add setup_lrn_rate()'s implementation in all ModelHelpers * define FLAGS.nb_epochs_rat in each ModelHelper * use ModelHelper's setup_lrn_rate() instead * remove unused code for learning rate setup --- learners/abstract_learner.py | 1 + learners/channel_pruning/learner.py | 4 +- learners/channel_pruning_gpu/learner.py | 4 +- learners/discr_channel_pruning/learner.py | 4 +- learners/full_precision/learner.py | 4 +- learners/uniform_quantization_tf/learner.py | 4 +- learners/weight_sparsification/learner.py | 4 +- nets/abstract_model_helper.py | 13 ++ nets/lenet_at_cifar10.py | 15 ++ nets/mobilenet_at_ilsvrc12.py | 25 +++ nets/resnet_at_cifar10.py | 15 ++ nets/resnet_at_ilsvrc12.py | 15 ++ utils/lrn_rate_utils.py | 172 +------------------- 13 files changed, 97 insertions(+), 183 deletions(-) diff --git a/learners/abstract_learner.py b/learners/abstract_learner.py index cabbba7..a9dcda1 100644 --- a/learners/abstract_learner.py +++ b/learners/abstract_learner.py @@ -77,6 +77,7 @@ def __init__(self, sm_writer, model_helper): self.forward_train = model_helper.forward_train self.forward_eval = model_helper.forward_eval self.calc_loss = model_helper.calc_loss + self.setup_lrn_rate = model_helper.setup_lrn_rate self.model_name = model_helper.model_name self.dataset_name = model_helper.dataset_name diff --git a/learners/channel_pruning/learner.py b/learners/channel_pruning/learner.py index 2b2b98d..689e9ac 100644 --- a/learners/channel_pruning/learner.py +++ b/learners/channel_pruning/learner.py @@ -27,7 +27,6 @@ from tensorflow.contrib import graph_editor from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw -from utils.lrn_rate_utils import setup_lrn_rate from learners.distillation_helper import DistillationHelper from learners.abstract_learner import AbstractLearner from learners.channel_pruning.model_wrapper import Model @@ -352,8 +351,7 @@ def __build_pruned_train_model(self, path=None, finetune=False): # pylint: disab global_step = tf.get_variable('global_step', shape=[], dtype=tf.int32, trainable=False) self.global_step = global_step - lrn_rate, self.nb_iters_train = setup_lrn_rate( - self.global_step, self.model_name, self.dataset_name) + lrn_rate, self.nb_iters_train = self.setup_lrn_rate(self.global_step) if finetune and not FLAGS.cp_retrain: mom_optimizer = tf.train.AdamOptimizer(FLAGS.cp_lrn_rate_ft) diff --git a/learners/channel_pruning_gpu/learner.py b/learners/channel_pruning_gpu/learner.py index 8579f9c..e9b802d 100644 --- a/learners/channel_pruning_gpu/learner.py +++ b/learners/channel_pruning_gpu/learner.py @@ -25,7 +25,6 @@ from learners.abstract_learner import AbstractLearner from learners.distillation_helper import DistillationHelper -from utils.lrn_rate_utils import setup_lrn_rate from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw FLAGS = tf.app.flags.FLAGS @@ -235,8 +234,7 @@ def __build_train(self): # pylint: disable=too-many-locals,too-many-statements # learning rate schedule self.global_step = tf.train.get_or_create_global_step() - lrn_rate, self.nb_iters_train = setup_lrn_rate( - self.global_step, self.model_name, self.dataset_name) + lrn_rate, self.nb_iters_train = self.setup_lrn_rate(self.global_step) # overall pruning ratios of trainable & maskable variables pr_trainable = calc_prune_ratio(self.vars_prnd['trainable']) diff --git a/learners/discr_channel_pruning/learner.py b/learners/discr_channel_pruning/learner.py index 9cde94c..6bcf631 100644 --- a/learners/discr_channel_pruning/learner.py +++ b/learners/discr_channel_pruning/learner.py @@ -25,7 +25,6 @@ from learners.abstract_learner import AbstractLearner from learners.distillation_helper import DistillationHelper -from utils.lrn_rate_utils import setup_lrn_rate from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw FLAGS = tf.app.flags.FLAGS @@ -225,8 +224,7 @@ def __build_train(self): # pylint: disable=too-many-locals,too-many-statements # learning rate schedule self.global_step = tf.train.get_or_create_global_step() - lrn_rate, self.nb_iters_train = setup_lrn_rate( - self.global_step, self.model_name, self.dataset_name) + lrn_rate, self.nb_iters_train = self.setup_lrn_rate(self.global_step) # overall pruning ratios of trainable & maskable variables pr_trainable = calc_prune_ratio(self.vars_prnd['trainable']) diff --git a/learners/full_precision/learner.py b/learners/full_precision/learner.py index 55b7762..473a2aa 100644 --- a/learners/full_precision/learner.py +++ b/learners/full_precision/learner.py @@ -23,7 +23,6 @@ from learners.abstract_learner import AbstractLearner from learners.distillation_helper import DistillationHelper -from utils.lrn_rate_utils import setup_lrn_rate from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw FLAGS = tf.app.flags.FLAGS @@ -139,8 +138,7 @@ def __build(self, is_train): # pylint: disable=too-many-locals # optimizer & gradients if is_train: self.global_step = tf.train.get_or_create_global_step() - lrn_rate, self.nb_iters_train = setup_lrn_rate( - self.global_step, self.model_name, self.dataset_name) + lrn_rate, self.nb_iters_train = self.setup_lrn_rate(self.global_step) optimizer = tf.train.MomentumOptimizer(lrn_rate, FLAGS.momentum) if FLAGS.enbl_multi_gpu: optimizer = mgw.DistributedOptimizer(optimizer) diff --git a/learners/uniform_quantization_tf/learner.py b/learners/uniform_quantization_tf/learner.py index e848ba5..2a608d3 100644 --- a/learners/uniform_quantization_tf/learner.py +++ b/learners/uniform_quantization_tf/learner.py @@ -23,7 +23,6 @@ from learners.abstract_learner import AbstractLearner from learners.distillation_helper import DistillationHelper -from utils.lrn_rate_utils import setup_lrn_rate from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw FLAGS = tf.app.flags.FLAGS @@ -189,8 +188,7 @@ def __build_train(self): # pylint: disable=too-many-locals,too-many-statements # learning rate schedule self.global_step = tf.train.get_or_create_global_step() - lrn_rate, self.nb_iters_train = setup_lrn_rate( - self.global_step, self.model_name, self.dataset_name) + lrn_rate, self.nb_iters_train = self.setup_lrn_rate(self.global_step) lrn_rate *= FLAGS.uqtf_lrn_rate_dcy # decrease the learning rate by a constant factor diff --git a/learners/weight_sparsification/learner.py b/learners/weight_sparsification/learner.py index 482e01e..d93590c 100644 --- a/learners/weight_sparsification/learner.py +++ b/learners/weight_sparsification/learner.py @@ -25,7 +25,6 @@ from learners.distillation_helper import DistillationHelper from learners.weight_sparsification.pr_optimizer import PROptimizer from learners.weight_sparsification.utils import get_maskable_vars -from utils.lrn_rate_utils import setup_lrn_rate from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw FLAGS = tf.app.flags.FLAGS @@ -185,8 +184,7 @@ def __build_train(self): # pylint: disable=too-many-locals # learning rate schedule self.global_step = tf.train.get_or_create_global_step() - lrn_rate, self.nb_iters_train = setup_lrn_rate( - self.global_step, self.model_name, self.dataset_name) + lrn_rate, self.nb_iters_train = self.setup_lrn_rate(self.global_step) # overall pruning ratios of trainable & maskable variables pr_trainable = calc_prune_ratio(self.trainable_vars) diff --git a/nets/abstract_model_helper.py b/nets/abstract_model_helper.py index 8e3a0ed..f91a162 100644 --- a/nets/abstract_model_helper.py +++ b/nets/abstract_model_helper.py @@ -102,6 +102,19 @@ def calc_loss(self, labels, outputs, trainable_vars): """ pass + @abstractmethod + def setup_lrn_rate(self, global_step): + """Setup the learning rate (and number of training iterations). + + Args: + * global_step: training iteration counter + + Returns: + * lrn_rate: learning rate + * nb_iters: number of training iterations + """ + pass + @property @abstractmethod def model_name(self): diff --git a/nets/lenet_at_cifar10.py b/nets/lenet_at_cifar10.py index 631d223..c5583c4 100644 --- a/nets/lenet_at_cifar10.py +++ b/nets/lenet_at_cifar10.py @@ -20,9 +20,12 @@ from nets.abstract_model_helper import AbstractModelHelper from datasets.cifar10_dataset import Cifar10Dataset +from utils.lrn_rate_utils import setup_lrn_rate_piecewise_constant +from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw FLAGS = tf.app.flags.FLAGS +tf.app.flags.DEFINE_float('nb_epochs_rat', 1.0, '# of training epochs\'s ratio') tf.app.flags.DEFINE_float('lrn_rate_init', 1e-2, 'initial learning rate') tf.app.flags.DEFINE_float('batch_size_norm', 128, 'normalization factor of batch size') tf.app.flags.DEFINE_float('momentum', 0.9, 'momentum coefficient') @@ -104,6 +107,18 @@ def calc_loss(self, labels, outputs, trainable_vars): return loss, metrics + def setup_lrn_rate(self, global_step): + """Setup the learning rate (and number of training iterations).""" + + nb_epochs = 250 + idxs_epoch = [100, 150, 200] + decay_rates = [1.0, 0.1, 0.01, 0.001] + batch_size = FLAGS.batch_size * (1 if not FLAGS.enbl_multi_gpu else mgw.size()) + lrn_rate = setup_lrn_rate_piecewise_constant(global_step, batch_size, idxs_epoch, decay_rates) + nb_iters = int(FLAGS.nb_smpls_train * nb_epochs * FLAGS.nb_epochs_rat / batch_size) + + return lrn_rate, nb_iters + @property def model_name(self): """Model's name.""" diff --git a/nets/mobilenet_at_ilsvrc12.py b/nets/mobilenet_at_ilsvrc12.py index 4d47015..78def3f 100644 --- a/nets/mobilenet_at_ilsvrc12.py +++ b/nets/mobilenet_at_ilsvrc12.py @@ -23,11 +23,15 @@ from datasets.ilsvrc12_dataset import Ilsvrc12Dataset from utils.external import mobilenet_v1 as MobileNetV1 from utils.external import mobilenet_v2 as MobileNetV2 +from utils.lrn_rate_utils import setup_lrn_rate_piecewise_constant +from utils.lrn_rate_utils import setup_lrn_rate_exponential_decay +from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw FLAGS = tf.app.flags.FLAGS tf.app.flags.DEFINE_integer('mobilenet_version', 1, 'MobileNet\'s version (1 or 2)') tf.app.flags.DEFINE_float('mobilenet_depth_mult', 1.0, 'MobileNet\'s depth multiplier') +tf.app.flags.DEFINE_float('nb_epochs_rat', 1.0, '# of training epochs\'s ratio') tf.app.flags.DEFINE_float('lrn_rate_init', 0.045, 'initial learning rate') tf.app.flags.DEFINE_float('batch_size_norm', 96, 'normalization factor of batch size') tf.app.flags.DEFINE_float('momentum', 0.9, 'momentum coefficient') @@ -113,6 +117,27 @@ def calc_loss(self, labels, outputs, trainable_vars): return loss, metrics + def setup_lrn_rate(self, global_step): + """Setup the learning rate (and number of training iterations).""" + + batch_size = FLAGS.batch_size * (1 if not FLAGS.enbl_multi_gpu else mgw.size()) + if FLAGS.mobilenet_version == 1: + nb_epochs = 100 + idxs_epoch = [30, 60, 80, 90] + decay_rates = [1.0, 0.1, 0.01, 0.001, 0.0001] + lrn_rate = setup_lrn_rate_piecewise_constant(global_step, batch_size, idxs_epoch, decay_rates) + nb_iters = int(FLAGS.nb_smpls_train * nb_epochs * FLAGS.nb_epochs_rat / batch_size) + elif FLAGS.mobilenet_version == 2: + nb_epochs = 412 + epoch_step = 2.5 + decay_rate = 0.98 ** epoch_step # which is better, 0.98 OR (0.98 ** epoch_step)? + lrn_rate = setup_lrn_rate_exponential_decay(global_step, batch_size, epoch_step, decay_rate) + nb_iters = int(FLAGS.nb_smpls_train * nb_epochs * FLAGS.nb_epochs_rat / batch_size) + else: + raise ValueError('invalid MobileNet version: {}'.format(FLAGS.mobilenet_version)) + + return lrn_rate, nb_iters + @property def model_name(self): """Model's name.""" diff --git a/nets/resnet_at_cifar10.py b/nets/resnet_at_cifar10.py index 1765212..f0709f5 100644 --- a/nets/resnet_at_cifar10.py +++ b/nets/resnet_at_cifar10.py @@ -21,10 +21,13 @@ from nets.abstract_model_helper import AbstractModelHelper from datasets.cifar10_dataset import Cifar10Dataset from utils.external import resnet_model as ResNet +from utils.lrn_rate_utils import setup_lrn_rate_piecewise_constant +from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw FLAGS = tf.app.flags.FLAGS tf.app.flags.DEFINE_integer('resnet_size', 20, '# of layers in the ResNet model') +tf.app.flags.DEFINE_float('nb_epochs_rat', 1.0, '# of training epochs\'s ratio') tf.app.flags.DEFINE_float('lrn_rate_init', 1e-1, 'initial learning rate') tf.app.flags.DEFINE_float('batch_size_norm', 128, 'normalization factor of batch size') tf.app.flags.DEFINE_float('momentum', 0.9, 'momentum coefficient') @@ -108,6 +111,18 @@ def calc_loss(self, labels, outputs, trainable_vars): return loss, metrics + def setup_lrn_rate(self, global_step): + """Setup the learning rate (and number of training iterations).""" + + nb_epochs = 250 + idxs_epoch = [100, 150, 200] + decay_rates = [1.0, 0.1, 0.01, 0.001] + batch_size = FLAGS.batch_size * (1 if not FLAGS.enbl_multi_gpu else mgw.size()) + lrn_rate = setup_lrn_rate_piecewise_constant(global_step, batch_size, idxs_epoch, decay_rates) + nb_iters = int(FLAGS.nb_smpls_train * nb_epochs * FLAGS.nb_epochs_rat / batch_size) + + return lrn_rate, nb_iters + @property def model_name(self): """Model's name.""" diff --git a/nets/resnet_at_ilsvrc12.py b/nets/resnet_at_ilsvrc12.py index 02e4619..8b139bc 100644 --- a/nets/resnet_at_ilsvrc12.py +++ b/nets/resnet_at_ilsvrc12.py @@ -21,10 +21,13 @@ from nets.abstract_model_helper import AbstractModelHelper from datasets.ilsvrc12_dataset import Ilsvrc12Dataset from utils.external import resnet_model as ResNet +from utils.lrn_rate_utils import setup_lrn_rate_piecewise_constant +from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw FLAGS = tf.app.flags.FLAGS tf.app.flags.DEFINE_integer('resnet_size', 18, '# of layers in the ResNet model') +tf.app.flags.DEFINE_float('nb_epochs_rat', 1.0, '# of training epochs\'s ratio') tf.app.flags.DEFINE_float('lrn_rate_init', 1e-1, 'initial learning rate') tf.app.flags.DEFINE_float('batch_size_norm', 256, 'normalization factor of batch size') tf.app.flags.DEFINE_float('momentum', 0.9, 'momentum coefficient') @@ -137,6 +140,18 @@ def calc_loss(self, labels, outputs, trainable_vars): return loss, metrics + def setup_lrn_rate(self, global_step): + """Setup the learning rate (and number of training iterations).""" + + nb_epochs = 100 + idxs_epoch = [30, 60, 80, 90] + decay_rates = [1.0, 0.1, 0.01, 0.001, 0.0001] + batch_size = FLAGS.batch_size * (1 if not FLAGS.enbl_multi_gpu else mgw.size()) + lrn_rate = setup_lrn_rate_piecewise_constant(global_step, batch_size, idxs_epoch, decay_rates) + nb_iters = int(FLAGS.nb_smpls_train * nb_epochs * FLAGS.nb_epochs_rat / batch_size) + + return lrn_rate, nb_iters + @property def model_name(self): """Model's name.""" diff --git a/utils/lrn_rate_utils.py b/utils/lrn_rate_utils.py index 94081da..6fa1e28 100644 --- a/utils/lrn_rate_utils.py +++ b/utils/lrn_rate_utils.py @@ -18,26 +18,8 @@ import tensorflow as tf -from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw - FLAGS = tf.app.flags.FLAGS -# set to values smaller than 1.0 to use fewer epochs and speed up training -tf.app.flags.DEFINE_float('nb_epochs_rat', 1.0, '# of training epochs\'s ratio') - -def calc_nb_batches(nb_epochs, batch_size): - """Calculate the number of mini-batches. - - Args: - * nb_epochs: number of epoches - * batch_size: number of samples in each mini-batch - - Returns: - * nb_batches: number of mini-batches - """ - - return int(FLAGS.nb_smpls_train * nb_epochs * FLAGS.nb_epochs_rat / batch_size) - def setup_lrn_rate_piecewise_constant(global_step, batch_size, idxs_epoch, decay_rates): """Setup the learning rate with piecewise constant strategy. @@ -51,7 +33,7 @@ def setup_lrn_rate_piecewise_constant(global_step, batch_size, idxs_epoch, decay * lrn_rate: learning rate """ - # adjust the interval endpoints + # adjust interval endpoints w.r.t. FLAGS.nb_epochs_rat idxs_epoch = [idx_epoch * FLAGS.nb_epochs_rat for idx_epoch in idxs_epoch] # setup learning rate with the piecewise constant strategy @@ -59,8 +41,9 @@ def setup_lrn_rate_piecewise_constant(global_step, batch_size, idxs_epoch, decay nb_batches_per_epoch = float(FLAGS.nb_smpls_train) / batch_size bnds = [int(nb_batches_per_epoch * idx_epoch) for idx_epoch in idxs_epoch] vals = [lrn_rate_init * decay_rate for decay_rate in decay_rates] + lrn_rate = tf.train.piecewise_constant(global_step, bnds, vals) - return tf.train.piecewise_constant(global_step, bnds, vals) + return lrn_rate def setup_lrn_rate_exponential_decay(global_step, batch_size, epoch_step, decay_rate): """Setup the learning rate with exponential decaying strategy. @@ -75,154 +58,13 @@ def setup_lrn_rate_exponential_decay(global_step, batch_size, epoch_step, decay_ * lrn_rate: learning rate """ - # adjust the step size & decaying rate + # adjust the step size & decaying rate w.r.t. FLAGS.nb_epochs_rat epoch_step *= FLAGS.nb_epochs_rat # setup learning rate with the exponential decay strategy lrn_rate_init = FLAGS.lrn_rate_init * batch_size / FLAGS.batch_size_norm batch_step = int(FLAGS.nb_smpls_train * epoch_step / batch_size) + lrn_rate = tf.train.exponential_decay( + lrn_rate_init, tf.cast(global_step, tf.int32), batch_step, decay_rate, staircase=True) - return tf.train.exponential_decay( - lrn_rate_init, global_step, batch_step, decay_rate, staircase=True) - -def setup_lrn_rate_lenet_cifar10(global_step, batch_size): - """Setup the learning rate for LeNet-like models on the CIFAR-10 dataset. - - Args: - * global_step: training iteration counter - * batch_size: number of samples in each mini-batch - - Returns: - * lrn_rate: learning rate - * nb_batches: number of mini-batches - """ - - nb_epochs = 250 - idxs_epoch = [100, 150, 200] - decay_rates = [1.0, 0.1, 0.01, 0.001] - lrn_rate = setup_lrn_rate_piecewise_constant(global_step, batch_size, idxs_epoch, decay_rates) - nb_batches = calc_nb_batches(nb_epochs, batch_size) - - return lrn_rate, nb_batches - -def setup_lrn_rate_resnet_cifar10(global_step, batch_size): - """Setup the learning rate for ResNet models on the CIFAR-10 dataset. - - Args: - * global_step: training iteration counter - * batch_size: number of samples in each mini-batch - - Returns: - * lrn_rate: learning rate - * nb_batches: number of mini-batches - """ - - nb_epochs = 250 - idxs_epoch = [100, 150, 200] - decay_rates = [1.0, 0.1, 0.01, 0.001] - lrn_rate = setup_lrn_rate_piecewise_constant(global_step, batch_size, idxs_epoch, decay_rates) - nb_batches = calc_nb_batches(nb_epochs, batch_size) - - return lrn_rate, nb_batches - -def setup_lrn_rate_resnet_ilsvrc12(global_step, batch_size): - """Setup the learning rate for ResNet models on the ILSVRC-12 dataset. - - Args: - * global_step: training iteration counter - * batch_size: number of samples in each mini-batch - - Returns: - * lrn_rate: learning rate - * nb_batches: number of mini-batches - """ - - nb_epochs = 100 - idxs_epoch = [30, 60, 80, 90] - decay_rates = [1.0, 0.1, 0.01, 0.001, 0.0001] - lrn_rate = setup_lrn_rate_piecewise_constant(global_step, batch_size, idxs_epoch, decay_rates) - nb_batches = calc_nb_batches(nb_epochs, batch_size) - - return lrn_rate, nb_batches - -def setup_lrn_rate_mobilenet_v1_ilsvrc12(global_step, batch_size): - """Setup the learning rate for MobileNet-v1 models on the ILSVRC-12 dataset. - - Args: - * global_step: training iteration counter - * batch_size: number of samples in each mini-batch - - Returns: - * lrn_rate: learning rate - * nb_batches: number of mini-batches - """ - - nb_epochs = 100 - idxs_epoch = [30, 60, 80, 90] - decay_rates = [1.0, 0.1, 0.01, 0.001, 0.0001] - lrn_rate = setup_lrn_rate_piecewise_constant(global_step, batch_size, idxs_epoch, decay_rates) - nb_batches = calc_nb_batches(nb_epochs, batch_size) - - return lrn_rate, nb_batches - -def setup_lrn_rate_mobilenet_v2_ilsvrc12(global_step, batch_size): - """Setup the learning rate for MobileNet-v2 models on the ILSVRC-12 dataset. - - Args: - * global_step: training iteration counter - * batch_size: number of samples in each mini-batch - - Returns: - * lrn_rate: learning rate - * nb_batches: number of mini-batches - """ - - nb_epochs = 412 - epoch_step = 2.5 - decay_rate = 0.98 ** epoch_step - lrn_rate = setup_lrn_rate_exponential_decay(global_step, batch_size, epoch_step, decay_rate) - nb_batches = calc_nb_batches(nb_epochs, batch_size) - - return lrn_rate, nb_batches - -def setup_lrn_rate(global_step, model_name, dataset_name): - """Setup the learning rate for the given dataset. - - Args: - * global_step: training iteration counter - * model_name: model's name; must be one of ['lenet', 'resnet_*', 'mobilenet_v1', 'mobilenet_v2'] - * dataset_name: dataset's name; must be one of ['cifar_10', 'ilsvrc_12'] - - Returns: - * lrn_rate: learning rate - * nb_batches: number of training mini-batches - """ - - # obtain the overall batch size across all GPUs - if not FLAGS.enbl_multi_gpu: - batch_size = FLAGS.batch_size - else: - batch_size = FLAGS.batch_size * mgw.size() - - # choose a learning rate protocol according to the model & dataset combination - global_step = tf.cast(global_step, tf.int32) - if dataset_name == 'cifar_10': - if model_name == 'lenet': - lrn_rate, nb_batches = setup_lrn_rate_lenet_cifar10(global_step, batch_size) - elif model_name.startswith('resnet'): - lrn_rate, nb_batches = setup_lrn_rate_resnet_cifar10(global_step, batch_size) - else: - raise NotImplementedError('model: {} / dataset: {}'.format(model_name, dataset_name)) - elif dataset_name == 'ilsvrc_12': - if model_name.startswith('resnet'): - lrn_rate, nb_batches = setup_lrn_rate_resnet_ilsvrc12(global_step, batch_size) - elif model_name.startswith('mobilenet_v1'): - lrn_rate, nb_batches = setup_lrn_rate_mobilenet_v1_ilsvrc12(global_step, batch_size) - elif model_name.startswith('mobilenet_v2'): - lrn_rate, nb_batches = setup_lrn_rate_mobilenet_v2_ilsvrc12(global_step, batch_size) - else: - raise NotImplementedError('model: {} / dataset: {}'.format(model_name, dataset_name)) - else: - raise NotImplementedError('dataset: ' + dataset_name) - - return lrn_rate, nb_batches + return lrn_rate From 6795bf4e550a1ed4f55b590ff5caa6233506b047 Mon Sep 17 00:00:00 2001 From: Jiaxiang Wu Date: Thu, 22 Nov 2018 16:08:22 +0800 Subject: [PATCH 014/173] Demo for self-defined data sets and models (#92) * add abstract method: setup_lrn_rate * add setup_lrn_rate()'s implementation in all ModelHelpers * define FLAGS.nb_epochs_rat in each ModelHelper * use ModelHelper's setup_lrn_rate() instead * remove unused code for learning rate setup * start tracking *_at_*_run.py files * data input pipeline for Fashion-MNIST data set * define a ModelHelper class on Fashion-MNIST * add execution script on Fashion-MNIST * move demonstration code to ./examples * add a new documentation page for self-defined models * add documentation for using self-defined models --- .gitignore | 1 - docs/docs/self_defined_models.md | 379 ++++++++++++++++++++++++++++++ docs/mkdocs.yml | 1 + examples/convnet_at_fmnist.py | 135 +++++++++++ examples/convnet_at_fmnist_run.py | 69 ++++++ examples/fmnist_dataset.py | 166 +++++++++++++ 6 files changed, 750 insertions(+), 1 deletion(-) create mode 100644 docs/docs/self_defined_models.md create mode 100644 examples/convnet_at_fmnist.py create mode 100644 examples/convnet_at_fmnist_run.py create mode 100644 examples/fmnist_dataset.py diff --git a/.gitignore b/.gitignore index bb3c14f..4829b8b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,4 @@ automl_output_* start_multi.sh ratio.list path.conf -*_at_*_run.py nvidia-smi-dump diff --git a/docs/docs/self_defined_models.md b/docs/docs/self_defined_models.md new file mode 100644 index 0000000..2e5300c --- /dev/null +++ b/docs/docs/self_defined_models.md @@ -0,0 +1,379 @@ +# Self-defined Models + +Self-defined models (and data sets) can be incorporated into PocketFlow by implementing a new `ModelHelper` class. The `ModelHelper` class includes the definition of data input pipeline as well as the network's forward pass and loss function. With the self-defined `ModelHelper`, the network can be either trained without any constraints using `FullPrecLearner`, or trained with certain model compression algorithms using other learners, *e.g.* `ChannelPrunedLearner` for channel pruning or `UniformQuantTFLearner` for uniform quantization. In this tutorial, we will define a 4-layer convolutional neural network (2 conv. layers + 2 dense layers) for image classification on the [Fashion-MNIST](https://github.com/zalandoresearch/fashion-mnist) data set under the PocketFlow framework. Afterwards, we shall demonstrate how to train this self-defined model with different model compression components. + +## The Essentials + +To use self-defined models and data sets in PocketFlow, we need to provide the following two items in advance to describe the overall training workflow: + +* **Data Input Pipeline**: this tells PocketFlow how to parse features and ground-truth labels from data files. +* **Network Definition**: this tells PocketFlow how to compute the network's predictions and loss function's value. + +The `ModelHelper` class, which is a sub-class of the abstract base class `AbstractModelHelper`, is designed to provide such definitions. In PocketFlow, we have offered several `ModelHelper` classes to describe different combinations of data sets and model architectures. To use self-defined models, a new `ModelHelper` class should be implemented. Besides, we need an execution script to call this newly defined `ModelHelper` class. + +P.S.: You can find the full code used in this tutorial under the "./examples" directory. + +### Data Input Pipeline + +To start with, we need to tell PocketFlow how data files should be parsed. Here, we define a class named `FMnistDataset` to create iterators over the Fashion-MNIST training and test subsets. Every time the iterator is called, it will return a mini-batch of images and corresponding ground-truth labels. + +Below is the full implementation of `FMnistDataset` class (this should be placed under the "./datasets" directory, named as "fmnist_dataset.py"): + +``` Python +import os +import gzip +import numpy as np +import tensorflow as tf + +from datasets.abstract_dataset import AbstractDataset + +FLAGS = tf.app.flags.FLAGS + +tf.app.flags.DEFINE_integer('nb_classes', 10, '# of classes') +tf.app.flags.DEFINE_integer('nb_smpls_train', 60000, '# of samples for training') +tf.app.flags.DEFINE_integer('nb_smpls_val', 5000, '# of samples for validation') +tf.app.flags.DEFINE_integer('nb_smpls_eval', 10000, '# of samples for evaluation') +tf.app.flags.DEFINE_integer('batch_size', 128, 'batch size per GPU for training') +tf.app.flags.DEFINE_integer('batch_size_eval', 100, 'batch size for evaluation') + +# Fashion-MNIST specifications +IMAGE_HEI = 28 +IMAGE_WID = 28 +IMAGE_CHN = 1 + +def load_mnist(image_file, label_file): + """Load images and labels from *.gz files. + + This function is modified from utils/mnist_reader.py in the Fashion-MNIST repo. + + Args: + * image_file: file path to images + * label_file: file path to labels + + Returns: + * images: np.array of the image data + * labels: np.array of the label data + """ + + with gzip.open(label_file, 'rb') as i_file: + labels = np.frombuffer(i_file.read(), dtype=np.uint8, offset=8) + with gzip.open(image_file, 'rb') as i_file: + images = np.frombuffer(i_file.read(), dtype=np.uint8, offset=16) + image_size = IMAGE_HEI * IMAGE_WID * IMAGE_CHN + assert images.size == image_size * len(labels) + images = images.reshape(len(labels), image_size) + + return images, labels + +def parse_fn(image, label, is_train): + """Parse an (image, label) pair and apply data augmentation if needed. + + Args: + * image: image tensor + * label: label tensor + * is_train: whether data augmentation should be applied + + Returns: + * image: image tensor + * label: one-hot label tensor + """ + + # data parsing + label = tf.one_hot(tf.reshape(label, []), FLAGS.nb_classes) + image = tf.cast(tf.reshape(image, [IMAGE_HEI, IMAGE_WID, IMAGE_CHN]), tf.float32) + image = tf.image.per_image_standardization(image) + + # data augmentation + if is_train: + image = tf.image.resize_image_with_crop_or_pad(image, IMAGE_HEI + 8, IMAGE_WID + 8) + image = tf.random_crop(image, [IMAGE_HEI, IMAGE_WID, IMAGE_CHN]) + image = tf.image.random_flip_left_right(image) + + return image, label + +class FMnistDataset(AbstractDataset): + '''Fashion-MNIST dataset.''' + + def __init__(self, is_train): + """Constructor function. + + Args: + * is_train: whether to construct the training subset + """ + + # initialize the base class + super(FMnistDataset, self).__init__(is_train) + + # choose local files or HDFS files w.r.t. FLAGS.data_disk + if FLAGS.data_disk == 'local': + assert FLAGS.data_dir_local is not None, ' must not be None' + data_dir = FLAGS.data_dir_local + elif FLAGS.data_disk == 'hdfs': + assert FLAGS.data_hdfs_host is not None and FLAGS.data_dir_hdfs is not None, \ + 'both and must not be None' + data_dir = FLAGS.data_hdfs_host + FLAGS.data_dir_hdfs + else: + raise ValueError('unrecognized data disk: ' + FLAGS.data_disk) + + # setup paths to image & label files, and read in images & labels + if is_train: + self.batch_size = FLAGS.batch_size + image_file = os.path.join(data_dir, 'train-images-idx3-ubyte.gz') + label_file = os.path.join(data_dir, 'train-labels-idx1-ubyte.gz') + else: + self.batch_size = FLAGS.batch_size_eval + image_file = os.path.join(data_dir, 't10k-images-idx3-ubyte.gz') + label_file = os.path.join(data_dir, 't10k-labels-idx1-ubyte.gz') + self.images, self.labels = load_mnist(image_file, label_file) + self.parse_fn = lambda x, y: parse_fn(x, y, is_train) + + def build(self, enbl_trn_val_split=False): + """Build iterator(s) for tf.data.Dataset() object. + + Args: + * enbl_trn_val_split: whether to split into training & validation subsets + + Returns: + * iterator_trn: iterator for the training subset + * iterator_val: iterator for the validation subset + OR + * iterator: iterator for the chosen subset (training OR testing) + """ + + # create a tf.data.Dataset() object from NumPy arrays + dataset = tf.data.Dataset.from_tensor_slices((self.images, self.labels)) + dataset = dataset.map(self.parse_fn, num_parallel_calls=FLAGS.nb_threads) + + # create iterators for training & validation subsets separately + if self.is_train and enbl_trn_val_split: + iterator_val = self.__make_iterator(dataset.take(FLAGS.nb_smpls_val)) + iterator_trn = self.__make_iterator(dataset.skip(FLAGS.nb_smpls_val)) + return iterator_trn, iterator_val + + return self.__make_iterator(dataset) + + def __make_iterator(self, dataset): + """Make an iterator from tf.data.Dataset. + + Args: + * dataset: tf.data.Dataset object + + Returns: + * iterator: iterator for the dataset + """ + + dataset = dataset.apply(tf.contrib.data.shuffle_and_repeat(buffer_size=FLAGS.buffer_size)) + dataset = dataset.batch(self.batch_size) + dataset = dataset.prefetch(FLAGS.prefetch_size) + iterator = dataset.make_one_shot_iterator() + + return iterator +``` + +When creating an object of `FMnistDataset` class, an extra argument named `is_train` should be provided to toggle between the training and test subsets. The data files can be either store on the local machine or the HDFS cluster, and the directory path is specified in the path configuration file, *e.g.*: + +``` plain +data_dir_local_fmnist = /home/user_name/datasets/Fashion-MNIST +``` + +The constructor function loads images and labels from *.gz files, each stored in a NumPy array. The `build` function is then used to create a TensorFlow's data set iterator from these two NumPy arrays. Particularly, if both `enbl_trn_val_split` and `is_train` are True, then the original training subset will be divided into two parts, one for model training and the other for validation. + +### Network Definition + +Now we implement a new `ModelHelper` class to utilize the above data input pipeline to define the network's training workflow. Below is the full implementation of `ModelHelper` class (this should be placed under the "./nets" directory, named as "convnet_at_fmnist.py"): + +``` Python +import tensorflow as tf + +from nets.abstract_model_helper import AbstractModelHelper +from datasets.fmnist_dataset import FMnistDataset +from utils.lrn_rate_utils import setup_lrn_rate_piecewise_constant +from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw + +FLAGS = tf.app.flags.FLAGS + +tf.app.flags.DEFINE_float('nb_epochs_rat', 1.0, '# of training epochs\'s ratio') +tf.app.flags.DEFINE_float('lrn_rate_init', 1e-1, 'initial learning rate') +tf.app.flags.DEFINE_float('batch_size_norm', 128, 'normalization factor of batch size') +tf.app.flags.DEFINE_float('momentum', 0.9, 'momentum coefficient') +tf.app.flags.DEFINE_float('loss_w_dcy', 3e-4, 'weight decaying loss\'s coefficient') + +def forward_fn(inputs, data_format): + """Forward pass function. + + Args: + * inputs: inputs to the network's forward pass + * data_format: data format ('channels_last' OR 'channels_first') + + Returns: + * inputs: outputs from the network's forward pass + """ + + # tranpose the image tensor if needed + if data_format == 'channel_first': + inputs = tf.transpose(inputs, [0, 3, 1, 2]) + + # conv1 + inputs = tf.layers.conv2d(inputs, 32, [5, 5], padding='same', + data_format=data_format, activation=tf.nn.relu, name='conv1') + inputs = tf.layers.max_pooling2d(inputs, [2, 2], 2, data_format=data_format, name='pool1') + + # conv2 + inputs = tf.layers.conv2d(inputs, 64, [5, 5], padding='same', + data_format=data_format, activation=tf.nn.relu, name='conv2') + inputs = tf.layers.max_pooling2d(inputs, [2, 2], 2, data_format=data_format, name='pool2') + + # fc3 + inputs = tf.layers.flatten(inputs, name='flatten') + inputs = tf.layers.dense(inputs, 1024, activation=tf.nn.relu, name='fc3') + + # fc4 + inputs = tf.layers.dense(inputs, FLAGS.nb_classes, name='fc4') + inputs = tf.nn.softmax(inputs, name='softmax') + + return inputs + +class ModelHelper(AbstractModelHelper): + """Model helper for creating a ConvNet model for the Fashion-MNIST dataset.""" + + def __init__(self): + """Constructor function.""" + + # class-independent initialization + super(ModelHelper, self).__init__() + + # initialize training & evaluation subsets + self.dataset_train = FMnistDataset(is_train=True) + self.dataset_eval = FMnistDataset(is_train=False) + + def build_dataset_train(self, enbl_trn_val_split=False): + """Build the data subset for training, usually with data augmentation.""" + + return self.dataset_train.build(enbl_trn_val_split) + + def build_dataset_eval(self): + """Build the data subset for evaluation, usually without data augmentation.""" + + return self.dataset_eval.build() + + def forward_train(self, inputs, data_format='channels_last'): + """Forward computation at training.""" + + return forward_fn(inputs, data_format) + + def forward_eval(self, inputs, data_format='channels_last'): + """Forward computation at evaluation.""" + + return forward_fn(inputs, data_format) + + def calc_loss(self, labels, outputs, trainable_vars): + """Calculate loss (and some extra evaluation metrics).""" + + loss = tf.losses.softmax_cross_entropy(labels, outputs) + loss += FLAGS.loss_w_dcy * tf.add_n([tf.nn.l2_loss(var) for var in trainable_vars]) + accuracy = tf.reduce_mean( + tf.cast(tf.equal(tf.argmax(labels, axis=1), tf.argmax(outputs, axis=1)), tf.float32)) + metrics = {'accuracy': accuracy} + + return loss, metrics + + def setup_lrn_rate(self, global_step): + """Setup the learning rate (and number of training iterations).""" + + nb_epochs = 160 + idxs_epoch = [40, 80, 120] + decay_rates = [1.0, 0.1, 0.01, 0.001] + batch_size = FLAGS.batch_size * (1 if not FLAGS.enbl_multi_gpu else mgw.size()) + lrn_rate = setup_lrn_rate_piecewise_constant(global_step, batch_size, idxs_epoch, decay_rates) + nb_iters = int(FLAGS.nb_smpls_train * nb_epochs * FLAGS.nb_epochs_rat / batch_size) + + return lrn_rate, nb_iters + + @property + def model_name(self): + """Model's name.""" + + return 'convnet' + + @property + def dataset_name(self): + """Dataset's name.""" + + return 'fmnist' +``` + +In the `build_dataset_train` and `build_dataset_eval` functions, we adopt the previously introduced `FMnistDataset` class to define the data input pipeline. The network forward-pass computation is defined in the `forward_train` and `forward_eval` functions, which corresponds to the training and evaluation graph, respectivley. The training graph is slightly different from evaluation graph, such as operations related to the batch normalization layers. The `calc_loss` function calculates the loss function's value and extra evaluation metrics, *e.g.* classification accuracy. Finally, the `setup_lrn_rate` function defines the learning rate schedule, as well as how many training iterations are need. + +### Execution Script + +Besides the self-defined `ModelHelper` class, we still need an execution script to pass it to the corresponding model compression component to start the training process. Below is the full implementation (this should be placed under the "./nets" directory, named as "convnet_at_fmnist_run.py"): + +``` Python +import traceback +import tensorflow as tf + +from nets.convnet_at_fmnist import ModelHelper +from learners.learner_utils import create_learner + +FLAGS = tf.app.flags.FLAGS + +tf.app.flags.DEFINE_string('log_dir', './logs', 'logging directory') +tf.app.flags.DEFINE_boolean('enbl_multi_gpu', False, 'enable multi-GPU training') +tf.app.flags.DEFINE_string('learner', 'full-prec', 'learner\'s name') +tf.app.flags.DEFINE_string('exec_mode', 'train', 'execution mode: train / eval') +tf.app.flags.DEFINE_boolean('debug', False, 'debugging information') + +def main(unused_argv): + """Main entry.""" + + try: + # setup the TF logging routine + if FLAGS.debug: + tf.logging.set_verbosity(tf.logging.DEBUG) + else: + tf.logging.set_verbosity(tf.logging.INFO) + sm_writer = tf.summary.FileWriter(FLAGS.log_dir) + + # display FLAGS's values + tf.logging.info('FLAGS:') + for key, value in FLAGS.flag_values_dict().items(): + tf.logging.info('{}: {}'.format(key, value)) + + # build the model helper & learner + model_helper = ModelHelper() + learner = create_learner(sm_writer, model_helper) + + # execute the learner + if FLAGS.exec_mode == 'train': + learner.train() + elif FLAGS.exec_mode == 'eval': + learner.download_model() + learner.evaluate() + else: + raise ValueError('unrecognized execution mode: ' + FLAGS.exec_mode) + + # exit normally + return 0 + except ValueError: + traceback.print_exc() + return 1 # exit with errors + +if __name__ == '__main__': + tf.app.run() +``` + +## Network Training with PocketFlow + +To train the self-defined model without any constraint, use `FullPrecLearner`: + +``` bash +$ ./scripts/run_local.sh nets/convnet_at_fmnist_run.py \ + --learner full-prec +``` + +To train the self-defined model with the uniform quantization constraint, use `UniformQuantTFLearner`: + +``` bash +$ ./scripts/run_local.sh nets/convnet_at_fmnist_run.py \ + --learner uniform-tf +``` diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 9a00639..27352d7 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -15,6 +15,7 @@ nav: - Hyper-parameter Optimizers: - Reinforcement Learning: reinforcement_learning.md - AutoML-based Methods: automl_based_methods.md +- Self-defined Models: self_defined_models.md - Performance: performance.md - Frequently Asked Questions: faq.md - Appendix: diff --git a/examples/convnet_at_fmnist.py b/examples/convnet_at_fmnist.py new file mode 100644 index 0000000..b95d788 --- /dev/null +++ b/examples/convnet_at_fmnist.py @@ -0,0 +1,135 @@ +# Tencent is pleased to support the open source community by making PocketFlow available. +# +# Copyright (C) 2018 THL A29 Limited, a Tencent company. All rights reserved. +# +# Licensed under the BSD 3-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model helper for creating a ConvNet model for the Fashion-MNIST dataset.""" + +import tensorflow as tf + +from nets.abstract_model_helper import AbstractModelHelper +from datasets.fmnist_dataset import FMnistDataset +from utils.lrn_rate_utils import setup_lrn_rate_piecewise_constant +from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw + +FLAGS = tf.app.flags.FLAGS + +tf.app.flags.DEFINE_float('nb_epochs_rat', 1.0, '# of training epochs\'s ratio') +tf.app.flags.DEFINE_float('lrn_rate_init', 1e-1, 'initial learning rate') +tf.app.flags.DEFINE_float('batch_size_norm', 128, 'normalization factor of batch size') +tf.app.flags.DEFINE_float('momentum', 0.9, 'momentum coefficient') +tf.app.flags.DEFINE_float('loss_w_dcy', 3e-4, 'weight decaying loss\'s coefficient') + +def forward_fn(inputs, data_format): + """Forward pass function. + + Args: + * inputs: inputs to the network's forward pass + * data_format: data format ('channels_last' OR 'channels_first') + + Returns: + * inputs: outputs from the network's forward pass + """ + + # tranpose the image tensor if needed + if data_format == 'channel_first': + inputs = tf.transpose(inputs, [0, 3, 1, 2]) + + # conv1 + inputs = tf.layers.conv2d(inputs, 32, [5, 5], padding='same', + data_format=data_format, activation=tf.nn.relu, name='conv1') + inputs = tf.layers.max_pooling2d(inputs, [2, 2], 2, data_format=data_format, name='pool1') + + # conv2 + inputs = tf.layers.conv2d(inputs, 64, [5, 5], padding='same', + data_format=data_format, activation=tf.nn.relu, name='conv2') + inputs = tf.layers.max_pooling2d(inputs, [2, 2], 2, data_format=data_format, name='pool2') + + # fc3 + inputs = tf.layers.flatten(inputs, name='flatten') + inputs = tf.layers.dense(inputs, 1024, activation=tf.nn.relu, name='fc3') + + # fc4 + inputs = tf.layers.dense(inputs, FLAGS.nb_classes, name='fc4') + inputs = tf.nn.softmax(inputs, name='softmax') + + return inputs + +class ModelHelper(AbstractModelHelper): + """Model helper for creating a ConvNet model for the Fashion-MNIST dataset.""" + + def __init__(self): + """Constructor function.""" + + # class-independent initialization + super(ModelHelper, self).__init__() + + # initialize training & evaluation subsets + self.dataset_train = FMnistDataset(is_train=True) + self.dataset_eval = FMnistDataset(is_train=False) + + def build_dataset_train(self, enbl_trn_val_split=False): + """Build the data subset for training, usually with data augmentation.""" + + return self.dataset_train.build(enbl_trn_val_split) + + def build_dataset_eval(self): + """Build the data subset for evaluation, usually without data augmentation.""" + + return self.dataset_eval.build() + + def forward_train(self, inputs, data_format='channels_last'): + """Forward computation at training.""" + + return forward_fn(inputs, data_format) + + def forward_eval(self, inputs, data_format='channels_last'): + """Forward computation at evaluation.""" + + return forward_fn(inputs, data_format) + + def calc_loss(self, labels, outputs, trainable_vars): + """Calculate loss (and some extra evaluation metrics).""" + + loss = tf.losses.softmax_cross_entropy(labels, outputs) + loss += FLAGS.loss_w_dcy * tf.add_n([tf.nn.l2_loss(var) for var in trainable_vars]) + accuracy = tf.reduce_mean( + tf.cast(tf.equal(tf.argmax(labels, axis=1), tf.argmax(outputs, axis=1)), tf.float32)) + metrics = {'accuracy': accuracy} + + return loss, metrics + + def setup_lrn_rate(self, global_step): + """Setup the learning rate (and number of training iterations).""" + + nb_epochs = 160 + idxs_epoch = [40, 80, 120] + decay_rates = [1.0, 0.1, 0.01, 0.001] + batch_size = FLAGS.batch_size * (1 if not FLAGS.enbl_multi_gpu else mgw.size()) + lrn_rate = setup_lrn_rate_piecewise_constant(global_step, batch_size, idxs_epoch, decay_rates) + nb_iters = int(FLAGS.nb_smpls_train * nb_epochs * FLAGS.nb_epochs_rat / batch_size) + + return lrn_rate, nb_iters + + @property + def model_name(self): + """Model's name.""" + + return 'convnet' + + @property + def dataset_name(self): + """Dataset's name.""" + + return 'fmnist' diff --git a/examples/convnet_at_fmnist_run.py b/examples/convnet_at_fmnist_run.py new file mode 100644 index 0000000..008aaf4 --- /dev/null +++ b/examples/convnet_at_fmnist_run.py @@ -0,0 +1,69 @@ +# Tencent is pleased to support the open source community by making PocketFlow available. +# +# Copyright (C) 2018 THL A29 Limited, a Tencent company. All rights reserved. +# +# Licensed under the BSD 3-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Execution script for ConvNet models on the Fashion-MNIST dataset.""" + +import traceback +import tensorflow as tf + +from nets.convnet_at_fmnist import ModelHelper +from learners.learner_utils import create_learner + +FLAGS = tf.app.flags.FLAGS + +tf.app.flags.DEFINE_string('log_dir', './logs', 'logging directory') +tf.app.flags.DEFINE_boolean('enbl_multi_gpu', False, 'enable multi-GPU training') +tf.app.flags.DEFINE_string('learner', 'full-prec', 'learner\'s name') +tf.app.flags.DEFINE_string('exec_mode', 'train', 'execution mode: train / eval') +tf.app.flags.DEFINE_boolean('debug', False, 'debugging information') + +def main(unused_argv): + """Main entry.""" + + try: + # setup the TF logging routine + if FLAGS.debug: + tf.logging.set_verbosity(tf.logging.DEBUG) + else: + tf.logging.set_verbosity(tf.logging.INFO) + sm_writer = tf.summary.FileWriter(FLAGS.log_dir) + + # display FLAGS's values + tf.logging.info('FLAGS:') + for key, value in FLAGS.flag_values_dict().items(): + tf.logging.info('{}: {}'.format(key, value)) + + # build the model helper & learner + model_helper = ModelHelper() + learner = create_learner(sm_writer, model_helper) + + # execute the learner + if FLAGS.exec_mode == 'train': + learner.train() + elif FLAGS.exec_mode == 'eval': + learner.download_model() + learner.evaluate() + else: + raise ValueError('unrecognized execution mode: ' + FLAGS.exec_mode) + + # exit normally + return 0 + except ValueError: + traceback.print_exc() + return 1 # exit with errors + +if __name__ == '__main__': + tf.app.run() diff --git a/examples/fmnist_dataset.py b/examples/fmnist_dataset.py new file mode 100644 index 0000000..be40576 --- /dev/null +++ b/examples/fmnist_dataset.py @@ -0,0 +1,166 @@ +# Tencent is pleased to support the open source community by making PocketFlow available. +# +# Copyright (C) 2018 THL A29 Limited, a Tencent company. All rights reserved. +# +# Licensed under the BSD 3-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Fashion-MNIST dataset.""" + +import os +import gzip +import numpy as np +import tensorflow as tf + +from datasets.abstract_dataset import AbstractDataset + +FLAGS = tf.app.flags.FLAGS + +tf.app.flags.DEFINE_integer('nb_classes', 10, '# of classes') +tf.app.flags.DEFINE_integer('nb_smpls_train', 60000, '# of samples for training') +tf.app.flags.DEFINE_integer('nb_smpls_val', 5000, '# of samples for validation') +tf.app.flags.DEFINE_integer('nb_smpls_eval', 10000, '# of samples for evaluation') +tf.app.flags.DEFINE_integer('batch_size', 128, 'batch size per GPU for training') +tf.app.flags.DEFINE_integer('batch_size_eval', 100, 'batch size for evaluation') + +# Fashion-MNIST specifications +IMAGE_HEI = 28 +IMAGE_WID = 28 +IMAGE_CHN = 1 + +def load_mnist(image_file, label_file): + """Load images and labels from *.gz files. + + This function is modified from utils/mnist_reader.py in the Fashion-MNIST repo. + + Args: + * image_file: file path to images + * label_file: file path to labels + + Returns: + * images: np.array of the image data + * labels: np.array of the label data + """ + + with gzip.open(label_file, 'rb') as i_file: + labels = np.frombuffer(i_file.read(), dtype=np.uint8, offset=8) + with gzip.open(image_file, 'rb') as i_file: + images = np.frombuffer(i_file.read(), dtype=np.uint8, offset=16) + image_size = IMAGE_HEI * IMAGE_WID * IMAGE_CHN + assert images.size == image_size * len(labels) + images = images.reshape(len(labels), image_size) + + return images, labels + +def parse_fn(image, label, is_train): + """Parse an (image, label) pair and apply data augmentation if needed. + + Args: + * image: image tensor + * label: label tensor + * is_train: whether data augmentation should be applied + + Returns: + * image: image tensor + * label: one-hot label tensor + """ + + # data parsing + label = tf.one_hot(tf.reshape(label, []), FLAGS.nb_classes) + image = tf.cast(tf.reshape(image, [IMAGE_HEI, IMAGE_WID, IMAGE_CHN]), tf.float32) + image = tf.image.per_image_standardization(image) + + # data augmentation + if is_train: + image = tf.image.resize_image_with_crop_or_pad(image, IMAGE_HEI + 8, IMAGE_WID + 8) + image = tf.random_crop(image, [IMAGE_HEI, IMAGE_WID, IMAGE_CHN]) + image = tf.image.random_flip_left_right(image) + + return image, label + +class FMnistDataset(AbstractDataset): + '''Fashion-MNIST dataset.''' + + def __init__(self, is_train): + """Constructor function. + + Args: + * is_train: whether to construct the training subset + """ + + # initialize the base class + super(FMnistDataset, self).__init__(is_train) + + # choose local files or HDFS files w.r.t. FLAGS.data_disk + if FLAGS.data_disk == 'local': + assert FLAGS.data_dir_local is not None, ' must not be None' + data_dir = FLAGS.data_dir_local + elif FLAGS.data_disk == 'hdfs': + assert FLAGS.data_hdfs_host is not None and FLAGS.data_dir_hdfs is not None, \ + 'both and must not be None' + data_dir = FLAGS.data_hdfs_host + FLAGS.data_dir_hdfs + else: + raise ValueError('unrecognized data disk: ' + FLAGS.data_disk) + + # setup paths to image & label files, and read in images & labels + if is_train: + self.batch_size = FLAGS.batch_size + image_file = os.path.join(data_dir, 'train-images-idx3-ubyte.gz') + label_file = os.path.join(data_dir, 'train-labels-idx1-ubyte.gz') + else: + self.batch_size = FLAGS.batch_size_eval + image_file = os.path.join(data_dir, 't10k-images-idx3-ubyte.gz') + label_file = os.path.join(data_dir, 't10k-labels-idx1-ubyte.gz') + self.images, self.labels = load_mnist(image_file, label_file) + self.parse_fn = lambda x, y: parse_fn(x, y, is_train) + + def build(self, enbl_trn_val_split=False): + """Build iterator(s) for tf.data.Dataset() object. + + Args: + * enbl_trn_val_split: whether to split into training & validation subsets + + Returns: + * iterator_trn: iterator for the training subset + * iterator_val: iterator for the validation subset + OR + * iterator: iterator for the chosen subset (training OR testing) + """ + + # create a tf.data.Dataset() object from NumPy arrays + dataset = tf.data.Dataset.from_tensor_slices((self.images, self.labels)) + dataset = dataset.map(self.parse_fn, num_parallel_calls=FLAGS.nb_threads) + + # create iterators for training & validation subsets separately + if self.is_train and enbl_trn_val_split: + iterator_val = self.__make_iterator(dataset.take(FLAGS.nb_smpls_val)) + iterator_trn = self.__make_iterator(dataset.skip(FLAGS.nb_smpls_val)) + return iterator_trn, iterator_val + + return self.__make_iterator(dataset) + + def __make_iterator(self, dataset): + """Make an iterator from tf.data.Dataset. + + Args: + * dataset: tf.data.Dataset object + + Returns: + * iterator: iterator for the dataset + """ + + dataset = dataset.apply(tf.contrib.data.shuffle_and_repeat(buffer_size=FLAGS.buffer_size)) + dataset = dataset.batch(self.batch_size) + dataset = dataset.prefetch(FLAGS.prefetch_size) + iterator = dataset.make_one_shot_iterator() + + return iterator From 97614a770d2c983a94e91141c4cd964aaeab23cf Mon Sep 17 00:00:00 2001 From: Jiaxiang Wu Date: Tue, 4 Dec 2018 15:41:08 +0800 Subject: [PATCH 015/173] Optimized Graph Edit for GPU Inference with *.pb Models (#119) * allow to config via FLAGS * rename conversion script for channel pruned models * support NHWC (*.pb & *.tflite) and NCHW (*.pb only) * support fake pruning for speed test * use transformation for NCHW-formatted models * measure elapsed time through multiple runs * detailed comments on importing * execute some warm-up runs to stabilize the measured time * add NCHW support for LeNet models on CIFAR-10 --- nets/lenet_at_cifar10.py | 4 + tools/conversion/convert_data_format.py | 10 +- ...s.py => export_chn_pruned_tflite_model.py} | 110 +++++++++++++++--- 3 files changed, 102 insertions(+), 22 deletions(-) rename tools/conversion/{export_pb_tflite_models.py => export_chn_pruned_tflite_model.py} (72%) diff --git a/nets/lenet_at_cifar10.py b/nets/lenet_at_cifar10.py index c5583c4..d81f859 100644 --- a/nets/lenet_at_cifar10.py +++ b/nets/lenet_at_cifar10.py @@ -42,6 +42,10 @@ def forward_fn(inputs, data_format): * inputs: outputs from the network's forward pass """ + # convert inputs from channels_last (NHWC) to channels_first (NCHW) + if data_format == 'channels_first': + inputs = tf.transpose(inputs, [0, 3, 1, 2]) + # conv1 inputs = tf.layers.conv2d(inputs, 32, [5, 5], data_format=data_format, name='conv1') inputs = tf.nn.relu(inputs, name='relu1') diff --git a/tools/conversion/convert_data_format.py b/tools/conversion/convert_data_format.py index a28319e..85c7572 100644 --- a/tools/conversion/convert_data_format.py +++ b/tools/conversion/convert_data_format.py @@ -20,8 +20,11 @@ import traceback import tensorflow as tf -# you may need to replace before conversion -from nets.resnet_at_cifar10 import ModelHelper +# NOTE: un-comment the corresponding before conversion +#from nets.lenet_at_cifar10 import ModelHelper +#from nets.resnet_at_cifar10 import ModelHelper +from nets.resnet_at_ilsvrc12 import ModelHelper +#from nets.mobilenet_at_ilsvrc12 import ModelHelper FLAGS = tf.app.flags.FLAGS @@ -29,6 +32,7 @@ tf.app.flags.DEFINE_boolean('enbl_multi_gpu', False, 'enable multi-GPU training') tf.app.flags.DEFINE_string('model_dir_in', './models', 'input model directory') tf.app.flags.DEFINE_string('model_dir_out', './models_out', 'output model directory') +tf.app.flags.DEFINE_string('model_scope', 'model', 'model\'s variable scope name') tf.app.flags.DEFINE_string('data_format_src', 'channels_last', 'data format in the source model') def main(unused_argv): @@ -48,7 +52,7 @@ def main(unused_argv): # create the model helper model_helper = ModelHelper() data_scope = 'data' - model_scope = 'pruned_model' + model_scope = FLAGS.model_scope # bulid a graph with the target data format and rewrite checkpoint files with tf.Graph().as_default(): diff --git a/tools/conversion/export_pb_tflite_models.py b/tools/conversion/export_chn_pruned_tflite_model.py similarity index 72% rename from tools/conversion/export_pb_tflite_models.py rename to tools/conversion/export_chn_pruned_tflite_model.py index 474cb4c..7f5ffdd 100644 --- a/tools/conversion/export_pb_tflite_models.py +++ b/tools/conversion/export_chn_pruned_tflite_model.py @@ -14,11 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Export *.pb & *.tflite models from checkpoint files.""" +"""Export a channel-pruned *.tflite model from checkpoint files.""" import os import re import traceback +from timeit import default_timer as timer import numpy as np import tensorflow as tf from tensorflow.contrib import graph_editor @@ -29,6 +30,10 @@ tf.app.flags.DEFINE_string('model_dir', './models', 'model directory') tf.app.flags.DEFINE_string('input_coll', 'images_final', 'input tensor\'s collection') tf.app.flags.DEFINE_string('output_coll', 'logits_final', 'output tensor\'s collection') +tf.app.flags.DEFINE_boolean('enbl_fake_prune', False, 'enable fake pruning (for speed test only)') +tf.app.flags.DEFINE_float('fake_prune_ratio', 0.5, 'fake pruning ratio') +tf.app.flags.DEFINE_integer('nb_repts_warmup', 100, '# of repeated runs for warm-up') +tf.app.flags.DEFINE_integer('nb_repts', 100, '# of repeated runs for elapsed time measurement') def get_file_path_meta(): """Get the file path to the *.meta data. @@ -64,6 +69,26 @@ def get_input_name_n_shape(file_path): return input_name, input_shape +def get_data_format(sess): + """Get the data format of convolutional layers. + + Args: + * sess: TensorFlow session + + Returns: + * data_format: data format of convolutional layers + """ + + data_format = None + pattern = re.compile('Conv2D$') + for op in tf.get_default_graph().get_operations(): + if re.search(pattern, op.name) is not None: + data_format = op.get_attr('data_format').decode('utf-8') + tf.logging.info('data format: ' + data_format) + break + + return data_format + def convert_pb_model_to_tflite(file_path_pb, file_path_tflite, net_input_name, net_output_name): """Convert *.pb model to a *.tflite model. @@ -105,8 +130,13 @@ def test_pb_model(file_path, net_input_name, net_output_name, net_input_data): net_input = graph.get_tensor_by_name('import/' + net_input_name + ':0') net_output = graph.get_tensor_by_name('import/' + net_output_name + ':0') tf.logging.info('input: {} / output: {}'.format(net_input.name, net_output.name)) - net_output_data = sess.run(net_output, feed_dict={net_input: net_input_data}) + for idx in range(FLAGS.nb_repts_warmup + FLAGS.nb_repts): + if idx == FLAGS.nb_repts_warmup: + time_beg = timer() + net_output_data = sess.run(net_output, feed_dict={net_input: net_input_data}) + time_elapsed = (timer() - time_beg) / FLAGS.nb_repts tf.logging.info('outputs from the *.pb model: {}'.format(net_output_data)) + tf.logging.info('time consumption of *.pb model: %.2f ms' % (time_elapsed * 1000)) def test_tflite_model(file_path, net_input_data): """Test the *.tflite model. @@ -127,10 +157,15 @@ def test_tflite_model(file_path, net_input_data): tf.logging.info('output details: {}'.format(output_details)) # test the model with given inputs - interpreter.set_tensor(input_details[0]['index'], net_input_data) - interpreter.invoke() - net_output_data = interpreter.get_tensor(output_details[0]['index']) + for idx in range(FLAGS.nb_repts_warmup + FLAGS.nb_repts): + if idx == FLAGS.nb_repts_warmup: + time_beg = timer() + interpreter.set_tensor(input_details[0]['index'], net_input_data) + interpreter.invoke() + net_output_data = interpreter.get_tensor(output_details[0]['index']) + time_elapsed = (timer() - time_beg) / FLAGS.nb_repts tf.logging.info('outputs from the *.tflite model: {}'.format(net_output_data)) + tf.logging.info('time consumption of *.tflite model: %.2f ms' % (time_elapsed * 1000)) def is_initialized(sess, var): """Check whether a variable is initialized. @@ -146,6 +181,25 @@ def is_initialized(sess, var): except tf.errors.FailedPreconditionError: return False +def apply_fake_pruning(kernel): + """Apply fake pruning to the convolutional kernel. + + Args: + * kernel: original convolutional kernel + + Returns: + * kernel: randomly pruned convolutional kernel + """ + + tf.logging.info('kernel shape: {}'.format(kernel.shape)) + nb_chns = kernel.shape[2] + idxs_all = np.arange(nb_chns) + np.random.shuffle(idxs_all) + idxs_pruned = idxs_all[:int(nb_chns * FLAGS.fake_prune_ratio)] + kernel[:, :, idxs_pruned, :] = 0.0 + + return kernel + def replace_dropout_layers(): """Replace dropout layers with identity mappings. @@ -166,11 +220,12 @@ def replace_dropout_layers(): return op_outputs_old, op_outputs_new -def insert_alt_routines(sess): +def insert_alt_routines(sess, graph_trans_mthd): """Insert alternative rountines for convolutional layers. Args: * sess: TensorFlow session + * graph_trans_mthd: graph transformation method Returns: * op_outputs_old: output nodes to be swapped in the old graph @@ -185,9 +240,11 @@ def insert_alt_routines(sess): if not is_initialized(sess, op.inputs[1]): continue - # insert alternative routines using tf.nn.conv2d + # detect which channels to be pruned tf.logging.info('transforming OP: ' + op.name) kernel = sess.run(op.inputs[1]) + if FLAGS.enbl_fake_prune: + kernel = apply_fake_pruning(kernel) kernel_chn_in = kernel.shape[2] strides = op.get_attr('strides') padding = op.get_attr('padding').decode('utf-8') @@ -198,10 +255,20 @@ def insert_alt_routines(sess): kernel_gthr = np.zeros((1, 1, kernel_chn_in, nnzs.size)) kernel_gthr[0, 0, nnzs, np.arange(nnzs.size)] = 1.0 kernel_shrk = kernel[:, :, nnzs, :] - x = tf.nn.conv2d(op.inputs[0], kernel_gthr, [1, 1, 1, 1], 'SAME', data_format=data_format) - x = tf.nn.conv2d( - x, kernel_shrk, strides, padding, data_format=data_format, dilations=dilations) + # replace channel pruned convolutional with cheaper operations + if graph_trans_mthd == 'gather': + x = tf.gather(op.inputs[0], nnzs, axis=1) + x = tf.nn.conv2d( + x, kernel_shrk, strides, padding, data_format=data_format, dilations=dilations) + elif graph_trans_mthd == '1x1_conv': + x = tf.nn.conv2d(op.inputs[0], kernel_gthr, [1, 1, 1, 1], 'SAME', data_format=data_format) + x = tf.nn.conv2d( + x, kernel_shrk, strides, padding, data_format=data_format, dilations=dilations) + else: + raise ValueError('unrecognized graph transformation method: ' + graph_trans_mthd) + + # obtain old and new routines' outputs op_outputs_old += [op.outputs[0]] op_outputs_new += [x] @@ -228,6 +295,10 @@ def export_pb_tflite_model(net, file_path_meta, file_path_pb, file_path_tflite, file_path_meta, input_map={net['input_name_ckpt']: net_input}) saver.restore(sess, file_path_meta.replace('.meta', '')) + # obtain the data format and determine which graph transformation method to be used + data_format = get_data_format(sess) + graph_trans_mthd = 'gather' if data_format == 'NCHW' else '1x1_conv' + # obtain the output node net_logits = tf.get_collection(FLAGS.output_coll)[0] net_output = tf.nn.softmax(net_logits, name=net['output_name']) @@ -244,7 +315,7 @@ def export_pb_tflite_model(net, file_path_meta, file_path_pb, file_path_tflite, # edit the graph by inserting alternative routines for each convolutional layer if edit_graph: - op_outputs_old, op_outputs_new = insert_alt_routines(sess) + op_outputs_old, op_outputs_new = insert_alt_routines(sess, graph_trans_mthd) sess.close() graph_editor.swap_outputs(op_outputs_old, op_outputs_new) sess = tf.Session() # open a new session @@ -256,14 +327,15 @@ def export_pb_tflite_model(net, file_path_meta, file_path_pb, file_path_tflite, file_name_pb = os.path.basename(file_path_pb) tf.train.write_graph(graph_def, FLAGS.model_dir, file_name_pb, as_text=False) tf.logging.info(file_path_pb + ' generated') - - # convert the *.pb model to a *.tflite model - convert_pb_model_to_tflite(file_path_pb, file_path_tflite, net['input_name'], net['output_name']) - tf.logging.info(file_path_tflite + ' generated') - - # test *.pb & *.tflite models - test_pb_model(file_path_pb, net['input_name'], net['output_name'], net['input_data']) - test_tflite_model(file_path_tflite, net['input_data']) + test_pb_model(file_path_pb, net['input_name'], net['output_name'], net['input_data']) + + # convert the *.pb model to a *.tflite model (only NHWC is supported) + if data_format == 'NHWC': + convert_pb_model_to_tflite(file_path_pb, file_path_tflite, net['input_name'], net['output_name']) + tf.logging.info(file_path_tflite + ' generated') + test_tflite_model(file_path_tflite, net['input_data']) + else: + tf.logging.warning('*.tflite model not generated since NCHW is not supported by TF-Lite') def main(unused_argv): """Main entry. From cd30eca98f8209f8ddaebce298c86fb9c9176f7f Mon Sep 17 00:00:00 2001 From: Jiaxiang Wu Date: Mon, 10 Dec 2018 16:21:24 +0800 Subject: [PATCH 016/173] add a benchmark tool for *.pb and *.tflite models (#136) --- tools/benchmark/calc_inference_time.py | 114 +++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 tools/benchmark/calc_inference_time.py diff --git a/tools/benchmark/calc_inference_time.py b/tools/benchmark/calc_inference_time.py new file mode 100644 index 0000000..d44dcb6 --- /dev/null +++ b/tools/benchmark/calc_inference_time.py @@ -0,0 +1,114 @@ +# Tencent is pleased to support the open source community by making PocketFlow available. +# +# Copyright (C) 2018 THL A29 Limited, a Tencent company. All rights reserved. +# +# Licensed under the BSD 3-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Measure the time consumption of *.pb and *.tflite models.""" + +import traceback +from timeit import default_timer as timer +import numpy as np +import tensorflow as tf + +FLAGS = tf.app.flags.FLAGS + +tf.app.flags.DEFINE_string('model_file', None, 'model file path') +tf.app.flags.DEFINE_string('input_name', 'net_input', 'input tensor\'s name in the *.pb model') +tf.app.flags.DEFINE_string('output_name', 'net_output', 'output tensor\'s name in the *.pb model') +tf.app.flags.DEFINE_string('input_dtype', 'float32', + 'input tensor\'s data type in the *.tflite model') +tf.app.flags.DEFINE_integer('batch_size', 1, 'batch size for run-time benchmark') +tf.app.flags.DEFINE_integer('nb_repts_warmup', 100, '# of repeated runs for warm-up') +tf.app.flags.DEFINE_integer('nb_repts', 100, '# of repeated runs for elapsed time measurement') + +def test_pb_model(): + """Test the *.pb model.""" + + with tf.Graph().as_default() as graph: + sess = tf.Session() + + # restore the model + graph_def = tf.GraphDef() + with tf.gfile.GFile(FLAGS.model_file, 'rb') as i_file: + graph_def.ParseFromString(i_file.read()) + tf.import_graph_def(graph_def) + + # obtain input & output nodes and then test the model + net_input = graph.get_tensor_by_name('import/' + FLAGS.input_name + ':0') + net_output = graph.get_tensor_by_name('import/' + FLAGS.output_name + ':0') + net_input_data = np.zeros(tuple([FLAGS.batch_size] + list(net_input.shape[1:]))) + for idx in range(FLAGS.nb_repts_warmup + FLAGS.nb_repts): + if idx == FLAGS.nb_repts_warmup: + time_beg = timer() + sess.run(net_output, feed_dict={net_input: net_input_data}) + time_elapsed = (timer() - time_beg) / FLAGS.nb_repts / FLAGS.batch_size + tf.logging.info('time consumption of *.pb model: %.2f ms' % (time_elapsed * 1000)) + +def test_tflite_model(): + """Test the *.tflite model.""" + + # restore the model and allocate tensors + interpreter = tf.contrib.lite.Interpreter(model_path=FLAGS.model_file) + interpreter.allocate_tensors() + + # get input & output tensors + input_details = interpreter.get_input_details() + output_details = interpreter.get_output_details() + assert len(input_details) == 1, ' should contain only one element' + if FLAGS.input_dtype == 'uint8': + net_input_data = np.zeros(input_details[0]['shape'], dtype=np.uint8) + elif FLAGS.input_dtype == 'float32': + net_input_data = np.zeros(input_details[0]['shape'], dtype=np.float32) + else: + raise ValueError('unrecognized input data type: ' + FLAGS.input_dtype) + + # test the model with given inputs + for idx in range(FLAGS.nb_repts_warmup + FLAGS.nb_repts): + if idx == FLAGS.nb_repts_warmup: + time_beg = timer() + interpreter.set_tensor(input_details[0]['index'], net_input_data) + interpreter.invoke() + interpreter.get_tensor(output_details[0]['index']) + time_elapsed = (timer() - time_beg) / FLAGS.nb_repts + tf.logging.info('time consumption of *.tflite model: %.2f ms' % (time_elapsed * 1000)) + +def main(unused_argv): + """Main entry. + + Args: + * unused_argv: unused arguments (after FLAGS is parsed) + """ + + try: + # setup the TF logging routine + tf.logging.set_verbosity(tf.logging.INFO) + + # call benchmark routines for *.pb / *.tflite models + if FLAGS.model_file is None: + raise ValueError(' must not be None') + elif FLAGS.model_file.endswith('.pb'): + test_pb_model() + elif FLAGS.model_file.endswith('.tflite'): + test_tflite_model() + else: + raise ValueError('unrecognized model file path: ' + FLAGS.model_file) + + # exit normally + return 0 + except ValueError: + traceback.print_exc() + return 1 # exit with errors + +if __name__ == '__main__': + tf.app.run() From 6e610bab9468a9d5beb425e6f5d1ce6f3a2e85e5 Mon Sep 17 00:00:00 2001 From: Jiaxiang Wu Date: Mon, 10 Dec 2018 18:43:27 +0800 Subject: [PATCH 017/173] clarify that FLOPs is used to measure complexity in CP results (#137) --- docs/docs/index.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/docs/index.md b/docs/docs/index.md index d3f6707..9d10fdf 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -42,12 +42,12 @@ For complete evaluation results, please refer to [here](https://pocketflow.githu We adopt the DDPG algorithm as the RL agent to find the optimal layer-wise pruning ratios, and use group fine-tuning to further improve the compressed model's accuracy: -| Model | Pruning Ratio | Uniform | RL-based | RL-based + Group Fine-tuning | -|:------------:|:-------------:|:-------:|:-------------:|:----------------------------:| -| MobileNet-v1 | 50% | 66.5% | 67.8% (+1.3%) | 67.9% (+1.4%) | -| MobileNet-v1 | 60% | 66.2% | 66.9% (+0.7%) | 67.0% (+0.8%) | -| MobileNet-v1 | 70% | 64.4% | 64.5% (+0.1%) | 64.8% (+0.4%) | -| Mobilenet-v1 | 80% | 61.4% | 61.4% (+0.0%) | 62.2% (+0.8%) | +| Model | FLOPs | Uniform | RL-based | RL-based + Group Fine-tuning | +|:------------:|:-----:|:-------:|:-------------:|:----------------------------:| +| MobileNet-v1 | 50% | 66.5% | 67.8% (+1.3%) | 67.9% (+1.4%) | +| MobileNet-v1 | 40% | 66.2% | 66.9% (+0.7%) | 67.0% (+0.8%) | +| MobileNet-v1 | 30% | 64.4% | 64.5% (+0.1%) | 64.8% (+0.4%) | +| Mobilenet-v1 | 20% | 61.4% | 61.4% (+0.0%) | 62.2% (+0.8%) | ### Weight Sparsification From f3ac17a23e08041f8674b9b27ea746e69e1e3a43 Mon Sep 17 00:00:00 2001 From: Jiaxiang Wu Date: Mon, 10 Dec 2018 18:50:59 +0800 Subject: [PATCH 018/173] compute the binary mask with instead of (#134) --- learners/weight_sparsification/learner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/learners/weight_sparsification/learner.py b/learners/weight_sparsification/learner.py index d93590c..fb2fe06 100644 --- a/learners/weight_sparsification/learner.py +++ b/learners/weight_sparsification/learner.py @@ -278,10 +278,10 @@ def __build_masks(self): var_bkup = tf.get_variable(name, initializer=var.initialized_value(), trainable=False) # create update operations - mask_thres = tf.contrib.distributions.percentile(tf.abs(var), prune_ratio * 100) var_bkup_update_op = var_bkup.assign(tf.where(mask > 0.5, var, var_bkup)) with tf.control_dependencies([var_bkup_update_op]): - mask_update_op = mask.assign(tf.cast(tf.abs(var) > mask_thres, tf.float32)) + mask_thres = tf.contrib.distributions.percentile(tf.abs(var_bkup), prune_ratio * 100) + mask_update_op = mask.assign(tf.cast(tf.abs(var_bkup) > mask_thres, tf.float32)) with tf.control_dependencies([mask_update_op]): prune_op = var.assign(var_bkup * mask) From 28485e53b3a1c011f2a81f41add02f2ff3554ecd Mon Sep 17 00:00:00 2001 From: Jiaxiang Wu Date: Mon, 10 Dec 2018 18:51:15 +0800 Subject: [PATCH 019/173] remove the division by 2 in FLOPs computation (#132) --- learners/channel_pruning/model_wrapper.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/learners/channel_pruning/model_wrapper.py b/learners/channel_pruning/model_wrapper.py index 6e1b766..1bd51f9 100644 --- a/learners/channel_pruning/model_wrapper.py +++ b/learners/channel_pruning/model_wrapper.py @@ -267,13 +267,11 @@ def compute_layer_flops(self, op): if opname in self.flops: flops = self.flops[opname] else: - flops = tf_ops.get_stats_for_node_def(self.g, - op.node_def, - 'flops').value - flops = flops / 2. / FLAGS.batch_size - + flops = tf_ops.get_stats_for_node_def(self.g, op.node_def, 'flops').value + flops = flops / FLAGS.batch_size self.flops[opname] = flops - return flops + + return flops def get_Add_if_is_first_after_resblock(self, op): """ check whether the input operation is first layer after sum From ddc8af16297d5cfaca65511871e583b99a92af76 Mon Sep 17 00:00:00 2001 From: Jiaxiang Wu Date: Mon, 10 Dec 2018 18:51:29 +0800 Subject: [PATCH 020/173] skip RL-based optimization when is 'eval' (#131) --- learners/weight_sparsification/learner.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/learners/weight_sparsification/learner.py b/learners/weight_sparsification/learner.py index fb2fe06..75b73a6 100644 --- a/learners/weight_sparsification/learner.py +++ b/learners/weight_sparsification/learner.py @@ -81,20 +81,22 @@ def __init__(self, sm_writer, model_helper): # define the scope for masks self.mask_scope = 'mask' - # compute the optimal pruning ratios - pr_optimizer = PROptimizer(model_helper, self.mpi_comm) - if FLAGS.ws_prune_ratio_prtl == 'optimal': - if self.is_primary_worker('local'): - self.download_model() # pre-trained model is required - self.auto_barrier() - tf.logging.info('model files: ' + ', '.join(os.listdir('./models'))) - self.var_names_n_prune_ratios = pr_optimizer.run() + # compute the optimal pruning ratios (only when the execution mode is 'train') + if FLAGS.exec_mode == 'train': + pr_optimizer = PROptimizer(model_helper, self.mpi_comm) + if FLAGS.ws_prune_ratio_prtl == 'optimal': + if self.is_primary_worker('local'): + self.download_model() # pre-trained model is required + self.auto_barrier() + tf.logging.info('model files: ' + ', '.join(os.listdir('./models'))) + self.var_names_n_prune_ratios = pr_optimizer.run() # class-dependent initialization if FLAGS.enbl_dst: self.helper_dst = DistillationHelper(sm_writer, model_helper, self.mpi_comm) - self.__build_train() - self.__build_eval() + if FLAGS.exec_mode == 'train': + self.__build_train() # only when the execution mode is 'train' + self.__build_eval() # needed whatever the execution mode is def train(self): """Train a model and periodically produce checkpoint files.""" From 1cae313902291eb6ecd2309751df9c9c6a030614 Mon Sep 17 00:00:00 2001 From: Jiaxiang Wu Date: Mon, 10 Dec 2018 19:04:42 +0800 Subject: [PATCH 021/173] update README.md (#138) --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2de0582..8403e1d 100644 --- a/README.md +++ b/README.md @@ -48,12 +48,12 @@ For complete evaluation results, please refer to [here](https://pocketflow.githu We adopt the DDPG algorithm as the RL agent to find the optimal layer-wise pruning ratios, and use group fine-tuning to further improve the compressed model's accuracy: -| Model | Pruning Ratio | Uniform | RL-based | RL-based + Group Fine-tuning | -|:------------:|:-------------:|:-------:|:-------------:|:----------------------------:| -| MobileNet-v1 | 50% | 66.5% | 67.8% (+1.3%) | 67.9% (+1.4%) | -| MobileNet-v1 | 60% | 66.2% | 66.9% (+0.7%) | 67.0% (+0.8%) | -| MobileNet-v1 | 70% | 64.4% | 64.5% (+0.1%) | 64.8% (+0.4%) | -| Mobilenet-v1 | 80% | 61.4% | 61.4% (+0.0%) | 62.2% (+0.8%) | +| Model | FLOPs | Uniform | RL-based | RL-based + Group Fine-tuning | +|:------------:|:-----:|:-------:|:-------------:|:----------------------------:| +| MobileNet-v1 | 50% | 66.5% | 67.8% (+1.3%) | 67.9% (+1.4%) | +| MobileNet-v1 | 40% | 66.2% | 66.9% (+0.7%) | 67.0% (+0.8%) | +| MobileNet-v1 | 30% | 64.4% | 64.5% (+0.1%) | 64.8% (+0.4%) | +| Mobilenet-v1 | 20% | 61.4% | 61.4% (+0.0%) | 62.2% (+0.8%) | ### Weight Sparsification From 558c40266d389e19e86c14fdf824318426806008 Mon Sep 17 00:00:00 2001 From: Jiaxiang Wu Date: Sun, 16 Dec 2018 14:11:39 +0800 Subject: [PATCH 022/173] Bugfix: Endless loop in DCP learner's channel selection process (#147) * disable logging output on non-master nodes * typo fixed * display masks' names and gradients' norm * add assert to avoid duplicately added channels * remove debugging-purpose outputs * typo fixed --- learners/discr_channel_pruning/learner.py | 30 ++++++++++++++--------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/learners/discr_channel_pruning/learner.py b/learners/discr_channel_pruning/learner.py index 6bcf631..12a2155 100644 --- a/learners/discr_channel_pruning/learner.py +++ b/learners/discr_channel_pruning/learner.py @@ -473,6 +473,7 @@ def __choose_discr_chns(self): # pylint: disable=too-many-locals else: summary, __ = self.sess_train.run([self.summary_op, self.block_train_ops[idx_block]]) if self.is_primary_worker('global'): + tf.logging.info('iter #%d: writing TF-summary to file' % idx_iter) self.sm_writer.add_summary(summary, nb_iters_block * idx_block + idx_iter) # select the most discriminative channels for each layer @@ -480,23 +481,29 @@ def __choose_discr_chns(self): # pylint: disable=too-many-locals if self.idxs_layer_to_block[idx_layer] != idx_block: continue - # initialize the mask as all channels are pruned + # initialize the gradient mask mask_shape = self.sess_train.run(tf.shape(self.masks[idx_layer])) - tf.logging.info('layer #{}: mask\'s shape is {}'.format(idx_layer, mask_shape)) + if self.is_primary_worker('global'): + tf.logging.info('layer #{}: mask\'s shape is {}'.format(idx_layer, mask_shape)) nb_chns = mask_shape[2] + idxs_chn_keep = [] grad_norm_mask = np.ones(nb_chns) - mask_vec = np.sum(self.sess_train.run(self.masks[idx_layer]), axis=(0, 1, 3)) - prune_ratio = 1.0 - float(np.count_nonzero(mask_vec)) / mask_vec.size - tf.logging.info('layer #%d: prune_ratio = %.4f' % (idx_layer, prune_ratio)) + + # sequentially add the most important channel to the non-pruned set is_first_entry = True while is_first_entry or prune_ratio > FLAGS.dcp_prune_ratio: - # choose the most important channel and then update the mask + # choose the most important channel grad_norm = self.sess_train.run(self.grad_norms[idx_layer]) - idx_chn_input = np.argmax(grad_norm * grad_norm_mask) - grad_norm_mask[idx_chn_input] = 0.0 - tf.logging.info('adding channel #%d to the non-pruned set' % idx_chn_input) + idx_chn = np.argmax((grad_norm + 1e-8) * grad_norm_mask) # avoid all-zero gradients + assert idx_chn not in idxs_chn_keep, 'channel #%d already in the non-pruned set' % idx_chn + idxs_chn_keep += [idx_chn] + grad_norm_mask[idx_chn] = 0.0 + if self.is_primary_worker('global'): + tf.logging.info('adding channel #%d to the non-pruned set' % idx_chn) + + # update the mask mask_delta = np.zeros(mask_shape) - mask_delta[:, :, idx_chn_input, :] = 1.0 + mask_delta[:, :, idx_chn, :] = 1.0 if is_first_entry: is_first_entry = False self.sess_train.run(self.mask_init_ops[idx_layer]) @@ -511,7 +518,8 @@ def __choose_discr_chns(self): # pylint: disable=too-many-locals # re-compute the pruning ratio mask_vec = np.sum(self.sess_train.run(self.masks[idx_layer]), axis=(0, 1, 3)) prune_ratio = 1.0 - float(np.count_nonzero(mask_vec)) / mask_vec.size - tf.logging.info('layer #%d: prune_ratio = %.4f' % (idx_layer, prune_ratio)) + if self.is_primary_worker('global'): + tf.logging.info('layer #%d: prune_ratio = %.4f' % (idx_layer, prune_ratio)) # compute overall pruning ratios if self.is_primary_worker('global'): From e05d426e4d02938fcd8db5b62dd6156efd01ed2d Mon Sep 17 00:00:00 2001 From: Karen Date: Sat, 15 Dec 2018 22:55:05 -0800 Subject: [PATCH 023/173] add english translation of CONTRIBUTING (#4) * add english translation of CONTRIBUTING * Update and rename CONTRIBUTING-eng.md to CONTRIBUTING_en.md --- CONTRIBUTING_en.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 CONTRIBUTING_en.md diff --git a/CONTRIBUTING_en.md b/CONTRIBUTING_en.md new file mode 100644 index 0000000..3d27599 --- /dev/null +++ b/CONTRIBUTING_en.md @@ -0,0 +1,41 @@ +# Contributing to PocketFlow + +[Tencent Open Source Incentive Program](https://opensource.tencent.com/contribution) encourages all developers' participation and contribution and we are looking forward to you joining us. You are welcomed to report [issues](https://github.com/Tencent/PocketFlow/issues) or submit [pull requests](https://github.com/Tencent/PocketFlow/pulls). Before contributing, please read the following guideline. + +## Issue Management + +We use Github Issues to track public bugs and feature requests. + +### Search for Existing Issues First + +Please search for existing or similar issues before opening a new one, in order to avoid duplicated issues. + +### Creating a New Issue + +When creating a new issue, please provide detailed descriptions, screenshots, and/or short videos to help us locate and reproduce the problem(s). + +### Branch Management + +For the moment, we have only one branch for simplicity: + +1. `master` branch: + 1. This is the stable branch. Highly-stable versions will be tagged with certain version numbers. + 2. Please submit hotfixs or new features as PR to this branch. + +### Pull Requests (PR) + +We welcome everyone to contribute to PocketFlow. Our development team will monitor pull requests and perform related tests and code reviews. If passed, the PR will be accepted and merged to the master branch. + +Before submitting a PR, please confirm the following: + +1. Fork the main repo +2. Keep updated with the main repo +3. Update comments and documentation after code changes +4. Add licence and copyright notes to new files +5. Keep the code style consistent (use `run_pylint.sh`) +6. Extensively test your code +7. Make a pull request to master branch + +## License + +[BSD 3-Clause License](https://github.com/Tencent/PocketFlow/blob/master/LICENSE.TXT) is PocketFlow's open source license. Your contributed code will also be protected by this license. From 355c9056e81c49bcef7c716d7a8b77f7bf509d7b Mon Sep 17 00:00:00 2001 From: Jiaxiang Wu Date: Sun, 16 Dec 2018 15:16:27 +0800 Subject: [PATCH 024/173] Update CONTRIBUTING.md (#148) --- CONTRIBUTING.md | 51 +++++++++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d9c40b5..152cdf1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,28 +1,41 @@ -# Contributing to PocketFlow -[腾讯开源激励计划](https://opensource.tencent.com/contribution) 鼓励开发者的参与和贡献，期待你的加入。我们欢迎[report Issues](https://github.com/Tencent/PocketFlow/issues) 或者 [pull requests](https://github.com/Tencent/PocketFlow/pulls)。 在贡献代码之前请阅读以下指引。 +# 为PocketFlow做出贡献 + +[腾讯开源激励计划](https://opensource.tencent.com/contribution)鼓励所有开发者的参与和贡献，我们期待你的加入。你可以报告 [Issues](https://github.com/Tencent/PocketFlow/issues) 或者提交 [Pull Requests](https://github.com/Tencent/PocketFlow/pulls)。在贡献代码之前，请阅读以下指引。 ## 问题管理 -我们用 Github Issues 去跟踪 public bugs 和 feature requests。 -### 查找已知的issue 优先 -请查找已存在或者相类似的issue，从而保证不存在冗余。 +我们使用 Github Issues 以收集问题和功能需求。 + +### 查找已有的 Issues + +在创建新 Issue 之前，请先搜索是否存在已有的或者类似的 Issue ，以避免重复。 + +### 创建新 Issue + +当创建新 Issue 时，请提供详细的描述、截屏以及/或者短视频来帮助我们定位和复现问题。 + +## 分支管理 + +目前，为简便起见，我们仅有一个分支： + +1. `master` 分支： + 1. 这是稳定分支。高度稳定的版本将会被标注特定的版本号。 + 2. 请向该分支提交包含问题修复或者新功能的 Pull Requests。 -### 新建 Issues -新建issues 时请提供详细的描述、截屏或者短视频来辅助我们定位问题 +## Pull Requests -### Pull Requests +我们欢迎所有人向 PocketFlow 贡献代码。我们的代码团队会监控 Pull Requests, 进行相应的代码测试与检查，通过测试的 PR 将会被合并至 `master` 分支。 -我们欢迎大家贡献代码来使我们的PocketFlow更加强大 -代码团队会监控pull request, 我们会做相应的代码检查和测试，测试通过之后我们就会接纳PR ，但是不会立即合并到master分支。 +在提交 PR 之前，请确认: -在完成一个pr之前请做一下确认: +1. 从主项目中 fork 代码 +2. 与主项目保持同步 +3. 在代码变动后，对应地修改注释与文档 +4. 在新文件中加入协议与版权声明 +5. 确保一致的代码风格（可使用 `run_pylint.sh`） +6. 充分测试你的代码 +7. 向 `master` 分支发起 PR 请求 -1. 从 `master` fork 你自己的分支。 -2. 在修改了代码之后请修改对应的文档和注释。 -3. 在新建的文件中请加入licence 和copy right申明。 -4. 确保一致的代码风格，可运行脚本run_pylint.sh进行一致性检查。 -5. 做充分的测试。 -6. 然后，你可以提交你的代码到 `master` 分支。 +## 协议 -## 代码协议 -[BSD 3-Clause License](https://github.com/Tencent/PocketFlow/blob/master/LICENSE.TXT) 为PocketFlow的开源协议，您贡献的代码也会受此协议保护。 +[BSD 3-Clause License](https://github.com/Tencent/PocketFlow/blob/master/LICENSE.TXT)是PocketFlow的开源协议，你贡献的代码也会受此协议保护。 From 3a1cdf635647df59d04483d8256413584165d94f Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Fri, 21 Dec 2018 14:31:09 +0800 Subject: [PATCH 025/173] collect all extra arguments in run_local.sh --- scripts/run_local.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/run_local.sh b/scripts/run_local.sh index 3354575..ecf020b 100755 --- a/scripts/run_local.sh +++ b/scripts/run_local.sh @@ -6,6 +6,7 @@ nb_gpus=1 # parse arguments passed from the command line py_script="$1" shift +extra_args="" for i in "$@" do case "$i" in @@ -15,11 +16,13 @@ do ;; *) # unknown option + extra_args="${extra_args} ${i}" + shift ;; esac done -extra_args=`python utils/get_path_args.py local ${py_script} path.conf` -extra_args="$@ ${extra_args}" +extra_args_path=`python utils/get_path_args.py local ${py_script} path.conf` +extra_args="${extra_args} ${extra_args_path}" echo "Python script: ${py_script}" echo "# of GPUs: ${nb_gpus}" echo "extra arguments: ${extra_args}" From 2ffe03f00b3469f637ae1595bc4b0e5ea953f0be Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Fri, 21 Dec 2018 14:42:24 +0800 Subject: [PATCH 026/173] collect extra arguments in run_docker.sh & run_seven.sh --- scripts/run_docker.sh | 7 +++++-- scripts/run_seven.sh | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/run_docker.sh b/scripts/run_docker.sh index bb398e8..68d3363 100755 --- a/scripts/run_docker.sh +++ b/scripts/run_docker.sh @@ -14,6 +14,7 @@ nb_gpus=1 # parse arguments passed from the command line py_script="$1" shift +extra_args="" for i in "$@" do case "$i" in @@ -23,11 +24,13 @@ do ;; *) # unknown option + extra_args="${extra_args} ${i}" + shift ;; esac done -extra_args=`python utils/get_path_args.py docker ${py_script} path.conf` -extra_args="$@ ${extra_args}" +extra_args_path=`python utils/get_path_args.py docker ${py_script} path.conf` +extra_args="${extra_args} ${extra_args_path}" echo ${extra_args} > extra_args echo "Python script: ${py_script}" echo "Data directory: ${dir_data}" diff --git a/scripts/run_seven.sh b/scripts/run_seven.sh index 4e54d16..e394807 100755 --- a/scripts/run_seven.sh +++ b/scripts/run_seven.sh @@ -15,6 +15,7 @@ job_name="pocket-flow" # parse arguments passed from the command line py_script="$1" shift +extra_args="" for i in "$@" do case "$i" in @@ -28,11 +29,13 @@ do ;; *) # unknown option + extra_args="${extra_args} ${i}" + shift ;; esac done -extra_args=`python utils/get_path_args.py seven ${py_script} path.conf` -extra_args="$@ ${extra_args}" +extra_args_path=`python utils/get_path_args.py seven ${py_script} path.conf` +extra_args="${extra_args} ${extra_args_path}" echo ${extra_args} > extra_args echo "Python script: ${py_script}" echo "Job name: ${job_name}" From c4ed35709f0a4ccafdd2fde1e95538d9ec4ef33f Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Sat, 29 Dec 2018 10:53:07 +0800 Subject: [PATCH 027/173] add code from https://github.com/HiKapok/SSD.TensorFlow --- utils/external/SSD.TensorFlow/LICENSE | 201 +++++++ utils/external/SSD.TensorFlow/README.md | 138 +++++ .../dataset/convert_tfrecords.py | 394 +++++++++++++ .../SSD.TensorFlow/dataset/dataset_common.py | 238 ++++++++ .../SSD.TensorFlow/dataset/dataset_inspect.py | 35 ++ utils/external/SSD.TensorFlow/demo/demo1.jpg | Bin 0 -> 27539 bytes utils/external/SSD.TensorFlow/demo/demo2.jpg | Bin 0 -> 31989 bytes utils/external/SSD.TensorFlow/demo/demo3.jpg | Bin 0 -> 35044 bytes utils/external/SSD.TensorFlow/eval_ssd.py | 457 +++++++++++++++ utils/external/SSD.TensorFlow/net/ssd_net.py | 255 +++++++++ .../preprocessing/preprocessing_unittest.py | 131 +++++ .../preprocessing/ssd_preprocessing.py | 521 ++++++++++++++++++ .../SSD.TensorFlow/simple_ssd_demo.py | 220 ++++++++ utils/external/SSD.TensorFlow/train_ssd.py | 498 +++++++++++++++++ .../utility/anchor_manipulator.py | 333 +++++++++++ .../utility/anchor_manipulator_unittest.py | 156 ++++++ .../utility/checkpint_inspect.py | 55 ++ .../SSD.TensorFlow/utility/draw_toolbox.py | 73 +++ .../SSD.TensorFlow/utility/scaffolds.py | 86 +++ utils/external/SSD.TensorFlow/voc_eval.py | 269 +++++++++ 20 files changed, 4060 insertions(+) create mode 100644 utils/external/SSD.TensorFlow/LICENSE create mode 100644 utils/external/SSD.TensorFlow/README.md create mode 100644 utils/external/SSD.TensorFlow/dataset/convert_tfrecords.py create mode 100644 utils/external/SSD.TensorFlow/dataset/dataset_common.py create mode 100644 utils/external/SSD.TensorFlow/dataset/dataset_inspect.py create mode 100644 utils/external/SSD.TensorFlow/demo/demo1.jpg create mode 100644 utils/external/SSD.TensorFlow/demo/demo2.jpg create mode 100644 utils/external/SSD.TensorFlow/demo/demo3.jpg create mode 100644 utils/external/SSD.TensorFlow/eval_ssd.py create mode 100644 utils/external/SSD.TensorFlow/net/ssd_net.py create mode 100644 utils/external/SSD.TensorFlow/preprocessing/preprocessing_unittest.py create mode 100644 utils/external/SSD.TensorFlow/preprocessing/ssd_preprocessing.py create mode 100644 utils/external/SSD.TensorFlow/simple_ssd_demo.py create mode 100644 utils/external/SSD.TensorFlow/train_ssd.py create mode 100644 utils/external/SSD.TensorFlow/utility/anchor_manipulator.py create mode 100644 utils/external/SSD.TensorFlow/utility/anchor_manipulator_unittest.py create mode 100644 utils/external/SSD.TensorFlow/utility/checkpint_inspect.py create mode 100644 utils/external/SSD.TensorFlow/utility/draw_toolbox.py create mode 100644 utils/external/SSD.TensorFlow/utility/scaffolds.py create mode 100644 utils/external/SSD.TensorFlow/voc_eval.py diff --git a/utils/external/SSD.TensorFlow/LICENSE b/utils/external/SSD.TensorFlow/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/utils/external/SSD.TensorFlow/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/utils/external/SSD.TensorFlow/README.md b/utils/external/SSD.TensorFlow/README.md new file mode 100644 index 0000000..f2b3a20 --- /dev/null +++ b/utils/external/SSD.TensorFlow/README.md @@ -0,0 +1,138 @@ +# State-of-the-art Single Shot MultiBox Detector in TensorFlow + +This repository contains codes of the reimplementation of [SSD: Single Shot MultiBox Detector](https://arxiv.org/abs/1512.02325) in TensorFlow. If your goal is to reproduce the results in the original paper, please use the official [codes](https://github.com/weiliu89/caffe/tree/ssd). + +There are already some TensorFlow based SSD reimplementation codes on GitHub, the main special features of this repo inlcude: + +- state of the art performance(77.8%mAP) when training from VGG-16 pre-trained model (SSD300-VGG16). +- the model is trained using TensorFlow high level API [tf.estimator](https://www.tensorflow.org/api_docs/python/tf/estimator/Estimator). Although TensorFlow provides many APIs, the Estimator API is highly recommended to yield scalable, high-performance models. +- all codes were writen by pure TensorFlow ops (no numpy operation) to ensure the performance and portability. +- using ssd augmentation pipeline discribed in the original paper. +- PyTorch-like model definition using high-level [tf.layers](https://www.tensorflow.org/api_docs/python/tf/layers) API for better readability ^-^. +- high degree of modularity to ease futher development. +- using replicate\_model\_fn makes it flexible to use one or more GPUs. + +***New Update(77.9%mAP): using absolute bbox coordinates instead of normalized coordinates, checkout [here](https://github.com/HiKapok/SSD.TensorFlow/tree/AbsoluteCoord).*** + +## ## +## Usage +- Download [Pascal VOC Dataset](https://pjreddie.com/projects/pascal-voc-dataset-mirror/) and reorganize the directory as follows: + ``` + VOCROOT/ + |->VOC2007/ + | |->Annotations/ + | |->ImageSets/ + | |->... + |->VOC2012/ + | |->Annotations/ + | |->ImageSets/ + | |->... + |->VOC2007TEST/ + | |->Annotations/ + | |->... + ``` + VOCROOT is your path of the Pascal VOC Dataset. +- Run the following script to generate TFRecords. + ```sh + python dataset/convert_tfrecords.py --dataset_directory=VOCROOT --output_directory=./dataset/tfrecords + ``` +- Download the **pre-trained VGG-16 model (reduced-fc)** from [here](https://drive.google.com/drive/folders/184srhbt8_uvLKeWW_Yo8Mc5wTyc0lJT7) and put them into one sub-directory named 'model' (we support SaverDef.V2 by default, the V1 version is also available for sake of compatibility). +- Run the following script to start training: + + ```sh + python train_ssd.py + ``` +- Run the following script for evaluation and get mAP: + + ```sh + python eval_ssd.py + python voc_eval.py + ``` + Note: you need first modify some directory in voc_eval.py. +- Run the following script for visualization: + ```sh + python simple_ssd_demo.py + ``` + +All the codes was tested under TensorFlow 1.6, Python 3.5, Ubuntu 16.04 with CUDA 8.0. If you want to run training by yourself, one decent GPU will be highly recommended. The whole training process for VOC07+12 dataset took ~120k steps in total, and each step (32 samples per-batch) took ~1s on my little workstation with single GTX1080-Ti GPU Card. If you need run training without enough GPU memory you can try half of the current batch size(e.g. 16), try to lower the learning rate and run more steps, watching the TensorBoard until convergency. BTW, the codes here had also been tested under TensorFlow 1.4 with CUDA 8.0, but some modifications to the codes are needed to enable replicate model training, take following steps if you need: + +- copy all the codes of [this file](https://github.com/tensorflow/tensorflow/blob/v1.6.0/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py) to your local file named 'tf\_replicate\_model\_fn.py' +- add one more line [here](https://github.com/HiKapok/SSD.TensorFlow/blob/899e08dad48669ca0c444284977e3d7ffa1da5fe/train_ssd.py#L25) to import module 'tf\_replicate\_model\_fn' +- change 'tf.contrib.estimator' in [here](https://github.com/HiKapok/SSD.TensorFlow/blob/899e08dad48669ca0c444284977e3d7ffa1da5fe/train_ssd.py#L383) and [here](https://github.com/HiKapok/SSD.TensorFlow/blob/899e08dad48669ca0c444284977e3d7ffa1da5fe/train_ssd.py#L422) to 'tf\_replicate\_model\_fn' +- now the training process should run perfectly +- before you run 'eval_ssd.py', you should also remove [this line](https://github.com/HiKapok/SSD.TensorFlow/blob/e8296848b9f6eb585da5945d6b3ae099029ef4bf/eval_ssd.py#L369) because of the interface compatibility + + +***This repo is just created recently, any contribution will be welcomed.*** + +## Results (VOC07 Metric) + +This implementation(SSD300-VGG16) yield **mAP 77.8%** on PASCAL VOC 2007 test dataset(the original performance described in the paper is 77.2%mAP), the details are as follows: + +| sofa | bird | pottedplant | bus | diningtable | cow | bottle | horse | aeroplane | motorbike +|:-------|:-----:|:-------:|:-------:|:-------:|:-------:|:-------:|:-------:|:-------:|:-------:| +| 78.9 | 76.2 | 53.5 | 85.2 | 75.5 | 85.0 | 48.6 | 86.7 | 82.2 | 83.4 | +| **sheep** | **train** | **boat** | **bicycle** | **chair** | **cat** | **tvmonitor** | **person** | **car** | **dog** | +| 82.4 | 87.6 | 72.7 | 83.0 | 61.3 | 88.2 | 74.5 | 79.6 | 85.3 | 86.4 | + +You can download the trained model(VOC07+12 Train) from [GoogleDrive](https://drive.google.com/open?id=1yeYcfcOURcZ4DaElEn9C2xY1NymGzG5W) for further research. + +For Chinese friends, you can also download both the trained model and pre-trained vgg16 weights from [BaiduYun Drive](https://pan.baidu.com/s/1kRhZd4p-N46JFpVkMgU3fg), access code: **tg64**. + +Here is the training logs and some detection results: + +![](logs/loss.JPG "loss") +![](logs/celoss.JPG "celoss") +![](logs/locloss.JPG "locloss") +![](demo/demo1.jpg "demo1") +![](demo/demo2.jpg "demo2") +![](demo/demo3.jpg "demo3") + +## *Too Busy* TODO + +- Adapting for CoCo Dataset +- Update version SSD-512 +- Transfer to other backbone networks + +## Known Issues + +- Got 'TypeError: Expected binary or unicode string, got None' while training + - Why: There maybe some inconsistent between different TensorFlow version. + - How: If you got this error, try change the default value of checkpoint_path to './model/vgg16.ckpt' in [train_ssd.py](https://github.com/HiKapok/SSD.TensorFlow/blob/86e3fa600d8d07122e9366ae664dea8c3c87c622/train_ssd.py#L107). For more information [issue6](https://github.com/HiKapok/SSD.TensorFlow/issues/6) and [issue9](https://github.com/HiKapok/SSD.TensorFlow/issues/9). +- Nan loss during training + - Why: This is caused by the default learning rate which is a little higher for some TensorFlow version. + - How: I don't know the details about the different behavior between different versions. There are two workarounds: + - Adding warm-up: change some codes [here](https://github.com/HiKapok/SSD.TensorFlow/blob/d9cf250df81c8af29985c03d76636b2b8b19f089/train_ssd.py#L99) to the following snippet: + + ```python + tf.app.flags.DEFINE_string( + 'decay_boundaries', '2000, 80000, 100000', + 'Learning rate decay boundaries by global_step (comma-separated list).') + tf.app.flags.DEFINE_string( + 'lr_decay_factors', '0.1, 1, 0.1, 0.01', + 'The values of learning_rate decay factor for each segment between boundaries (comma-separated list).') + ``` + - Lower the learning rate and run more steps until convergency. +- Why this re-implementation perform better than the reported performance + - I don't know + +## Citation + +Use this bibtex to cite this repository: +``` +@misc{kapok_ssd_2018, + title={Single Shot MultiBox Detector in TensorFlow}, + author={Changan Wang}, + year={2018}, + publisher={Github}, + journal={GitHub repository}, + howpublished={\url{https://github.com/HiKapok/SSD.TensorFlow}}, +} +``` + +## Discussion + +Welcome to join in QQ Group(758790869) for more discussion + +## ## +Apache License, Version 2.0 diff --git a/utils/external/SSD.TensorFlow/dataset/convert_tfrecords.py b/utils/external/SSD.TensorFlow/dataset/convert_tfrecords.py new file mode 100644 index 0000000..4ce3ad3 --- /dev/null +++ b/utils/external/SSD.TensorFlow/dataset/convert_tfrecords.py @@ -0,0 +1,394 @@ +# Copyright 2018 Changan Wang + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from datetime import datetime +import os +import random +import sys +import threading +import xml.etree.ElementTree as xml_tree + +import numpy as np +import six +import tensorflow as tf + +import dataset_common + +'''How to organize your dataset folder: + VOCROOT/ + |->VOC2007/ + | |->Annotations/ + | |->ImageSets/ + | |->... + |->VOC2012/ + | |->Annotations/ + | |->ImageSets/ + | |->... + |->VOC2007TEST/ + | |->Annotations/ + | |->... +''' +tf.app.flags.DEFINE_string('dataset_directory', '/media/rs/7A0EE8880EE83EAF/Detections/PASCAL/VOC', + 'All datas directory') +tf.app.flags.DEFINE_string('train_splits', 'VOC2007, VOC2012', + 'Comma-separated list of the training data sub-directory') +tf.app.flags.DEFINE_string('validation_splits', 'VOC2007TEST', + 'Comma-separated list of the validation data sub-directory') +tf.app.flags.DEFINE_string('output_directory', '/media/rs/7A0EE8880EE83EAF/Detections/SSD/dataset/tfrecords', + 'Output data directory') +tf.app.flags.DEFINE_integer('train_shards', 16, + 'Number of shards in training TFRecord files.') +tf.app.flags.DEFINE_integer('validation_shards', 16, + 'Number of shards in validation TFRecord files.') +tf.app.flags.DEFINE_integer('num_threads', 8, + 'Number of threads to preprocess the images.') +RANDOM_SEED = 180428 + +FLAGS = tf.app.flags.FLAGS + +def _int64_feature(value): + """Wrapper for inserting int64 features into Example proto.""" + if not isinstance(value, list): + value = [value] + return tf.train.Feature(int64_list=tf.train.Int64List(value=value)) + + +def _float_feature(value): + """Wrapper for inserting float features into Example proto.""" + if not isinstance(value, list): + value = [value] + return tf.train.Feature(float_list=tf.train.FloatList(value=value)) + +def _bytes_list_feature(value): + """Wrapper for inserting a list of bytes features into Example proto. + """ + if not isinstance(value, list): + value = [value] + return tf.train.Feature(bytes_list=tf.train.BytesList(value=value)) + +def _bytes_feature(value): + """Wrapper for inserting bytes features into Example proto.""" + if isinstance(value, six.string_types): + value = six.binary_type(value, encoding='utf-8') + return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) + +def _convert_to_example(filename, image_name, image_buffer, bboxes, labels, labels_text, + difficult, truncated, height, width): + """Build an Example proto for an example. + + Args: + filename: string, path to an image file, e.g., '/path/to/example.JPG' + image_buffer: string, JPEG encoding of RGB image + bboxes: List of bounding boxes for each image + labels: List of labels for bounding box + labels_text: List of labels' name for bounding box + difficult: List of ints indicate the difficulty of that bounding box + truncated: List of ints indicate the truncation of that bounding box + height: integer, image height in pixels + width: integer, image width in pixels + Returns: + Example proto + """ + ymin = [] + xmin = [] + ymax = [] + xmax = [] + for b in bboxes: + assert len(b) == 4 + # pylint: disable=expression-not-assigned + [l.append(point) for l, point in zip([ymin, xmin, ymax, xmax], b)] + # pylint: enable=expression-not-assigned + channels = 3 + image_format = 'JPEG' + + example = tf.train.Example(features=tf.train.Features(feature={ + 'image/height': _int64_feature(height), + 'image/width': _int64_feature(width), + 'image/channels': _int64_feature(channels), + 'image/shape': _int64_feature([height, width, channels]), + 'image/object/bbox/xmin': _float_feature(xmin), + 'image/object/bbox/xmax': _float_feature(xmax), + 'image/object/bbox/ymin': _float_feature(ymin), + 'image/object/bbox/ymax': _float_feature(ymax), + 'image/object/bbox/label': _int64_feature(labels), + 'image/object/bbox/label_text': _bytes_list_feature(labels_text), + 'image/object/bbox/difficult': _int64_feature(difficult), + 'image/object/bbox/truncated': _int64_feature(truncated), + 'image/format': _bytes_feature(image_format), + 'image/filename': _bytes_feature(image_name.encode('utf8')), + 'image/encoded': _bytes_feature(image_buffer)})) + return example + + +class ImageCoder(object): + """Helper class that provides TensorFlow image coding utilities.""" + + def __init__(self): + # Create a single Session to run all image coding calls. + self._sess = tf.Session() + + # Initializes function that converts PNG to JPEG data. + self._png_data = tf.placeholder(dtype=tf.string) + image = tf.image.decode_png(self._png_data, channels=3) + self._png_to_jpeg = tf.image.encode_jpeg(image, format='rgb', quality=100) + + # Initializes function that converts CMYK JPEG data to RGB JPEG data. + self._cmyk_data = tf.placeholder(dtype=tf.string) + image = tf.image.decode_jpeg(self._cmyk_data, channels=0) + self._cmyk_to_rgb = tf.image.encode_jpeg(image, format='rgb', quality=100) + + # Initializes function that decodes RGB JPEG data. + self._decode_jpeg_data = tf.placeholder(dtype=tf.string) + self._decode_jpeg = tf.image.decode_jpeg(self._decode_jpeg_data, channels=3) + + def png_to_jpeg(self, image_data): + return self._sess.run(self._png_to_jpeg, + feed_dict={self._png_data: image_data}) + + def cmyk_to_rgb(self, image_data): + return self._sess.run(self._cmyk_to_rgb, + feed_dict={self._cmyk_data: image_data}) + + def decode_jpeg(self, image_data): + image = self._sess.run(self._decode_jpeg, + feed_dict={self._decode_jpeg_data: image_data}) + assert len(image.shape) == 3 + assert image.shape[2] == 3 + return image + + +def _process_image(filename, coder): + """Process a single image file. + + Args: + filename: string, path to an image file e.g., '/path/to/example.JPG'. + coder: instance of ImageCoder to provide TensorFlow image coding utils. + Returns: + image_buffer: string, JPEG encoding of RGB image. + height: integer, image height in pixels. + width: integer, image width in pixels. + """ + # Read the image file. + with tf.gfile.FastGFile(filename, 'rb') as f: + image_data = f.read() + + # Decode the RGB JPEG. + image = coder.decode_jpeg(image_data) + + # Check that image converted to RGB + assert len(image.shape) == 3 + height = image.shape[0] + width = image.shape[1] + assert image.shape[2] == 3 + + return image_data, height, width + +def _find_image_bounding_boxes(directory, cur_record): + """Find the bounding boxes for a given image file. + + Args: + directory: string; the path of all datas. + cur_record: list of strings; the first of which is the sub-directory of cur_record, the second is the image filename. + Returns: + bboxes: List of bounding boxes for each image. + labels: List of labels for bounding box. + labels_text: List of labels' name for bounding box. + difficult: List of ints indicate the difficulty of that bounding box. + truncated: List of ints indicate the truncation of that bounding box. + """ + anna_file = os.path.join(directory, cur_record[0], 'Annotations', cur_record[1].replace('jpg', 'xml')) + + tree = xml_tree.parse(anna_file) + root = tree.getroot() + + # Image shape. + size = root.find('size') + shape = [int(size.find('height').text), + int(size.find('width').text), + int(size.find('depth').text)] + # Find annotations. + bboxes = [] + labels = [] + labels_text = [] + difficult = [] + truncated = [] + for obj in root.findall('object'): + label = obj.find('name').text + labels.append(int(dataset_common.VOC_LABELS[label][0])) + labels_text.append(label.encode('ascii')) + + isdifficult = obj.find('difficult') + if isdifficult is not None: + difficult.append(int(isdifficult.text)) + else: + difficult.append(0) + + istruncated = obj.find('truncated') + if istruncated is not None: + truncated.append(int(istruncated.text)) + else: + truncated.append(0) + + bbox = obj.find('bndbox') + bboxes.append((float(bbox.find('ymin').text) / shape[0], + float(bbox.find('xmin').text) / shape[1], + float(bbox.find('ymax').text) / shape[0], + float(bbox.find('xmax').text) / shape[1] + )) + return bboxes, labels, labels_text, difficult, truncated + +def _process_image_files_batch(coder, thread_index, ranges, name, directory, all_records, num_shards): + """Processes and saves list of images as TFRecord in 1 thread. + + Args: + coder: instance of ImageCoder to provide TensorFlow image coding utils. + thread_index: integer, unique batch to run index is within [0, len(ranges)). + ranges: list of pairs of integers specifying ranges of each batches to + analyze in parallel. + name: string, unique identifier specifying the data set + directory: string; the path of all datas + all_records: list of string tuples; the first of each tuple is the sub-directory of the record, the second is the image filename. + num_shards: integer number of shards for this data set. + """ + # Each thread produces N shards where N = int(num_shards / num_threads). + # For instance, if num_shards = 128, and the num_threads = 2, then the first + # thread would produce shards [0, 64). + num_threads = len(ranges) + assert not num_shards % num_threads + num_shards_per_batch = int(num_shards / num_threads) + + shard_ranges = np.linspace(ranges[thread_index][0], + ranges[thread_index][1], + num_shards_per_batch + 1).astype(int) + num_files_in_thread = ranges[thread_index][1] - ranges[thread_index][0] + + counter = 0 + for s in range(num_shards_per_batch): + # Generate a sharded version of the file name, e.g. 'train-00002-of-00010' + shard = thread_index * num_shards_per_batch + s + output_filename = '%s-%.5d-of-%.5d' % (name, shard, num_shards) + output_file = os.path.join(FLAGS.output_directory, output_filename) + writer = tf.python_io.TFRecordWriter(output_file) + + shard_counter = 0 + files_in_shard = np.arange(shard_ranges[s], shard_ranges[s + 1], dtype=int) + for i in files_in_shard: + cur_record = all_records[i] + filename = os.path.join(directory, cur_record[0], 'JPEGImages', cur_record[1]) + + bboxes, labels, labels_text, difficult, truncated = _find_image_bounding_boxes(directory, cur_record) + image_buffer, height, width = _process_image(filename, coder) + + example = _convert_to_example(filename, cur_record[1], image_buffer, bboxes, labels, labels_text, + difficult, truncated, height, width) + writer.write(example.SerializeToString()) + shard_counter += 1 + counter += 1 + + if not counter % 1000: + print('%s [thread %d]: Processed %d of %d images in thread batch.' % + (datetime.now(), thread_index, counter, num_files_in_thread)) + sys.stdout.flush() + + writer.close() + print('%s [thread %d]: Wrote %d images to %s' % + (datetime.now(), thread_index, shard_counter, output_file)) + sys.stdout.flush() + shard_counter = 0 + print('%s [thread %d]: Wrote %d images to %d shards.' % + (datetime.now(), thread_index, counter, num_files_in_thread)) + sys.stdout.flush() + +def _process_image_files(name, directory, all_records, num_shards): + """Process and save list of images as TFRecord of Example protos. + + Args: + name: string, unique identifier specifying the data set + directory: string; the path of all datas + all_records: list of string tuples; the first of each tuple is the sub-directory of the record, the second is the image filename. + num_shards: integer number of shards for this data set. + """ + # Break all images into batches with a [ranges[i][0], ranges[i][1]]. + spacing = np.linspace(0, len(all_records), FLAGS.num_threads + 1).astype(np.int) + ranges = [] + threads = [] + for i in range(len(spacing) - 1): + ranges.append([spacing[i], spacing[i + 1]]) + + # Launch a thread for each batch. + print('Launching %d threads for spacings: %s' % (FLAGS.num_threads, ranges)) + sys.stdout.flush() + + # Create a mechanism for monitoring when all threads are finished. + coord = tf.train.Coordinator() + + # Create a generic TensorFlow-based utility for converting all image codings. + coder = ImageCoder() + + threads = [] + for thread_index in range(len(ranges)): + args = (coder, thread_index, ranges, name, directory, all_records, num_shards) + t = threading.Thread(target=_process_image_files_batch, args=args) + t.start() + threads.append(t) + + # Wait for all the threads to terminate. + coord.join(threads) + print('%s: Finished writing all %d images in data set.' % + (datetime.now(), len(all_records))) + sys.stdout.flush() + +def _process_dataset(name, directory, all_splits, num_shards): + """Process a complete data set and save it as a TFRecord. + + Args: + name: string, unique identifier specifying the data set. + directory: string, root path to the data set. + all_splits: list of strings, sub-path to the data set. + num_shards: integer number of shards for this data set. + """ + all_records = [] + for split in all_splits: + jpeg_file_path = os.path.join(directory, split, 'JPEGImages') + images = tf.gfile.ListDirectory(jpeg_file_path) + jpegs = [im_name for im_name in images if im_name.strip()[-3:]=='jpg'] + all_records.extend(list(zip([split] * len(jpegs), jpegs))) + + shuffled_index = list(range(len(all_records))) + random.seed(RANDOM_SEED) + random.shuffle(shuffled_index) + all_records = [all_records[i] for i in shuffled_index] + _process_image_files(name, directory, all_records, num_shards) + +def parse_comma_list(args): + return [s.strip() for s in args.split(',')] + +def main(unused_argv): + assert not FLAGS.train_shards % FLAGS.num_threads, ( + 'Please make the FLAGS.num_threads commensurate with FLAGS.train_shards') + assert not FLAGS.validation_shards % FLAGS.num_threads, ( + 'Please make the FLAGS.num_threads commensurate with ' + 'FLAGS.validation_shards') + print('Saving results to %s' % FLAGS.output_directory) + + # Run it! + _process_dataset('val', FLAGS.dataset_directory, parse_comma_list(FLAGS.validation_splits), FLAGS.validation_shards) + _process_dataset('train', FLAGS.dataset_directory, parse_comma_list(FLAGS.train_splits), FLAGS.train_shards) + +if __name__ == '__main__': + tf.app.run() diff --git a/utils/external/SSD.TensorFlow/dataset/dataset_common.py b/utils/external/SSD.TensorFlow/dataset/dataset_common.py new file mode 100644 index 0000000..046dcca --- /dev/null +++ b/utils/external/SSD.TensorFlow/dataset/dataset_common.py @@ -0,0 +1,238 @@ +# Copyright 2018 Changan Wang + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +slim = tf.contrib.slim + +VOC_LABELS = { + 'none': (0, 'Background'), + 'aeroplane': (1, 'Vehicle'), + 'bicycle': (2, 'Vehicle'), + 'bird': (3, 'Animal'), + 'boat': (4, 'Vehicle'), + 'bottle': (5, 'Indoor'), + 'bus': (6, 'Vehicle'), + 'car': (7, 'Vehicle'), + 'cat': (8, 'Animal'), + 'chair': (9, 'Indoor'), + 'cow': (10, 'Animal'), + 'diningtable': (11, 'Indoor'), + 'dog': (12, 'Animal'), + 'horse': (13, 'Animal'), + 'motorbike': (14, 'Vehicle'), + 'person': (15, 'Person'), + 'pottedplant': (16, 'Indoor'), + 'sheep': (17, 'Animal'), + 'sofa': (18, 'Indoor'), + 'train': (19, 'Vehicle'), + 'tvmonitor': (20, 'Indoor'), +} + +COCO_LABELS = { + "bench": (14, 'outdoor') , + "skateboard": (37, 'sports') , + "toothbrush": (80, 'indoor') , + "person": (1, 'person') , + "donut": (55, 'food') , + "none": (0, 'background') , + "refrigerator": (73, 'appliance') , + "horse": (18, 'animal') , + "elephant": (21, 'animal') , + "book": (74, 'indoor') , + "car": (3, 'vehicle') , + "keyboard": (67, 'electronic') , + "cow": (20, 'animal') , + "microwave": (69, 'appliance') , + "traffic light": (10, 'outdoor') , + "tie": (28, 'accessory') , + "dining table": (61, 'furniture') , + "toaster": (71, 'appliance') , + "baseball glove": (36, 'sports') , + "giraffe": (24, 'animal') , + "cake": (56, 'food') , + "handbag": (27, 'accessory') , + "scissors": (77, 'indoor') , + "bowl": (46, 'kitchen') , + "couch": (58, 'furniture') , + "chair": (57, 'furniture') , + "boat": (9, 'vehicle') , + "hair drier": (79, 'indoor') , + "airplane": (5, 'vehicle') , + "pizza": (54, 'food') , + "backpack": (25, 'accessory') , + "kite": (34, 'sports') , + "sheep": (19, 'animal') , + "umbrella": (26, 'accessory') , + "stop sign": (12, 'outdoor') , + "truck": (8, 'vehicle') , + "skis": (31, 'sports') , + "sandwich": (49, 'food') , + "broccoli": (51, 'food') , + "wine glass": (41, 'kitchen') , + "surfboard": (38, 'sports') , + "sports ball": (33, 'sports') , + "cell phone": (68, 'electronic') , + "dog": (17, 'animal') , + "bed": (60, 'furniture') , + "toilet": (62, 'furniture') , + "fire hydrant": (11, 'outdoor') , + "oven": (70, 'appliance') , + "zebra": (23, 'animal') , + "tv": (63, 'electronic') , + "potted plant": (59, 'furniture') , + "parking meter": (13, 'outdoor') , + "spoon": (45, 'kitchen') , + "bus": (6, 'vehicle') , + "laptop": (64, 'electronic') , + "cup": (42, 'kitchen') , + "bird": (15, 'animal') , + "sink": (72, 'appliance') , + "remote": (66, 'electronic') , + "bicycle": (2, 'vehicle') , + "tennis racket": (39, 'sports') , + "baseball bat": (35, 'sports') , + "cat": (16, 'animal') , + "fork": (43, 'kitchen') , + "suitcase": (29, 'accessory') , + "snowboard": (32, 'sports') , + "clock": (75, 'indoor') , + "apple": (48, 'food') , + "mouse": (65, 'electronic') , + "bottle": (40, 'kitchen') , + "frisbee": (30, 'sports') , + "carrot": (52, 'food') , + "bear": (22, 'animal') , + "hot dog": (53, 'food') , + "teddy bear": (78, 'indoor') , + "knife": (44, 'kitchen') , + "train": (7, 'vehicle') , + "vase": (76, 'indoor') , + "banana": (47, 'food') , + "motorcycle": (4, 'vehicle') , + "orange": (50, 'food') + } + +# use dataset_inspect.py to get these summary +data_splits_num = { + 'train': 22136, + 'val': 4952, +} + +def slim_get_batch(num_classes, batch_size, split_name, file_pattern, num_readers, num_preprocessing_threads, image_preprocessing_fn, anchor_encoder, num_epochs=None, is_training=True): + """Gets a dataset tuple with instructions for reading Pascal VOC dataset. + + Args: + num_classes: total class numbers in dataset. + batch_size: the size of each batch. + split_name: 'train' of 'val'. + file_pattern: The file pattern to use when matching the dataset sources (full path). + num_readers: the max number of reader used for reading tfrecords. + num_preprocessing_threads: the max number of threads used to run preprocessing function. + image_preprocessing_fn: the function used to dataset augumentation. + anchor_encoder: the function used to encoder all anchors. + num_epochs: total epoches for iterate this dataset. + is_training: whether we are in traing phase. + + Returns: + A batch of [image, shape, loc_targets, cls_targets, match_scores]. + """ + if split_name not in data_splits_num: + raise ValueError('split name %s was not recognized.' % split_name) + + # Features in Pascal VOC TFRecords. + keys_to_features = { + 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), + 'image/format': tf.FixedLenFeature((), tf.string, default_value='jpeg'), + 'image/filename': tf.FixedLenFeature((), tf.string, default_value=''), + 'image/height': tf.FixedLenFeature([1], tf.int64), + 'image/width': tf.FixedLenFeature([1], tf.int64), + 'image/channels': tf.FixedLenFeature([1], tf.int64), + 'image/shape': tf.FixedLenFeature([3], tf.int64), + 'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/label': tf.VarLenFeature(dtype=tf.int64), + 'image/object/bbox/difficult': tf.VarLenFeature(dtype=tf.int64), + 'image/object/bbox/truncated': tf.VarLenFeature(dtype=tf.int64), + } + items_to_handlers = { + 'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'), + 'filename': slim.tfexample_decoder.Tensor('image/filename'), + 'shape': slim.tfexample_decoder.Tensor('image/shape'), + 'object/bbox': slim.tfexample_decoder.BoundingBox( + ['ymin', 'xmin', 'ymax', 'xmax'], 'image/object/bbox/'), + 'object/label': slim.tfexample_decoder.Tensor('image/object/bbox/label'), + 'object/difficult': slim.tfexample_decoder.Tensor('image/object/bbox/difficult'), + 'object/truncated': slim.tfexample_decoder.Tensor('image/object/bbox/truncated'), + } + decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) + + labels_to_names = {} + for name, pair in VOC_LABELS.items(): + labels_to_names[pair[0]] = name + + dataset = slim.dataset.Dataset( + data_sources=file_pattern, + reader=tf.TFRecordReader, + decoder=decoder, + num_samples=data_splits_num[split_name], + items_to_descriptions=None, + num_classes=num_classes, + labels_to_names=labels_to_names) + + with tf.name_scope('dataset_data_provider'): + provider = slim.dataset_data_provider.DatasetDataProvider( + dataset, + num_readers=num_readers, + common_queue_capacity=32 * batch_size, + common_queue_min=8 * batch_size, + shuffle=is_training, + num_epochs=num_epochs) + + [org_image, filename, shape, glabels_raw, gbboxes_raw, isdifficult] = provider.get(['image', 'filename', 'shape', + 'object/label', + 'object/bbox', + 'object/difficult']) + + if is_training: + # if all is difficult, then keep the first one + isdifficult_mask =tf.cond(tf.count_nonzero(isdifficult, dtype=tf.int32) < tf.shape(isdifficult)[0], + lambda : isdifficult < tf.ones_like(isdifficult), + lambda : tf.one_hot(0, tf.shape(isdifficult)[0], on_value=True, off_value=False, dtype=tf.bool)) + + glabels_raw = tf.boolean_mask(glabels_raw, isdifficult_mask) + gbboxes_raw = tf.boolean_mask(gbboxes_raw, isdifficult_mask) + + # Pre-processing image, labels and bboxes. + + if is_training: + image, glabels, gbboxes = image_preprocessing_fn(org_image, glabels_raw, gbboxes_raw) + else: + image = image_preprocessing_fn(org_image, glabels_raw, gbboxes_raw) + glabels, gbboxes = glabels_raw, gbboxes_raw + + gt_targets, gt_labels, gt_scores = anchor_encoder(glabels, gbboxes) + + return tf.train.batch([image, filename, shape, gt_targets, gt_labels, gt_scores], + dynamic_pad=False, + batch_size=batch_size, + allow_smaller_final_batch=(not is_training), + num_threads=num_preprocessing_threads, + capacity=64 * batch_size) diff --git a/utils/external/SSD.TensorFlow/dataset/dataset_inspect.py b/utils/external/SSD.TensorFlow/dataset/dataset_inspect.py new file mode 100644 index 0000000..a94e6a6 --- /dev/null +++ b/utils/external/SSD.TensorFlow/dataset/dataset_inspect.py @@ -0,0 +1,35 @@ +# Copyright 2018 Changan Wang + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +import tensorflow as tf + +def count_split_examples(split_path, file_prefix='.tfrecord'): + # Count the total number of examples in all of these shard + num_samples = 0 + tfrecords_to_count = tf.gfile.Glob(os.path.join(split_path, file_prefix)) + opts = tf.python_io.TFRecordOptions(tf.python_io.TFRecordCompressionType.ZLIB) + for tfrecord_file in tfrecords_to_count: + for record in tf.python_io.tf_record_iterator(tfrecord_file):#, options = opts): + num_samples += 1 + return num_samples + +if __name__ == '__main__': + print('train:', count_split_examples('/media/rs/7A0EE8880EE83EAF/Detections/SSD/dataset/tfrecords', 'train-?????-of-?????')) + print('val:', count_split_examples('/media/rs/7A0EE8880EE83EAF/Detections/SSD/dataset/tfrecords', 'val-?????-of-?????')) diff --git a/utils/external/SSD.TensorFlow/demo/demo1.jpg b/utils/external/SSD.TensorFlow/demo/demo1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e0ca8c5edf9d87e70894f369c16c76c5e22e2752 GIT binary patch literal 27539 zcmbTdbyQqU@HRLEcMBFQz#zebORzxD5Zqk`x4|X2L-4^pxI4q(1P|`+9!QXZ07J0l z{q48kp0j`LZq2>t%ssbzrk>N?Rn<>bKQBLT0|-9LD#!wmkdOeTFCW138sHP)6%x{a z*Z&e^ROJ5z4Fv@m6&(#7{l6On6B`o)0}BHk9Sau=3mfM}&@u7wadGhe`~KgL{MY@z zUN09820F%nE&jjB^A7+a24EP89T|xp@QM%#nGorD06+@>Afdi!`=7%9OORe6qoAU_ zsDy?6(f~^EqCPV6s~2TaQBYo5`@g&gpb(-G(Q!+n5v!S@)4ROp2~H}&V32C)B~hQb z0P>onZl+?8JjLfXUqT-U$vhs?`#-?Uy3#_%Ry{~^@aA^4F$ms0c z{KDeW^2+MY?%w{v;nA<-6ZqxT-|L%O#NEIDa3KMZ|4&%|8`=K@7vT%8S1)6Lg8m;a zq*q=qH!>j#Djhc(k)#^BnF}#JPcX)7sicC2UQ7mF^$QYn*BLBQAm0un{6A>_gY5qu zu#o>>$o@C5{~OmT00$ZAW$=&*0TO`6){a&S$I6M@gsl}%qN0aW>)p{;p?(RBugcL|4f9mYZ^PN1!G zT6pEv1j2f?h}iM=fm?qdM6Scb!3jKBly)Xd>(j&g#7~=nc~DW;))c##IFo4zYxCu&1@sdVNzU%q4#Q74wk&?tfZCD5FVKCO5?V zE1T8Ng@*02I6-zQ;NR^WRC!H~AP=#v)#3IXhMX;^@fI~Z$_4^Vq;Y#TQF1wTw@%$ zbC%YRYu8`)0Cn1m3zV4O&grMn_HO^2&z6(bk@`0W_L==LLR2H9Uy9rgQsF_qx!$tK z{nc3E8DI?CKVM{HGC)8!$o@3Mc<=cIOQZyMK(MHirz2Q=sr2WS)6z%-%oDi1awHEv zf4*}GBBxV#-A`@{?sa~3p`u0GpUaG$?fDGonGJ*+_g_}fBP1piTf&EJw2>*-SM z6q_i_iYcpUA5${u={*8V+dj2f&O#qcmJDQ1JrO-M%3`Kalh;O+dd+w22WfIDLQ>V5 z3b&f#T6#g03Qsh?gv!wQ)7rl_X4u6B>vF87G7fL#jP7VVfsm)|95%NHC0gfvtY*>t zzXA_T$#DKM!b>MER`IDYX3TBpt(VpS#=a&suS61s zpqdX~%qaxkft-h*dYGR9yHy8Z1Uqxo^)n!dwz2cWQm_?d=-R`Zp(ppWaQ>S?xB17n z?FtO8))4U@p-nlYb+aeCO&HAYEomQcp8=OeJLNmhZg$)i?;nsK%Kp(hf>WqSEyXZ| zcT*JU<0-P&ooacd5jv%V#rksYOG`QmDE54p8y`kQgFkwK*~8W+AfA9W2^M64kxBT6 zci=3_f@c6YIS{Uy#!L&|k$VOp2R@XG#iKt3C_cY<1i)F;Nt0kz;NE=fWzuc-9HJ`g z84#{gNPB1L@eBxl>8K?)XoBk=_r9;aWFFfyopjMnktH4RdVPa@ zh3x@c_Si2w@7($!l>uWRbxm=*x7sZ8^|hVbY@DM0u|((9$>5L^$7X+;)z7h)1;$fzyS+d&YwweJkbsLn&>p-=3_?LfZ*I9-+ zeD_;FS-RBfe;#kLqv@MsbdVWqv&95`=hgpiAhhnPJu4ngs$1BL;-Q6h+oL=a9pKxF zJhPjM!+Hy~#~say4!NY3me?dxixdh9GT3Vk-BXf6x`DV!a|!)#poq1lF6D_l7cHj< z^81`6B?@<*6g{eq;~68|o&X z*@wYOY$?b;6H$L4zQQk2&k`NE{O45#Dq&BoMPF=ImsO8pxpUl11u?=kI=vl02Tb`? z*rC@ouhZ4pM~~57K~zd~v0)zK7-_s3VcOcmhT5)a5e*BO-DC-GBI*+Ep1_YtYb!~4Fwc|SZeQsRBhm$a&miwd&@9ElbF zv3Fx_($0D<&w$h>6PmzYdp^ugjeBD6_RR&qv24BO^mtz8^ux4y5GoRRued>wD}Wgo zYvG^n;)NUDoKA+*(EExH!b?zD`5mGl>Wu2A^bA0aMpRgDC#)#H~3I?;|^!a;Bk<@Wtw)Bb!ufV{3Rpw#q!GV=0E8QvqSwfs^*{hxWv=7JL5Cx;z64vuXuLQ7ye^J;m`C(I15j z1An#DviF^1t3LzA-jC#FP)JnIJp+gyW&-#uDe9HA;@hMx}lA^cY@gi)rGX7HKAE_#L>%;HnEc1^XXkx$4DU$s* zylpTnR{vSE(0crZz3zeGxBN5UWbN@J*z(c@toCh&Xjlcq>$VcIy8Whc4kC!7L3ulN z6_9F!vr_Xao(or!{R^5LJ(T&2AbrIM@ynd~+8mq7@!C+Z=W`uTx)WlU#HG-Dfo7u7 z@ihLrlWDIar_lQJ$MCJ~2+hIqX0-yo_K)GKVq~!0pEw!EFP4O1P9&7LWviu^$JK4S z&9MuA(Q0L~%*PTs7Z{lEzsjH1tIRg5bES%h5O)b3W?$i-YXq?=tF>z9j3<~6o&CI1 z5vtgAK>k}wz$($!`p+u$8Bmw0R%Ts68)hQ-b(peeZcXBLbn&n9>x+ePgcX?cUf0Cn zPy(^L2py`U4zErpCaCZ0Sl_FE@)1Q^UkXop2DH70t2_e~lg_OEndcWv{Ef<1xG8dofC-$ng${knygz zVNk;`k>vqImL3=_X$wr_Qym{=+ulO3EcM5kym*EX2gKGe4J+*OArQ_skOf?Fp?#tv!_os;()!NEnJlDIIS)&o68mp16`$JO#b;@N~870l~6lOM58 z5Z?jO;%I;ox&$?|=4-fnM;)0DV81zW0*jj)kh{mJHkp=- zTqsCpcSoQvnp~vxO&h$#OzdY78q7Tty~7lJ|88%K;lX--lvRznqpSYC8YiNqy>+B> z50=0zi3kT^JA^W76SFp!T`!2G?PE2b?2&H7w6a$8FjJTL?y)t(x}4oo_fy%rth_K~ zngDv!NtKk$F*8x|*< z>-jxP`$!6{$jYI;?~=F}zLWt;BuKy|CX$kp)Y`ZAph`_{Ugi9h8SYp38ArVG^dGq) z1Ef!%7pC1(GjGa=`RWBT+Sc;?Svlm?<7Zd0Y2Ck@dtfIt0px4mt|yB2 z3EUPs&f>DdTINB2&tVdANL9O?ulGbhj{*2|$s^hcU8eA+)-QReSs=6DO z+q7Hc@g=%p@?Ow!BwS^W3s$WzhAF)$(ws4+1Z}pk8dk8+l)#x+v2V`GFggc!OvrIe zG5^-CCYGfiYk6!D82lo&I)N`zdJpN#-d7#x*8zN!PI&idp(dx|TUdNX z`q-4ejl};AZBqWS;);$LJbItm@6uepmWb`(L+w_N>{7FC*hP3cC(s*$A)%c58m89E zGj#Ut&l-td{rHAw@cJGzGBaRlteg0;8?9{b=L1gt>WII+wJW&xPmuSY%}rV!$@@Mc zIZi1vLz1<&(Q_9vZaCR4<7P*Jk$=;@`mh?2T-oZeoATxQsy9B9JiLJmhpu^Ny zita8;wDY>!UfOsfUmLNvOPvr@y$87wiAHG{;fb0jY~f!&65bCN?%PlLiGpnGMs}*+ zUN#UThDOf*7XdD5i^q=1l)W#)QSFd>bB0<%W7!u}yuWsIwoRMpXM1n&-SI>ul2Fgs z_23C3y);m$Vq@%uF#0ehkH}yRYHbRGILC?Tw|=AjgKyzJ$S<%503SpaG*pw7n!4wg zwB~Y>-zU45uB5HM`ctd6`*X7((7RmzvfJ?BTlYZgam7E&J(E@5{y^lj;)DTL6G<0; z*DdQuD?Jwf=))y>%Mzp~UP-1Rf{%B({ zsP9Bk;CyD$)5G_IruZLIi8lD-*I6KD@Z!o4KFyDIhugJf%r`O{siW(oSr(qd4vi{Q z@g#?4L}wgVAHt3PKv@2X+&IHgk3!G?im6}j)bSaSartG1Zuu9fCdz)n8vVh?RauO+ zvOG(M^+Wa~ko&JhG+7=~%MwFFcT;1UKy!pJLp%jz9vH?kc3EaFy#i61X#@Y&oqj(# zDc%@!lNlcvJgs~DH7xC)V@<6(jZEs3e~hK$ELOs z1-w|zUTyyOsJ}Lep>uy<41uJF&D?~mg>0T4+0;JLPZQo8K}ORdFJjo#Ghn-PtHmiD zKDHf^U|%v~$qv~(hMmk1(nmusrB15D4$O%RDhw}cF-|A~bQZFUvq`NOxKLfb?I$2zpoX3NC5=#%RdMKhnvyO`!rjW_azL}CFMqavHX4L~Z zEh$2siZSUkv?Y?^*z(8Zp7cX^)7{6$r*_(a6A9Ah9+HvWxc5q(4X{i7mD2QaBF2cZ zR#LVqJu+5?WZSp0%8M-wW@e$k61{^x4006}=OB`9)^2Q_NkzjDI}0|Cr%w8&leK%Og=_9WbzHfUJCi|3_}{LZ-!^wm7KQvBy7UpL+|v5*lgeErfIyh#KpKI8&|Eoql?1L z2~*`w`=GG%KkiiE$1|XF?%E_veCh)FFZb2(;h~64YJ?B^mR{dB5w8)h90Z_oDecj=D?Xek6j3*;{heRRmjEl@q#@jhY& z+_=OQ0WZ;oNVYcx9>4#&eot*^dmHD}xT@6n9 zaP#@u->VGyN!0MiNP1rt-wJTH5w>?4o8A|sgj|e%C%~|z7*ghPdCCUM5B?XwQo9B%!v7@xBK@+ui$78B@k`nW0uAT{E* z-as!9PpYcZsXuZ&5UuX|{567TN$27q{x|LyOk>7*r6PkbQ4I>j-&HzmDDD#2xfGaz zhYO>Ja$6C%?o;{aw1;*!=g}HZ{%k0ApC&+8{#UCtz2zZaa~3B#C}d?%jL-Dly#fVy zj`g?=gPsAte=8QR(9Z)>V%kb!l!#-=%;f3YN;P9=aVIfWHP2#A=(3k(orv{ixuxY4 z#e;3Wo#e~*rfy-}!JySWuV2p-6cUcOu#(7iXo3~ae zkYk-zd1z*awnEH0KZ4cg#f{`PpMyV@aHNzaqkENlC#&pVm5ivUw&6k0>NG6!Gd4E0 zHl$A_Szj74yi4nUhf`bt1eK%3TYX8$c?LL4t$p;PXzqwqy?7kB`OvRY_B~!YUU9jl z^kw@zEyAL$)~1_VBQU*b@CR{z6ZH7qJwNWNgD+n$uWG_QkFP(`s@t18{8vJ`&j7zVT42RQ;8wYFp;oGXU+YQFeVg6 z>O(43E3S5Nbc)OzQ}i@B4`q7>VE=xTd$iI$N2~z!? z&fKcNUyTiQbIo#QtF%Msz#eRFOsST(xEtHwZ1O;MN#j*acl<1lV+s+8ovvY&4^1Wl zPeWS-q7u8$fEZpwdmWdR;}0qSay&=t6}j4yXhgzcZ_jw;aMHh={b+@-On>HqN8Y?A z@uPza&CpRf8#BH8o|ZbB$1%DNY$P@14iA zN8}dh|GsJYZM`$rC2Ml(cB4nS^tq!03YoR9FlT#;HQrQEz)>U(-eT341{=cV!kK3I zVtjqc_ZFM6Md+B)T{!v-PT<8}kQeq~;}dIBDAAg{U-6jVRm9~ddFa855s3^pc%pa) z4ARieu>V0V7N>-qYYO7yB%PRVip9Cv$$BYa3d%|o%25CGA(ED5ozw9pm~Hqi&W5lr zdr(AJUi5A0@){f2<>M92ox1NQk9qZk?Aq>4qPUhmCbJ%y|A}{qt2_ws>8uau({K@+ z7t}B@wDr;$y&<~OzR_6!)Oae-zPs!bl@MtZ=lm_e?5)Z72YZt9s0Y-G^qufOI>uyh z%Y>k9xPSV$8kIGuBa45b>2ySlxPN<+O%N}TlT$i8wAw}Cpr`qC{fXehaWcI|A7S!o z0s8piyni({wWP^VvUJOqn6nK0YV?js4hPI`N6bCSsQI|UZZI}d` zQX=LJQ^U@f3ciw0(Yme95@~ed1iSiY`m{ry##Q|hwgZr9qLXX3q(~I!rzLq0e}g_6 z;1WHYG#6~?wH?L(9xE)*4ERCw$=&v7GG8#GLHbL0KkrXS#Fs__H$N>nwkkqw`;HHv zz;?pR+PZpqQx+n%c(qntFV9J+D}qt`g?Ch-p6Q)MzKD?~-buExT&`R8#d#PAOtQWv z=To?Pe8VHwZs)clk?lrlhzc>#MMigQtUd`x@?d}2Ed;4p%sikLAJYiWT=ov~_~O`c&+aFAk-HUEdreETdb~uYpO^n` zff?GBHLAf@wWV0GUaW`-@v-V@!Dbj24jn@cVdd-QzfcKx6qU2f3V>Rh0spP)HtP?L z!$0}5ZuS)?yFWQ^#v`LgM~APWQprMVC&3|q zYX(PwY2~i4YWZlXWc1T5#?$*+MCFv>(t{+G+c4w2;L@99LM$2fcM@sVQ7J|RY4!?u z1b@VRR@!ttbP>b58@T3a_Z#;Z=wnws0JPUThq(( z<_6w#{e3HnMr@f(6AH`Qt}W)r%+s>j$+CoS{p3-8A$$lq1x|57ai~x?FUAFRB#2Gx zZND$_<9@#6i&|3#8d#_R2ZPG?*&*aJTXwM(K$gJw3Gk_JPMq%8H3WM4>-y`WjWJ)^)GU=>1mYwrFLhDoQ#Fu}Cd+^V}YXkon7vj3_e^;D|Pu&l~&{Xq4#Pq2itB1Ua zvdAo(m_N$9BiUh|;+q{8lJJ}4lBp_#;o@fi=(y!}k(BgLY?Jb3Gxm{uc@bXAuG9{S z70W)IT1;^Ey@a<*H$%X(3C^jRR#UnXoI18;rgO)l9oBdSwn}`TmikeMw zWg*oW*uNE*bsHh-V!FDS-$FT94}--EN`E_>WnkdpYW+yV_;#DPg@+gkFy@h!Flb*x z`Gh5Wfp@Mj^Ea1Fr4>#w&5*1TUE@Tvy6ejZvg83)H~ME(C&_WG9m#R)Y|^6~#i!qM zsL0(>ZZF+Si|HK3XB#3?cM=HvY~W$8c3)E`UWQMvZa+q)HTzz`z<1@((*(igEgL2Y z^C(!IFjXP(LtPyRWoZR@L@Ura`%@UvGNT(Xbps>r8=8} zicDYofGO_$2$it&cE=}qP=&V4`H5LS#?=Ud_a=X>&;QRwk7EpvlmSg=-ZGM8P zL%s=caF*^@`}?X(sUaC(pPqAx@%y^ozzdBygPmtUyt@R3pKf4NYQmp4xvO^#t44od zo6MO`n#yx+sFb5-6dtCkxs`7baeK7LerkxI)zSnPmWB_8UaF%j`0f^M+_emGvDH8N zjJe>&1zG%9@^nwY)^G2lV99nmZ#eVu@;$6^CreoQxJ&C?t_I&2bUG5LP-K=!6IRQk zC==V@Rs0G>?o|1XKpw8LWTzT@w-h*fKsVLoUL=En*=r)quinvs9OH zl=hG2(Yn?8o)|3Km_;$!fUTC zG7!}xkkn=q*uL1@ocuyQ)38x6H+?Fbas;y*B&v`elVnL-vnr;r>mGLGQ~Yl6$+PNw z280Ew8bsBZVi%1$-FRW>QmCA7ShO*mhqQq+OjFml=%3b(ceV)pjGFCFdG+V>sQF(h znAY=GvL{b)>&|5A6SiwJk)jM8EV%*5H^W@7YyuADDp}=M?mH61uU*%)w9fx&BT0zh zJOcjMduU!3(tY|l*r?+qF3HZ3eE2FL`LUwY#*gx%NuyP75j)0Pd=As=a@NY^&dM;Z zDkqlo2M*Fv!9yV;9p0hQo~&ym{t}+ab7$68Gj!oPHIC;4(@guQO&nZvfbp0}3lWT? zNAH$1snZAXBhvGPyJzD=_t0RAq92d(Qj6^f0v=|^ihfa*x%#tQJ^)*zU+H`9o+Fq0;^6nX2`nOsC#!*AH z^H)RMzksHO_zTUcq~4X-{X6gJI`=gw;}=8QcN(26??-G~KGdUrN`yJK3%+ z@|0-g2$j`Y)8Ip!VmNjUnN71wbN6t^#?#op;mGEEJnyDHSi0JTKU#i8zl&|f>M-MA z^P)>a6Z=vFQB%{~ORJ=i$c&FWny}Q8U6U2@;k~WCDXQV^p4WT{<+hi{tGQiInP#Qy zHnJuHt4F7%G8n;o5i;wI3IsMwe)i5dt~xDbGme_BiJpASJmI)z zSVLhHgBnAzEXDq%p=g2OT}#4(dEW4p1mzRs#6{!S4KJd^2ZVzS-j3lQ9`IEA{5Eow zsAC8sO{nJAPqNqK^tcn;&+-Fgasw$2ZV9Ol0GGIxXrTy@Uz#8 zC2OIw~l;}tm4m)DviMO`&74F zvk2zbjyIuab7?Lhl5bfO zm^%;V8A0(JaKvOBm}z+paopBX>ayxAa%1wp#1IId@viTE*70YL{R@0>+T`!oubI z>g4E>qJE6lEpAG{nZvKa%b&CzITt4v*Hqtpb;tsN-ATY4VfAaWdSOY`NuR$|wCS2f z$v$p;Ra+3rk+a=4dUdsPdrDkxDm^RrKH0$^Rs>meXmv zJOf~5LlX>pOEG^pK1t7e;My&2Vw^dh@l(#w-Aj)_cjCFCDoY%%?whHC0OpeX?t2#W zHhz*dhQ~a1`T{XK*&70r$o@VKi`u)F_R$u4n^M)h!g3krT+R&{45O!7)h?DrzqW{- zmgJymn~O(M&0a0{rkz(zARxH$`g~7rsE`8K9r5%d}yUeqErmtGKfEptQu!zC#P@ z_Ee%-mA2K5b~0N-aRQi+h5XoQBqL|W0{gv29D{axq61K~?7iK6P4LzW@=o6BWtnh# z>4h|~0)4PTKDP?R6h`P=o%$LvX12cd}LUWSA zJAzT{eE6%_eU26S{rs`UZ1tTT&XQN=5ui2-0l|#A9}ES^w&*T=yIp&jil6=@L~UZh z-}lA2skTNA>Li`ekfndVI34(~9iozrRTWs!q+&yn(V=zMDfc$n)qjwgKlkwcEoQHl zbvdLYt1!E>-OXz<<#Gc;TI(FU!T8A2;RNA-%f)T3LWt0)Uf&2laX0a`kJ8AcvA`%8veMhDg5k&ihheVWS&l>DZ7N*r77>^yT-Ds$IG##=D~E$1YLc z7ZYQ08V;AI964xQ6!&C2GqjN6p7(Jx|K3Y~Lk?ay_@<;gznU}u8Gy-D zXW?{4FbqDz{$1)___#T*I+2J?^oJ?gt_sj?ebL|hHF6Nwe3psWP#+O6cYApFD}&=L zjm2CF1C4$mUz1}Dry+09__DDdRX!tMPilbt5(UXA@E&%TFXhKQ=e|X0`wUoZzfS+g z=R2V%la|V(ugDmCQs2zOSJ@~%<;L3a-RW)I8)ocI8oR%QaWcUtX`sDKYI}mY&b#Z??epatE-x~?n1WaOr2(`U0xJ*!! zUhkcufqtmI{;FP?-0-t7cdAjG{lSMuV5Zt}c33(7#Jg*0HAbk1&IqS|D66Cg9sC93_a@ysEy#5a~IL~=^QgwOkSaPH*8Q6}&D z%CM6H_Cn=<{H0cb>EE5-R3I+0JpQiwny8rB+Y|aI_Kxj?rI8PN8*+J~oSzcH=BLUcYbVkt?@Tz+ssU(~M;} za+#8%Nbx)oxyqv(<-1E8O3CMIlR>=jyr_iWNYa?HVH}lO)oM1mG*Kam>pkq|3;lnLll zd>k$mHU|Q?Sk?Heon`*5L&~q)hJ@}Sa9v!yFDW>)RkI)6XQpUYJbxDGN$%~BN@QTa z!lb&P?+(yiY1#9o8E~#|oc{Zs0<%Bpj)-kP1wwM(X;H=Vs2JNV zh56}uczrh%8@N>+cGVC>{qk+LIts&URoYB}F6wQax-*wwj}D^1wlrVlkL}lnH9Q(U zfr-%8G>Hu%Q9SX$^hDPDU_S-YBtC(3WAl*)5(^pAS@DBS6`?&hE+JslJk^=qrB8|- zYh1Rnr!uL8N{{)3@$T8~kfXq^Bh%0&qk$C9;-&02q)e&?LsIoJ%m;Y@rT0s2Rx3^D z)d-sc>*MWP+~HqBb>mwr5_jgd;xYgUrl1-7LS&;+^s9 zcr?~Vr7lvbGIl^;W0hs_m?^65s#y1rK>n3lCUcBP9d6{`jFF|We%be<^?@T0RRfEwa*^z3R!POax<^ac@~f$tRv3Du!m^(WE0O4uav&(CSqS5$ zy`w08TnX()h_5zmzKqzAeR0pNu}HdzdD`hgNqlc_vTSDq?U1B#_Au*CsUz)*dNjvx zfYf3mAw>WU9S}(EcCVO9QZl9S*dSrO)`R11rxKNxwVCVUWCeK#YJXDO$w@5GH6Os9f5M zFf+V?K)z-@5zISw3h#D^j6hCAL=3{O#&L+i44%Nt{a$RQ^;2^^N_6ys$Cs2u2#1u=u z)HUs&&5(?+eY!2qurXizxC|6%j8Uw<;jt|}7;Vq5YuD+Iv+wKMR4AF*m+kQ3$J}iv zQ;(~7t1Z(SPogCPQYV483S&d|FKJ2N|D`24`DY*4XS-KL87H6^!`SKjJ5J(@>IC|# zbh>Nba%HDIa^reV1kf z@*$_9%rA5;MA*a4Ua7ilncW;nqNX?c1DRXDIMv0}KtdQ7B2abSBksdgZy_6H8(3*{>dgrm z$5CbsDMZ`+KKjlj)@ca6KZ^r8W)nxHeJN>Qx(Qe-*`l!C2oCXPYknAsji8)j!C$4l z(0L5l8)%pN@u;D0*}4b?M{+AK=|odSxZPAGH}e2A^97yzS?**N2~hns$YL4W;SSHaFI6d=YfEJ|8;Xq=2=rE1N=PAW@O0M&^H-` zKC*1-nG_|AWfY-{a}Y2cK+$j^IKW}>88%?vugn6BLcwU2Y)W2C?<;{4oyV~aht;uX$|1m@>779C5Rfh_;7VEG4d*iBy52%`z;eBxGgj~ zfN6)FCQn4zb@h~j-D#BG%ZB}q6jq-YNTk->nmng=*M6$v+x?}DxM@+jsQqW-^v0@H z69U(#Z&(|D35oh?yTX^((qp?ftAfYkn~Lhv598Od1J;zMF0HJp$_wE04r>;K>qTZU z<)#)C=t;qsIj6$_51|3UHy(?y>Y=nAn=Hp|AJ)kx!=KKT1m988tbQ09v{DF_d4Jon zgy+l}*8W`Fl@vG1%Gr5EI}|l8?CFV4j5yCt)$iz55s`VimRWx~ z`6IcVR4@U5$;>}SciP*-Klcykv1fmpdr;xqLsXA0x_Nt&jL7LH?A=T49j)Dd zL}viJQud**7i9)}fr{*jXj`zsY>uy5PCTU zr3xCJb_u==g26!>YI!nQwP#gg&42oLG2a|Q!709k!*DzGl62i-AX>ivjs~PUkcthB z*-@b!DMhqwKG8PyO|Okm`O>ovP!UO@&~n8Md;0F7m=_^njB$gDxwyG?$H(yU-Q}UE1Swxt7#rmXjv(cm&Q2UBa)mhS90Hjfm2_JgqIK*Wq$e z%%Pguxji;0;Y8dlv^FBq+Ypx@+p*e;-e*^O-$;{HFBdl%Lx6qt(g0Iev8!vQGB`D~ zy8uKzA7Iv+B1f6b`BKu74oGyyswGdvJ3C^o(VVN}B|e2%8XZwmB2!lq+k;fB*B|+4 zThPBfh}7xz+k4b1f$UuqzZ-m<#*lI}TA|K%qdL0Ns*9Fr(Qeq4K=q090Cro>L;A}i zv|71G$1B3mD$5vRgF_>FN$!@m9Z9LJwOiwDI0c46q;;$q&hdP&^6ieSmJTD}wM4ht z(FHgGijhM+Zt=XZZn586H;LV_`>9OVVcD!h=fkBDAKGf;7k)S-)jO~+5U;LaL84`E zq4{pj!p#8ON0mwmL+urjAEEWFqmTeT1U%O9pGO&U`X2LF4K>?t{u?EBUWQo?Z4@;a z3ciiKk6oH-045>Kr__DNqM0t$Ru!l+g`eO~yi=+!vF@dEhjyMk<9_`(Yz^`ZMfu7S z9SK97onyo!%JAje##K%?YX1i8xC-uq#dLEkxFcx5i+ixUZ~s$G%rP)X<-&|!&uG=zJY zLhQ$|M1uZZ#;7Z#{cGNaT~ZkBJN~mS4}na3{LFS;10PnvR?0qTiJ=E%lpo56ah9<9 zp3@WpvIpLCa?Y-3n2W}etOv~!WT-*L2Ux3!kS5zIT&`k8)}Sqz;)V}3!#GtlUg%+DW)XP02{nq?>(qnPjlp8 z_BlbIS@G(47PO|*iSC;>meHDG$B{ZRH-@-Whi)w1tELZ{J?L|i2^t0yPUMGBShKX8 zPc0oGPJ|zM2HtFjI!j%@fQea*JSpu1i*8}pE0Eh{Pg7hZ5 zR-VFIi^59(ucO8dF=9TrG;-T(usnwRzPR$l_Q=f|caAAU9F)kz^|6;q=d^0Wra;d#u6SowD0@6L=;IgZi|E7$ zD(A68XMqQ0|3NKhp)#qg@Kl)7J-FaTm=SylLX%#?zcxV z0d(^~M>0CA{oeUyk7z56-FJ2zm=@j8F)p z-!gw+<^A#dFYR2(S%G7fuA(*ArA!7``i^38dYF)9j!0IA0SitP7-J6t#!kFs*2q5< ztsBZSUhdewz1&R{KRDtJobe4=4%e)4`oa^Ei!8sl`e-a-(4n;3&+W>$HfYSHdEy;sO}vbUCENp)!uHM zx-%JKlZ=h-uk|SEuYqaGt}yok0p9)FHyz+&M1S8C_eEau;)oAwH?+`5ZwiQ-i_&ZH zaqUaeC^AlL!9Iwey0+vo+!_o_<5dFF{^Zz?koW6JP5qoh67oM@#kcRv9qgzGUsw5ib?WFCBLx zNBfx$+h1P!7pfaHTZ0m>0=1MUd7bWMKgCxw?Le_n8~uJ44r`a`O_>ptGl zLaoR=UPB!6-O;K#olI$3&C&RB_T(8ryg_cQtDmo%GV3{KAmWhp(*(h*da+L0s6-_J z=bPK>X~`qIdc}}fGvn4SOys?8y0h!xG^EG(C~$K)dU#^<&?b82S~hZLWr@?PNpkne zdE>RCTHjK;H|-8u!(LA$=^OvrY;0sZV40;ui4z25+heL*0{>Dqy+%OPxoU121@ z*Hzb>-{W@`=^P(_Px?p2wG{}ZPwmdPcO}t+1d}tve3suHDw)++O!uAJC3UP@xC>2Dpi}B}Q|WP*+B{7;%vping^e8~ zM>ycb4Trp(KeQKf%U(UqS=G+A1~K&W4@Z@5q6z5{sZJ^!G_)oHe(cjMj760nDalP; zf2dDMv2*KsDXrzeqa4(p*fEJBw3}fkQ?akC*;jNSNJGyQ`0^88Pq&aqf>TBp+`B#B zL)0nf;sU2MgxLK3DYHA`y?C=YBj<$O*bx?m+Yn@Nr}h!aSxjMT$q$nzdzY=|7JGkl zGg1p3P#!RF-3z!H84|Lmj3(FTo>BOjh4d0XKVDS~%8i0y%d*;Vw~C_0`q!ye9%FFz};!Uf zu?>uFRH#sKirbyg*5WoOC-O05`f^&vN{s0E?_z7uf}m|nLR=}` zP0-xu(bE*MF{B{z19IPcmVb2SMRv2-LcWx1F%ukx1`$v#*FsfcORBTK!pNdG6FTVw zf{Fm`$Ng!}E7#XnhE#q(s3Kkt)QPbziSk|q%0#zRx5qY&fcKMuBdJpGDx@$Z*V-R9 z3{Y`!7!4(CR**lgzj|i)bw};~k$V*RMPEAuF|OW@eqby=B~<&KR}L&%g{G%F6nDid zNQEXOCHqz2t0Ut?g`b^V5e)JlbC`;MtLY<~C8ib?+_VZ#3r^(hM0M@eS3xM?IX zq{J+nke@dj0nf`BUa}R)Cjmc*=E3m3Ei@ID>iXh1*w8sf)JR%r*!k;rlfQTKa)`m3+~qnqRW5!CV7bfTBB zv;P1bzs%?_wA~gxDQ#|B#%sPI0h$|jk}^V&Uujax4mwCU+PN7fu6TD@(fk|X?-N_c zsfE1Nme|I12$~X`XC=0@h!uAR#|P6qVyVfj+(o6rs==B>QTLC2wQ9h_KZHC*_VtNw zW|rPdY>fQ4lXvo8$X1mqLy>b>^6vSYf_nMi@i|zum=!NFG&_d8WhUZF=+U;S1VvtU zfQym{&I#|FoUK#9j2lam9ZOBKpGH`|#PJweNnT&faNs)Qk-*^g12ng~jl5Fd3801A z9qhlkQXFl^r?4M@qfWcB*RP1+skQd z2rSN2O|-0}s&3{ajv+DgB#J@~M8qlwj1EqJdCMG< zG_*|@Sck#WO&bP=ODPuBmJ$YV#|lRtVvO=~dh^C>gS@-eAk?Bpyu3?t%B;rbV8bK6 zbAyptMx#o}2K%2=f59OxbzcbhH}-_S(zQ#C7gO-}fUjh=zqGqsTZ>7gxzn(G?aELo zwBYbRc`rF5KAYk%+Gpa_nm({BJ}P*!ON+@Py0b|9VRtNulCo@7+V~v~0l>o^#G3PO z*uDhPei;7LQbh`BGC#uGg7aG@-?Qz#x4{wup|V-=oxKhS99Nh8HT|e=z7b1#Z*dl> z;!AtQhfiD6aSJ;~wZh?0oE*r1<)a*?PBUH{urzTvn(~}EwM$gLmM^OFK8B7ubTIT` z891vwul&t)en-+U{?h*d88sO7O-f&go<6eCP2?7qvd`lQqh0B6u)YgigNNGeGs6tv zoY%5^C-{;3K=`A@vnyOwD>PH`;QMn36kH>jsDoeb&=WQ&UfZ0PB*uf0)v$#SF!lB{t7(W z!s$1fE!L-FHKvoP*<4zPmIb)FxrO1klx%4e_YNbOUC8ptG52=r|ZB73IIQgkK*# z8Q`ryU(~)Qd|XfMT|!G=v+15DyH>P@Ex_DL+G{KkM&5O@C?#E_S#Y=j*8$<5_$g(b zuYfIM@g=&pwy`a>tnu4Je%G#$K%3>;GGXO0ODk@SDR~G7t~pN?f5A>Jn?;r#2SvV` zc{FM4Vwsi=G-=mzFduDnHwYYhT+B z;$^mxcXg=zR`{K5aIDbd{u8}gI7F)N9yz%xppbp~b_-QBAKIJZ>%o1hU3_i$wGr_Ja=I&=fg{VZtPn4 zlKva5QuF&lXdUEVFf*8uIK--*oc@h5KUk!X!@w@hU_#JUI?xEr@5nJh6oOfT_ zay*wec+?mP46fU^ZBewA$&NMU@Cx{^#{U3_x`w%}e#bU)JTQ$@;bxzxUKD~gibPw@ zMV>r-zF1{KpoZF{t`GrVUVKo#`!B)YgC;loV%q5Xnt8eZ09`9;h9aPm&cqY1ARdDY zfGfel;AvC(wc=C9X9Q}6HcK7-rNtgFS};g8?$eaFT=64tce4*V*<(6uiTd^OXq zHrrT59mS5#LdF%E2mr&b}nMz7{qXHmh-SYip^3tdcoK zCI=5Ff17B?h;9HmSmRPT0=|^f{{Us%T{q%Ih1RL3S$M}!Tj_4KJ1H(_f*nQQWTobo zHdX}4RzQW66CeN-az; zZA?OuNac`WczFOMWUg>X=sH-|z7FczObsMNZ!FxG3Ca064)db$tJq} zcS!!)@sEUWqqwq?-reLf6u5=Fz;Mr&-az@4##Fiaask6*hmXR#wz4g*6=u6@SrwI5 z*5ct<1CSJ$!~_f|$C#=+o^e|D{u6CmUK)g!{vx`XIVO=IxVN{rR})0&{$#B?G;B+= z7D(hd^#D}zzGd$9`5RKh{9LbfvFmzQz>}|7wDG;1TVwlVkj;N??`WwImD!89Pm>N? z8R}-#yX8TXRSth!UXne_#Ncx?o4#seZyXF~ZRYS2!Jahoybsin~hpJlmR$V7g zwD7K;i*9uLtAv+O)NcmpOl%B~a~wgLT<~BjMlr`;Z(O&BLDQP!UhyWO4V&$VDo!-m z<~ho!`!Go*W1qUr**G}J7}M&g&N+2nUwv75oX>%DE6sn#+E<4BEqNTiCDhQtHTAsS zbUJ)5o$t9+g2yZusRfAykOBH9_Dk^XufhKS+Uw!JgY@{>q0@YCrNH+dX7!#}wOe(a zn}Bl9I*?9s3HziFHU9wMo?o(Vsib^uy7AS=gY_>7X>)1D*4j-}RzmNLQ*}rH{iE#OGgBOamU-nK~`cu!Y zHj|z?Cz|D=)$VmGFEZltTZWF=EucwUGTX$GPQobt5Hqs@*XKN(X0^X)4+(1We$)Q| z6RvIS;MAbF@h+;!AeYVx{{U^6Y+=+n2c`-3t}sL5EiUdz?=&4zOrsG44ℜ$0r%d z+<5I=n0!PdN;>pkkg-)M`$=om=zJ}H(i#3$9#)wXBQD&9$>)`CyPw0Qb;A)E=@xVQdNF#*7jEyG+rk#>vtvmkmOH-pDtXEE;-R%~ zJIrM&!_00+)~jllH&AJC!z33?<+`bL+lU0OS|TcO6#dE@z4(|f&-nRqz3sNZdm zjjuo8pXSTq{CbpzX1sV@<2l{^(T;28zlk~&+BUf*uvTbeNYvqQGr94a{du0!*6T_! zHpYi&7|8A0{{XFD1%A=L2&{C^AI%ItSV^sg$0RmfNiRJH4$?UMJ*&&i@am;IBiO;z zbluO9wOu|quid!&q!3gqbSI`wRn}~yvTJ#u+Z?e*L2dyj1A+c~S4XYrvPY?ANbt=g z3}dLqDvqI}O(vY*yOYjFQWqS#Jbos=dJ1uxwa-GlV&k$oyFCs~XH=PHo93E8<(Tr@ zdVU-l(38MxZD}BmX8F)#aqYN&g-t!A_P0^UQMX`#Td)}8>B;=-s)E@r;+y4JnrmSk zeavmTGmr+@)SEM>bvEDE<9x2VqHZ!XS9`&(-}3#jtg0~$uWy~g|_Dm-Er57^gj}Q z$=BK(7qdraB-?Wdk!{#=LY(k$I^&A<@N<;l*~e2yq2{`Ojbtb7%jUF(`pz-u%E&h~VI!3}9dVb=f7li3+9$zne@F0# zh;8lcZW8NOiI5Q&@F<2>iFm34~4hhIM953r}&!tN%*gzPvN;`(B!vU$WYA-?oYI4KfN4- z<_8(b1duD(r1(+$A^bwPp8M=SA1uU^e2pFUjT*bB%kttbGq=7!3ivM4_VY-*x0=>@ z?Ps!PFwCK|7VK`uIraM2zIX*A@bAHED0MrJKKeWV0JTV}>$xF2j^4cV2mSZw1s%&nd{itrCky7s3XmxK7yf-1h zFA_0>j5blXVD3;egIE)MFa4CXZ6X;K+xEUTuZv!w2yNS?Tro0 z1({p>N5beAsW|<}%D1*_4_3VWjdev+I0T#wab0T#{{T-V{{VqM^Ep;It?9?^A-Z4Kd^hhjpW8F1AL+gt{{Z6^ z9)Dp`wOdL3v%c;&{lCLQKHus%uN(0vg)aO(sL5-4Z#AsS*x4%WC+Y3(D=J$DXnAfA zBRy+XgR}l5{{ZkO{$cg!Z7Ij@{{V(RN55!)h(EHI!w3DE^^X!*{?QF*AHwT+yf*fe zX$>8wkuA;qA@9eRyxpToEAF?#t{^ID;tX& z8_yBES=8a0;z>mDPi~^w;bl2iQV7c~2mhsl*D-qH|RiBdu113%qW^Z-GBs9V@*Qo^2em(oa$2teCaOj4st#BuT$SpuF+ zhA;u=9QK|2X?!5Qn&Kacp9h;ux|Txb=TPvR6G*SqiQ@?%BeMn1wRPSa@ptUg@g7B8 z7sT3!hHe{cD#PN7yLMmqVZ6+JeK^k*^a~Q6=MT4oha*-n{6T%>Ld&@$nq#p}dZ_!v zdloH^_eFB|`kXd%d2#fCFt|T6NZm2--vhmSrPu6_;hz=gkzaU!;wOtFykZq*zqGN` z?A><(jkfW`;|q{6Fyv&8$CB26VNGJ;)m!2=lMl_yL*Y40j@>tW*CY?ttJ_8Fc1Imf zF7i+AIKPRS{{V!c_;sb}dK_zgt4M9HFZBntk~ty&07fe%p=VVt4APPU?nPJ=VI|*p zjbHe#!rxRDqHnco6JcVownK4sF*yJvX_jJr2Q~GVhQHvRUJ<&PTf}}K@wbb%^3b)< z_HC1&c;FGgbJd%#PW9{`4t@^!E8xX|xA6Y}hkQ5TrOr>;EYU7yC#y5<-tF!|G`!6J z0JR-BxYA!Xe6R3}_89n)@mA6s&1y?8hWZ@XB1_Ge+Qg_wNa0rUU}84^0Ig?WMhtur zUru}z{f|5Y`#)X{Pf607Q1JvxcX*3jo9%j$U*g!rD1~|H62!a=gN}P=@hYHah9yv- zpE~7_PhL)cGQUdZb?+U=DOCntLPa67!osK5IQ>)-J!$hJ&8>*5SL1)=bywabwGAZ8 z=3FhDPb*9JRd_rD!RI*0z~`-bw}}4$XH7Tb2kis!N5$4H1b!m$kBamO(&fHY)NCTu zEv25?Tn*df2xN;mA;T|6Y}WA?kNm#}T}rn$?JlKonsvIT+yszVdY*VEC-AP5;Af8# z>*Du=qrvj-yhEqQBw5LVR=J2P4`$EhgX>u0+IVWK`?CCu@Va7YukR)Jqw^#7(fHe| z_&4^D_^7rwms)&_<4tBbzj_d^%jxn%ba3A(Dt8bHVc_k1*y@!is`VS{-{xvsc=!Gi5%!4h{>vZQ2$J4gWaI;eT*tY9 zY-2g;PS^Z*u4+1syw9!a_VAgbh=s&wzd&U9LZ|>P2Lqw5W5OYP%}BFD^J*7lgEJg{ z^~YiV0N1Jfa3io?HPuG-zl{8s82|)wK*-~duYN0c7^TZ)Vq;F5eAd|1o?U87i#aZQ zP|UEo%fmErmi8bva6X2zJX7%=PYUYIZ*#3QlE^&V%Q1}Lk_G^b@yA@&r-${(ArMB^ z*Be8C!_P#<6byn_uRM&`oPN?5Fx+_V)vaThVQoGl1d_(bmUGx+9Q3El9;hQ2%S6xU zCQCbYnTqWSIqG@j`~LtsMMz4iZKvkznxie*hT#0a#n*4#T%TNilu36Vn%j#E;~8uY z_^)4?oE)gVd|_ytbZYF}0L1mLIPsUk9}f7U>E^uCt>uwa#z=M<8P7!^WA(1q;_YCA z%3`WmU@qOD=k%=0+tBXFOE%IlS3G)tG@iSUW5ji@*nh!VuC3=Klxj``;iW1woZtxn z85s8ZS0%0g0Kq>rd9?YBo%O<&`J$3Q`;dTsQ5Yn*`^2c@q0TGl>&uovOKxBr1b_fN zxzBp9B%gHmx)ZHLS{b$OaR;IT+`D z*2wNL+m5w|rvA(NwwbI2w;ylYHGJeGojPZcRphX@yOd@Nc-aX#k|1{Q!yxg=&s-13z7osnN$4e7SGnb10{#i<=^{1G zn-27w&dP>t@JCQV$0v|T&wBTrW8nUStm%?PC9SKh?i=Svjok6?S^gdn>5+L+h~rdU z`%0)$_<*a)u8H*&x`%9y8%EOZ+A>y5amE;^KDg^$)ak|%<%%J+-Ofi`_%WpE+CYxh z!6BPush13-@7<0Y9oY88erNvOKMmyY*T$x}5VUspmhkM5${0-HFh)AA2Hpq*jw|(Y z_f`GVD>Q7XnGU=VJp%wZ?apiG{{Y$-;R^gff>UWV?TzC`SX|n_mcwYuZeq$94tn+V z6jh_m8zFihn0jTDHxDS8Rzx5w{N4CI{@;PE?Q>exCAacqd!>fqTVQO3kaRfXk;xvF z^rwuz0?pw)I^Gt&d+9CP&PgtAOlrtZMg)(Hlas;0`N-on&Ul~UwxYft6H5-E1-`*6 zOA;R?(;P8mAy^)m-N@q==gNy;5u{|#8q+*WeWKV!JC<)H04nQ%p%`I%dSlxaxo7)I zY9<>yX7eM4#vdXjSe3AzWRjnoDo@QLADbs9@U0JvQmi^$Rxp=ssEZVb00zk?00249 zUMtGxx0+>NE##G0kaFXKI`N#0d*d~S!1OEHGt)oetFsQiWvrp{%r1<0E>uML)GfrArV?fNRqyh#PjNst%NIdb5r&>{UWy(yd zG}A|!>l(Uv(rDt=Gmq*5W%@&+txUR2&1r z&m*Qi`&Zh34ZZ?+S4z?T)h>&t$$21X%fo52b^#naAYKMRI6ZpTmj2FO1pfepZ{bFx z_JvublJ0nw3KrZWD5nZncRpF;1HZ2woRdvmlPN{55$cwIvP=AM(!Xav6ll7qfN!Gs zci3@3L zB(}Ga>6$<6x#P_m#^T>tkKtae9L`@FZ;K&;n4X}=aU;i%Y>LJ4Z~PNe!ag>G?HY_f z4fKiF11XclmQXZYWRTEHxA7yjd3K%gCc|8|TTP7|if3__M!_y$AA33E_UTq{Z!Pr^ z43X?9t=2*Rig9Uic&X zRq9u4vrqd!s^cpU{3jNwi|LgzbNKbHt5xvK68zT?#c{WfHZ{X9<)7k{^KZ zTv^Iw`+NLDwFxo7ECK=mBM88bm~7+_M)&Oh0Q)`u(Ow??r2ZZ2zAEttz{@WU=)NV> zH4Sm>{9|VmfUx8*l(EAylvut5?xtY zuA$+(TUL(YTYa~jR#UOY-}Q_+4U>#?uCG9iT_!9eZ}f={NIx*%+~+>@o-DJ5Xf4jw zW%HDB;Nb0Hz$4z8ted@oMf7G5!w-!f9q{GM$>MJe>z7Nvde32YiRJX*tcq8*bDsXy z%lv%thlhM;<7jUDH*2m(r>&fE71Ru%zrN4nD1Vt;V%$seh`q0TbW~yS1eLM z8A--^I3U+2@r3^X!cF4ky12Kqj#jsgPubvD)w=RRjAZvUqLeRoh>x^SOCPz?+_T8c ztQV3_=Ocn~$ESZvj!TA)233ri0vtBo-VywvH+aJ!oBNoeY>P;e#Breb#w%?bg2P2Qj=BnObvO8^P)UV64%*F?7 zoRB{bPfqm&vc<8yw^rHnZu1BdV2o#+egdo9P37gh#$crEQvCYl6YD_9uC51?yLWux zGw$~11a{63rB#~c;6w85+&Tlej-B(@o(^g`XLj5qeXaL`jk!D?gz=tFU(%~daIzL@ zS~PABTjo+dI3u1uy?amvyNiWcSyz?I1q1-Y1a|3;hZ(By$+|3k?!^%Z4bM^3lk3N& zLw6%VGAl@{<&4OH@w6T}&pk)Ki#R7}g(+MF@~yMSyG z7_bbW6Y5C-RoJzIusfZU<=r;PBP$`m1g;NGF|^>-q`#C# zh^@7h{K3kz9QO1F@#J)^5(7fxRg4yd%L=n(s+J}A-c z^{cm@OR0nu?K3T+Hy_8F(h%-^23WYe{dV*~fFf=1CE*CosDeh$4c4fM62xg2RS>r6mNnvJUH#x@E0Chc%dezmqhR(>c2xe7~?i*Xn z34_NZ7Uv^y-Pat~C$HG9qhkL6x{lr`erGdWH}1OyZbl9O$3gBo*7DdlJQv~!Tb~_B zi@eDcizms5RmeMr1F+8{r#bYl@%(3gntW`s#kiv^gY7q;&i$=D*_0tqC;v?*9O@p;l)C zPc}Pdkxv|MY_J@Iz~FSpCk|-cWL0~}iHQOnaldNgJe53-yf$;xRi73utUejhZP|RN z{LH*XT~Ws7ZUW#E3lq*ypdD*2=A7*5v`>gP%>~uL275ms~iq(k!ga@@~pHJ7ce41#=z~@a%Uwwxx9>c6jqj#~F;a0N`MUBRq8kk?m3ZMAl`B z<{M`6)a}{~YGrKnISN4l`-AnM+Bl6)qsE$rocdLr#jG*gU2k~h0y5c9k>i|gJ$eC= z>sdY?_=TzXUR7&*&$Gw7C1Qd!$vtug(tf1oxjSDyfrpecmM1+hp@-*PQtG;X zzPY!WCt<(|^7HS;Pvu`LKZ&m`bnGqUi5zTD<${G6;|DqC+ofOd*Tx-N!op83OUXV> z(FVMgqg)K+g2aE{HKb}Tr#z@%RL`O{eL0rHP*hfCc(;tG`Po}=&#(9sTwlk38V?3| zDoZ;#qifAbHM(c-F6F~XfOEh+b_CUrhF=xo@$C1O?{N&$TEn_sszDox9PmC~I@ix$ zGVvU~HSu}XF0LfF+vP!QOh9d3qyoLa-oZ6WrICY;n3|V@tZyM$T0N?}RX{sP+JFJZ zI}cyTaap?efEMkTi6oC`S1KPnfdPk1^MWf%{{UHEHYQd>>`7jE9CKH!?w3-TqgkO{ z(WW;@cD5B)f_k1hdsh!Fo~13$8rMX*u#3o8cPf%g?&k#Ou03kC{1*{T2bl?7vW6@h zs48-QQC^7}eT>|!+jih6AP#sx-S@{@k6Y6%j{YnF|aHo%ME4bCXGz+ZBBS*9V%g~dP@9kLkUKd%X zVNlQ73G_J~G08;$ zR}y)M#ia8Bf;MA>Kak_69QCTx-+hp?0>8XZkv8B4&Tx19km>syAp8+^4^Y~&0CEssoe2il4%qDiF;S4P~*u}8JojlhiYfJoyb zxA=Fe?ReTz`>o}$$#Jwv%9Ga|o;seCQB(t7_)7(*(C)IR!xap1(Dfs~9kJG_>+m*^ zVe{jWW12vVxxq4I8~{FGr;*pviYq`cH8>vbU|K=tu-sE-M2a??_2ZH1c>32lt6WJB z+8|jZjg?V@x9=Hn30>Q=7<2@2pHW2=8Jw=6rs8e4Vh=h1o8*y9!)^dMP66W_5_rxC z`?by6_@C_-am_8w#hNf4RJJk|hBDlh5n+0O2H}iv#~@KfM7IL;5bATu3=Jjy$Jit) z*?%mm5yLd1V(+jF1t90w7_IF&eN{{U?A+U zGQ-5mM%I(>W1Zfa2Pc35>E5t)3&w)xMQJV7Z=)c(+mo=Wu_GaY$F>h%v{6wlfZ#NL z4C-3vh^DfN>MQGAJb>{{3G&c}IlwXw3g^EW0CAe7@fX5+Ww(een%?70x4w^ZrWdw3ENbAr7G4E09 zk0g<4vOt14tq5}4_HD?(IbKOQKA6es2^3LWCq#={727PAU+UdvanD`c zb}NR|qmCPUdy9EEDJE63*E!vf%wmcvWDzstww50=g^hP*p&NGaa5KRTf=L+%1mpqV zA_x{ud&_RIqYML(2P41MiYqF+FH=f?YtXK@va}bTSDLW4eoey!HhAM4a1ISmrkA?D zbRkk@w{}-52y9?rkN&u#ibR`V3Uy}i-;d^zRv84!h{yq1IOjXEN4Vp^w>Ho_Df0J-Bi9rHyMQusi|)nfkuOkh6jXKz8%+O25vskni9oZu7c2<=4`irfud zR%ZJ=dsvU&uu`KP^MlVrQOkSwi{y=$=1BLFery4s@gLzu6>d!c+pETr<5<;HF+I5- YGk=Xykcl0#+i*x9E^|c`!dsvJ*@5oW1^@s6 literal 0 HcmV?d00001 diff --git a/utils/external/SSD.TensorFlow/demo/demo2.jpg b/utils/external/SSD.TensorFlow/demo/demo2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..568105fe8152e73710f3ae3e90deaca8a47fc60c GIT binary patch literal 31989 zcmbTdbx>SS6ec=^;7)=&43MC~Avg&Z0tA8&5`qN=cPBt_hXBEY>p+lUaMz&0-Cbvp zVesX*yR}>O{(A47+g)|5ZgNq3JHsdioJdJUS2^_N%_NPEo~iLJ$(Z+a|=tWZ`L+0 zu5Rugo?hO;KSDyo!XqLRlYS6>SEiyew~unH-WHT{Jt^Pn|swfF`#UH8UNPPIVRSN_vzkQ~au>KrbHByAt7k zMjlh7?Y%gCNmrttSR?Gc8LUsbq{xMDlsbG$?nySOR_qe7n(>bLT=O-a(uBy+BwNiX zG>3=o5#TH0q6>XfbF6K2&>t_hXgK<()?nGoVf*^Ih1l9Xr|0xkr8PwVtKFC5u@t^T zEqkc?8{x|YZEt{QnFj^GLOUbAbc|nXInT|3Jj_CednrJAvJgzQR5DmIf`rh>nBx!C z9RrYMSEO9k_B!|#0+};pN9RN6Zsx}2(f&{kx4om|6t<}Y6T&jUBLbrzZ_lTm#8H>pZz1WS8fl->9C`213 z)Dsy=lWWOHG!+RcUpdvMNW>LUQ>HxDN2VTo_}Ov#E?UUTw0mZCM{Xf!$%`HVae_Xl z0MwY)xZ5jW^^{;5HO`s`p;PdGn8}tbHnnU9WwX;uD6&{_q88`HTX%14%*BoVoUK^fV zdze*H(*Mvz#v2T|7}y;2hofTe)2T@GYk4I(liC`5Jnz~CcKqM;7y5OtjiK0#Gwg1U z!Mzmeb&`KWGW-d8zTYHzR#?RryQr7t zpQCrL61B)~w#xJu?RnSd1Ap)^1tP0=?bY!PXIe$N1K?_7;r?u%t2(M8GBBXM{C@|tyWHVwH6%S zR@~c0@CXR{x>sM#mEKoacHrZ3Mo{mV5ZFI-o_&`zwr-riCOwzm`GBdn^9R0?j33U= z{_W#=z^eL)-!nnf*W2pr5XF_lyRw*JE@f#hr<>gyU7B*?6Sr4|c@oIm4tH;?IUR7g znUMl}Hul}bNO)GhgTG$?)0EBmAYENyXD5ShsCa&>kUul{hCn- zc9Qd2kfg}mEppKpsg@Y2mvuU;2F6R7R29-L>!%Hw&_k{?VyF3{NyVv0+T=-clK-@t z&_*6u!oG1YNRH76zcG@v?AOe1U)P5%7#o}3Qym~B-?LsY}})S z6Ho|>M#V4{#5q^kB#LHo(l_^i^7E_!-;*l-%j+;&bHrr3)Ml_DYTUj`fEi?{R&bACqIv`j3_SvB16q9C+nA?-Q@Dek6W6x8 zXrFci;|W*zPFEU1_Xd`YUu#VkFn5On75+T}X3Np9Mo;_ueB3M((kwTN0T+jtl~_Fn zQ(qPJDUS{tvH>^9JMgwTvCV@tLNQ9h%Bkp1E>0c#2l0M^KEui-z{Td~$Asi?ysTaL z0&po_4AWgk1!Hsr!^Pb@r{_38iQkEGCjYt)3=S+VvFWkG;D`shy~DcC6wS( zsVKRK@Ai3evGm4g@(yPshw#n;m1rwOz}rV{J$+i>IjMxe$Rux4$b8`=7O6soKlT#) zWP;Pu@?#0%x>{JE)QfrWjYWNR3CMHfTo%E4e~Nrb{`MiqB7i%l`P$h0wR7oxvG{PU zBJ!y~iZqi-(PFcc@YrIl*om1sD@O;F1 z1sB1ccc8)^E)A=y+ljJY7SM?8GLK&iJqHF^==HPP4+qYR61-&Uv4AUW2}KR-^vqdn$INHmSCKZ{r9$+hFnVBWx})7@moje#25Ayl2*EzA!>V zN-_K?j!oXHuhvEIMg6wrVE?ixp>NCJ!P-CuES z&Jfkb1$l?Aq%RF=YAn{2>+VX;fj3rnTi%hND6xVtkCa9KeU}b=m+$H_YJ_N8Ya4aQ z%;stZf;8%-5lXInM%whQbh&sXNT)gF)s3!-*KZ>uat2F3$anq&e&r52?_0P11I`AN z0Nkac4VcIoN9QX8gCGCiQBjfPKifIbJ{sP~lc&BdPf$4|sV}@Yg5|Cwo8*y=-u3nF zR}`tA88}3lqX$m2`RZIu2_0vV=k8^%*>%?9C`Ex%?3oT?t z)S@ZwiREoiR(5in+pQ`J*QyIVAzq5>zMmSH>S9a#%PHvEFk9rAK)lRK8??nLV+ijB zkB7q-L$llN5@`d&3DECR%04(xe0l8Lri1%~X#4UNNlSw`iaV z!z~XnJxqE!e9OuVQ8v$FcD;uCx9v{UjWY>f)n(_pu$|~>xaXvQv=SRt)EJwnv26NL zOE*Ec>WQ`pbuiv{KRXV6HSBMUx%Q@*_KUVhFut@)+?%!JcV{;)6Tb|%E2%fEjZ&n! z_(oaRir!vcmg(xiRQN0R)-7oQ8Td3F#xE+%&PKnwY#(=6xByMXj>p8?ew*#NPFg=Y&?RO z^~)f#mVz4y`doE_LZ1{0VXcA8%xy;aj^=h^1a30Us+X?zZl64`Q;}lOXkcN9b!~aS zNZ)~Sf4;)!ULA_jYE1r0%2{&!qo0>mBxfh3XJZk#l~Dl9ER zz4BxJIe%sCbjaMw7EWv!b>!#h*oM-HS!H;p+rj%NQz*qGF{P4+ZLYliT32t3M!NoX zJI+;f#C1@wAOfN;;s{o_(oP949A{((&*y@`S@s;Gr7f23E|tw^bU?VA?Ds^+8Y zy?)U)Wd{5(LVY|i5brFY?aOp)cDO*EmV2uP{9?A=g1<`rJ0x1Fu)Wi;G79$s@o@mBtz=OirQ*4iL)fF){xG ztBv(6jt+h~2g9d)tvJ($JTxXt()@RdQIH5suN=`l*_b_M6FYJE zAzq0?0mEI~T>4x6Y}a(Pl=nxAG+QcI(Az!Roasj-_Y|zwj5hfCR@VaFyyhQ_AnJKz zEu4-oasy4jFV=_^7m5{sPFt0J3iT4D)STa+`fjP^Wj4Z|OI)&bOgF~Wc%RR+J5_P= zWHgzcU91vYdvBM#AnkO$)jR$YnDN{N{~KK&&X=B71cAv_sFc3h^|<+pN+!F{eW^7z z)Qdf6!u8`zZ$WzXGD{GSK{sZ4T0FMr5-ig1JZn1{a3And8Jf^3!DWhG>M1kVe;^R^ z+yb9r_#bSp{#?_JD+%3b8{UG$x7{Q3Nd+A>&C2yOs=u8b3(&`CUHTj)4_^1@j5jL3 zo7#{N*?cX*`}=Ul(I8ITq+UfN^Y7yCun|%f>e6eP;Y9}a9SpQlXSL!`f1uvJ6fLxR zMnaOl-6^kmEPu(+sO8x}wa&MKPlUH${sE(&NywV}q!Ki_bGADnh!n=R&%(m)=)B2( zICG`EA>Uo{?Fc?h-)@l+OtQJsiLj<>@W8YA!4*%2)M(U-j#uhL2gY7yN$mv}P`dMzd_OKD4~9OabTKd&WXdYj-h4jaerhy5b- zye!y;&j;nZ|S7k`Nj!ET-HgbN`V*OrOp7!5ii|g+=L8>0YCUsSTTv9;L!MO4|4IQrE z*ZxmID=TA0O;fH>An*@Y{rs$lPTQ}rRXe8ITgBB^5!>Q}^p;^csNIcF=w=iz-ojx3 z;U!zrhZl#bJmg@VDad;!^Yk}=b|_)5Qk0$7x4)juGsX!ZxP+x%O`bpr7`kBIs*vEk zRl~xbR6c{+L>6Ve;=jZ1I8JP&!PKkCD`e8lUEaTj6+zQCit&#E=xrL2^ zjxeh^Uco(!Z^lgVe@5(Q4#9yB32H^ z$4ZPp7JwFHY@Lw~X5fgzaCFO0Tuur_iWH_oF2fQO9_&H3l~7fcPUHjQqy_YrtoD$2 znJ44@tP8 zsandJZ`Oi_Mr9<%dUTT3S3}p~g0HGgUWaKQcnp=yrreb@r-TDMkl(D7w&ciVoq66- zk&EJra7HNfO}9lAetTMf57m%&d3Pc$w-O{E6(^VSF{d&;$(~eo+$huSzVi@6g5G#Y z(Fflng4L);03HyoPytI_p@t7zuh$aK3erg%C!Xvqh)Lv9ws9xydR1)?F*~s|&>PAt zml171Xgb@_5DKXUp6gI)98^(9F1#ngjo@s#MH%UJQgCKgAHBGwP-7ot?GZlnEQ5zN zffqH@XH3&N+s5db*xzHIx&uKl>t@qDuWctqxmBIKzX>v`nAv~PPi)QIQ~zKdtNB`+ z#nM(;o+0SQI2FOUp}jlbBEDAc1Np!S;ucqKpFZK@+M2TGVbV({$St(6e))b}BJDMw z(c1S7I~t^Y8!ENf(6d(61b^TEMeGg86|ZzS<=hLCSKw%(&R$1MVv7&GF;b~47fEir zN}&JEpBMprLXXO_&P4^wPqE!%LS3v%<`-8Nm7kUPSitHV8e>03B{1gLgm~&>t}|E8 zhCRIURPxnM?OFJ_a8?N^y?z9kLwlR8YbpCP%9Uw2yK0BT_zOrPHtR9E(w>tjzIvW& zup3(!uppxK&sM_v5rCbwy)<}VEFB8nUz@lX68~nbFZInx+6fHE`g`1ky~Ml~Qkl9o z+iOI$v3l%O?AhE)J$-KDelwF}v7SNq^{?uuI%Q^A;_QzJJ6xuEFscUkB+dZ|%zXG? z=_nJ*?TH7f5yD1fGi!UAX;UKO8Rh2f*A54AobC5k7Bo}UpN@ut)>ogb;4WUosXBtJHFM<+JSBBg zg;P8#w3d7Zc16USg_PqVoahrYYm8lr8J26(%?Gb=AmX|q=$&u#4zk4%RGb$Z`ht|;p$V~s7o6-8uZq$Lyh1~Pu<@P%*-(g3JDgP7UF;o`6s(Yodm)>! z0)+RAtrs0(k)My!e?0;|TKr6-koewPJ1HGkWti>kJ==9d7_mX1WTRm{(*)VrgG&{b z^xHkJQM9AQ&@N075+y0sq6wFN^9lRFuZ&?Kbwi=*w;}kAr&OOE+p>X5)Lcbn_R1R| ziGW0&T>jOM*IY|4BLCKbK6Lmfun`S?X7HaxU3>r5G*s*_7z19uLtA6{_rn> z^l#16&vL?Y>c+yQfT&7Isk46hnu$&2Dk08+;}c6TrzmrxO2kW?Hd}@0UbE|^-x{r9 ziIZUJ;vS+GzKjmkRS~XnI4Flt**-pc49cy|iR^KrA%(%}iSrhcXdbFtFQ8)DA2K0- z=%bMqCPNF8N(E<-?3QSvzM85O75PMET!p`434?lneS*K!thD(f+B+S>A6QW01*lCB zYI38ML2-IB`G~utDejmn$q;m(WfiO@3>J{A3o#R6JbpD&C(1t1LUxO3>_3r1Z0jqP zErodB6k6p(%gI)~Ar1yPnkO?G)|H-DTT_rI&|6Y1(rBM~pdV%__AhX}V|tjHK=x8< zWI&T4Q(FMJMCV0ygY_1pTLlzv*W@5FOV{a3wEviulK}Is1H4ah$A|h{eeU)F=g#i# z^~MND5%FHwTQYN|+QuWB#U~i!_q-ExHCRRLxdul$R-$G&1nNO9TBQWL&)lj<9PcE% zZK&hh5brH$dIoC=UqK47^UjYGxYLc)C|T;W6&oWwRB~Pet;vADfQ0wy=)-rqw?3Zq zUHSzeX^Ml-1IItB$9aN|CD6`aNn~@5_^LkwFnDTV?;MKm1W>`QOc=^wF&1rFXF?pT zY$BF+(W>%G-XMP*xMGM<0LdZsOm3OAle!goPVrvTbjbEpj0?eHlr=A7Bv=yB+SGRW zH^U44TA3>kd!fDdxj*0Jg{Lo5MS!ib&5QjRNXa{2Ue`IcwuV?BtSMGBX5`18x__1` z4aq#uoqx?>DQ%=*>#X?)cs4)a3T)EMG~8l!`Dse_yB{`#QqQJ*xMX%$Be(pSALga$ zE_RGo2$JOEK0*K1y=7UdjsjYFjKEH^TjGhl@99z2@TBflG6z@A5uU4_dIS8Mf^@Zf zWHml4$f0Iwso#lMJ%J=~IaucO*PJbdK)F}>C~9*3hyrhIF$d?}22N__^cDm$0zIXwOeLkyX9zq?>T5 z{QNzChNSn?bx&MVo>a7UtE!tuq7z>|$tmcoXpIPl~r<~uH zl7!dCm3gXkUL~O{XOEh;TPuGxS+wQ)@$mvRX?TGH^}UJ9_|D;5VE0t6nwsxVO2gGQ z*Ha@4-AqvOcN&f1_-YZ3^Y-0RCMMfjv+KoC{Jp-fH+F{TdR@H+#@|B1+}pY(lK3IcWFCZFZSo@r~#n>3NRm*8Uey%8|Z--PIPw&Oy7wTOd#VW0MO zQucV(QJ;%ELnRe`Rd^R>&X5OUMpow%Yq=klSaDtqkrq_q?k0SL3GhJtw#!A*L`GoQ z#}7h#^mNsp-y442jhuZ~{D`b%?@EL~R!ZhM^8CyA>RY?l+qUmEWNI ztaP90|8kHpf-QYvb}ptR)g~tTte_m`=+3LY#*X?U#f=CH-wNvbxGdCrG;eK=?++h5 zP!?~rjaQImp*!r2vHn`Nz`HEIm5ja2t(Og))zxl(voI*2M4PA7kH@B_=CbI-&Rztb zdceP+Wwv&3K)nMk>aI9Ufx%|pv=z+2+`CqG zcBXyHougrx)^TLK8cg2jxwz6u$FSiM*+R1(^R8Ps9SJSmyNUYyv?mmMnl|w?o0Bs3dA!zU7waM@ zB-~h$2A{)n+q}*d0vHwdP`^4ZGeQ7EZs+a`1h&H zJnkxrC!b4`25gu12uKN~QrQjP2#Hnhurb@PuQn}Fbh>XX?RK*(4FI!1rvf)yH}4tg z+bp$}%#xDnoK(tCR}ZATSf~J5ShON$6tZ;NtwBK9&qE?dXpo0LW=j!F*zsX!xq!w% zx6BEw!($FY1*}o}g4n9v{3KGm)xwGf{hZuc@E=Tv-4TpHPAYcKPFmt1ORBSS-Sw*< z$G+grV>SjV^$pgXzJD|jshug--jSt=Wfh7AR?;_?>Th2sXV66GNX9t-Te1;Ox0}hX zF_g7(t2sF#MVA+lZ!}AoI>CQ2>iGMD$7=`HB96f7^X3gbY2hq9Tw(k(0ev>Hv|nxV z;m@cAE`PVv=qG^7ELyBywY9gs_GNX5+xqc`F}>IkyPejShIADl4NC}LTY^!(L#^Fd zTI%PR?}>|wni%%rc7h00nsi7Vs0121sV6J=@CEcf`<3Ftmc9$@sCodXof^VdVK@D~ zGOnkc;DO!KswQIr03At_bjITZ0i|+u>l#`+KJ8Hl;$@0^JI(u zt%!-IdcT2XqJ05ae4l>0@_osNGKa` z4ladC4~B`wgWDH-txZG_hbMEl2VwO?hz{4zN27I0@DV`J&G!e|G!{r@eN7Ucyv+fYF4xK!9&4{uL{}75`vy7R$n_Fa#m5WALIOYx&=cq^-i z3fo-mQ8oKC#oZUPN!|jjr!ETOwcDE9Y7zhvJ-1ElJ`1r`I80qJVU4H?Z^M65^s0w9 zBHcbNz$&*J?6pH>QLjv{MbdNjngAFR+$ggtVYa==@~_=u1aFh#I1JJCG!XuT@X92g zf;k(_#RKYy`d|IU22nv(z=|tQnr8Nqm~N+s6GwDThL7M!z)N4IO*(y@*`Ax-(12>8 zb%RdD^K^lw=}Dcr8Rj93TOVQ2 zt+yUnt`|x?&@&KDSD=w&=1qzA5Nx2lZr%(Fxu9d|dSIMxW2lEzfIuCyO##S?PmS@F zM#omMl4)OZ$9_dCX!-umj4W2urV#v8*BWKe1A(8ehs4GaPESkdTag!5G#d4BQhc-L z;ud44w@Rhky!-t#am2Yu1S=075F;Uwr~i$5WHwxAs8mq$&77g9<_Se3Y$}iIV@m03 zDJ|ie`dJISCdrDS^B*jm=(l>F=_bK94&eop7a3U}&JEip$p!F7NoOWTM`Z;pRd3TO zsdVvW7bh80=3DNoY~5lEIt9haLh($9#(c^ZOGb7K9|6{R^`z)DyeAQOotjFcs&@F| zKfNl|`CH>4t`77lA0-t(cL!pyLC^T!pFIyL%2rsS-bpfx(!%2L$8GWfRhlu<-cBeNwKCfkIWOs!%X!h&I7cQK&!VN8&-Dni$QPgp zod0UmEmxFFA@MbE-hpj=!GA&(n zD^}YrV~g0H)wh;WM@uFu?j;Jec1mU4M4oDacy_3w>byOc=4R((<(h#F6zAfGekf&t zGcP$A7p^bAUF}vO%m>#~-bKi?H+q1biJy5~+S1Z~m&)QZPL^oU`5@L z&+4?XQW)?gxY;KjQv15+ePfH&`lbm+>5f=G4|+{z5ud{^nnUvs_s-(!#=?aXSZ@!P zu4`?N^kprxPQ6;;j6{e0LUMgtE>+T5hjE;czSk%~Q!wCPhD$Zx|B+(wbm(TlZ{fTJ z`V#%~m3J%I^DfHqC|AwJXb5 z2(?1`;LW&~Epgl<`_$&=$_~OKiJ8Z*z7C3ZN^*t~#X3Wd7M51@$djxp_5!HtcyVo- z{0oawRkGC$lWeg&qZ(z4p@Exjo^$4FWE|pOZrn>0dsp#SR|g&2xZq1LQoiIYHx=jk z(e=2TVq2ft9lD(~pY=&Uu#e@;nG6+n3U{KM&$bdl`lcQvtztfI-^&zOVCKeLXfn@CRa4H^!nTw}B6 z4vkv5c)D@o`N=Kw zFa;e?fWtQ}Fy6;;eH>m_dVbduZN^yZmCz&oZQU+7VfE+1oj^HvQLYmW&sN&?$MpjC zKUL$a^8#xQcT7P;hArbKm!NB|PxH#Y)&0f$B->6eP55{Pl7-FJFAf+=_*kVb?_YVTu7}6pJflJD4t zg8)0(IYYP*@3HmQP?82^-cvpIF@}4aJt@(L>9?o78Mgt)c;!(IkS`>luA1anVny~Z z*Iox|0@;__H<(pURvR=5%R35dr7|mxRMH=J;gy+~bQ1-QF#d|`gLzv&`QNC}Rk;b< zi(k92+C>05JH2Hko_I}Vi12&qPS8SPBw#d>eVK+i5u_g9r%M z&Yq!pKYfx5^9(2XTl!i?%YJ}?_bJrTy3Z9snlYbly4RIg%kL0i)~YfDOgH99QU&6LzQ^hZsmCNuY4=KFOd7j-D|5a=WG(fO{cg|=-CzqX>%hV0hfo; zec96E7mGQ6mTYDaD`MiN1ZjLHQopJ+bewg>rEB+lUFcuKok#n-S4m7K?mn_MO*m+R z#340a?|(-RwDawsE94|AulcAPIz-+WC#{!QonOvF?2b*FoCKpMN){+@UP~&%qo)@+ zA>^mXF!-&Fs)JjBx(`{)u^`vK96maVt>>g3{1>8h=f<12BEAASU8`_SyeuituRd`& z#c97}5@{@MC2Wd4u9Eely#rC3k>$3)!w(|RF*etXil1~)Vd=Fh^0PZ0IM(;NIdjD% zrL*CR;0J^6{2kFABmUlZLU1IcLKc|f;jh|Sas5#~4XauRd38s2-j}nGP8eIMomQME zLm?j9XD=rC2E*3_j@KJlICW8|*E^MeOuRm7Zenw7OjkN4JM&?%_MrNuA%}uMO1)Gp z!XxmBeIZ7@?NYf7iP6NS%dDdR8tWFtIESe^NE$FS+A z*q=`0bw#35u>S~XN)};l@jRhhO4*FS2P%G1cmm z{b3>K7*)f51RT!#-CBY3hB~PYXKL&k>jFU>XuIARv^VisoBg7dc*x1vjny9E2_b0z zJWsq_s->{rwkwwBJD9b-Rzfy9;p@*ui!JIo5DUsGmc;wy3-{E`$_yv)%Mm0c4_n%I z=_ZB1&82`f%~wxuaNRvF={n0&_|UpS>UukoiTe5}QAfx^x2g-Ckus-Ok}6AFx}r5z z+(=h&og?Sw*j(3$YYdTY+bz$~h}_{(kS%$_dd&OEET_7fTnVdfI?m}9PR#O6$qy8uCiBl^jv72o77|5p82 z4`KJBSA8Od-ZBIS*zCf?q6%h;QTj-gM*tZ&B$iq+?DnlC5$Pp=ASn-8=UEbSgB@qz zBLJ(8fqK8jGe8#6A|hPN0of@nmAfV^k`VkL9p%N&&LVU{(OV9ACy&VK9#Q#t=0BGX z=?fz6=e^SmB}JmtV$OZwnh^`a!3B;>c~?G1C@)H+ev_$c!0`wR$1F-(OZHtZv88pMWUzk zoFA!m`@{3TAr&SDJ1TuuiD2=sO#a9 zn>j|J+$*^Tx0iYigRXXwEdQ1T)!u8&1;@HE?kec```mK}y%mW=f=5i*i%=4^Ubv7OEo5@k? z%Dz+$O^v>9K1JZ%=vQVK&xG@uVFawXcYCr)?*D zI1i#um|ixMuz_-up5@ltf9%Ok@}d%KaQ=;+ajFHUphXW>d{aN$eTblXiijKCR5@6EC7Zhv*lAykz6X@ha?WjX} z1Meh@BC5}q^b{rn=j;~(WR3VAMk$i4w=Yn3DcdaE)S#6Nwy#q5G(#NNfOzcAJ|ctL zQq$I^y4(QCIPxdg-^5PidH<1JfwdGKYH+;@|4t760ClkXeIRG2)UQ$&jVO6H+w{fqwu_9f&XlPg$-}R|~o9VWU0!CL}_!L*?bO_16Q$yVO^TE== z5Oz7?v5E@+_jxPw1F?R^X#!o@*>f8W&E>b0CxwZt1L+GM)=Y^XwIKpk?GeJ#wO0{$eojUEZ#-jf7PW~DR2*YyXoM1-SZrzY zg{L|t#VW;-Q>d}LyN?3GN$V;jxT}A|dpY?s#y*|Ma#7HkbE41}O71vPDDPx9zJcxy zVcR`lOw~_WyVjg~odW6~0g=GT3w3PkwQFuWp`OCVp|TRdiz9J*fiYw79v>AUwQI`j z)f~9Qff&@l8Offybf~$c_OGIXEEG6~pVurM)P`l}?NG;`q9x5sTQ5Bd7Fna{4wvGU z6<772xe191ba_^GKRI5hpz6oB&Ku^Q{c^$-7Awv5``X{ExtP7mAKVW)*^4v7d49j6 zG3n-~-kA~^!LV@rb`zDfua`d77gg?&;wd=EkUFHw0w~E(y4h&9`>afXL{z@lxS#M6 zS~G$6#~%BN<;mvH*MYtKXj;U!IHX8R^GS_KwdF}QDklr`zK&O60 zP8h$x0>8k|^|)OM=!g;Mf;rgPSwGJ?n@{d|*rk*XLq8K-B}REUxJpVLsanyxlWS0! zC4QWR4l)DS~eWQM%aqI^2TV>=`K7pd+T3>Dl9j9 z++n}@;uA#VeXi#($lJM!H|XZTA9dNTLqa*du*3Z0Z57q$z**7V^qbm^s-EV6n07Q* z6Y_5cv@`YYZwF#`KQ+L5f`x}j6;8y=e#t(ZOtfA1lV}RuA0;)7pA1?3&7zO{+<(Un zhqO_8M$T-2`8DzL>&y#D2Lq zrT%#uiDQF&(ZxW(gy20UD&DU@Q<}N?s^~u4@A|!Ha<)>xKJtbEsf!56dAhBmqp~FH z_zV^AlUboXJ$B@Ac3biSfvYb%26K>yyS!lmbp$yb(or|XQ77EG`b`r@26R%>IaeYc zZRQG$w~mf)PNUbZzt9HB?zMj*3WwNjR#Af~YFK+`8Vig@fGd_`{E98QY2yJTV!n(W zW47PHv|Y9O7Vn#~St%QL_<|O86z^W<_d23-WC#ze^|k5YLzv~u1-{}tcLLt0ZPNf(2w4`kUh-K%2m%2$a4 zl`g9oLjElMdR07KyS#y$a5IuScayZn+vA6|(U)_di;Mz^!*A_>XNHLPYqsUDLcA$r z+ZgAI!UcV(LXHySRtJimHA2$uXwmTf42006BI%mYfh}h*~9NMaATTsl81t6Gv2=bFQtJm1>?zp9ZQR zD(9tmyG4TBR0b#d=H<78L$A3ifqnaY_mg$cIBQ#{nI?`6_UkMEHM_t?cUpost0Ex} z^mp=c+AY?b)gVvJceDyJKWf)9U7|gXjd$J=+8k@?#>6F!aeNTYu(R5nWpatn7N#=KK z@#;abQ$25yD2T5L9ms%7n_8~>{_vyWs z!pLfzO!>wz0IgYiKVLDmTc%FGyCT&Gj*l4|3_V$9-3$b-$6x1v>z@$E&Mmfkgy?1xFD{#+oB~) z(}z52rf3)2Jk&E`xELW^MSei!%^y3vW<)bN_pIaevrGEuX0Q)(Q(Yv#xw}{dn{~8F zqXvBi*5fb_9EU=y5g5?9R|-J6M5*=t>k}mrWrn{x z6G48J9%$#hL!n!CH)F`qrx>}ME;?{edh&E!IMji{8sTZLG9spy!#YfItrrW7`OsHd zt09{) zJl<`@Y;2#8>Q8BVgMv8vchz4}EEuv;c{mVrfl2ujy%XJCj=(vnDg;=)sb#8ez%)ft zcs*ol@9Cc`2VX*Jpt)IRjQ-~xN(^N>n@7M=DO*keqSJq5NA3|YuTImK#5MlMDm;Zd zIv=f1Ao2#^k|WErRA!R3SCJsBTrB_$rav$7^w9bjU~Yngn!w)H^EI_9G19x%|MO-oHo)L) zzl*lMaXAo~zL>iafG#2LM-={E#Fz%bIw`iqo1v{t8PHPL zF2Xm?V*8VD!OSnolj58jdw(ARa5)Crp<78`QiO5p2In`(r33K@eK|q!-%{2VycbL< zc1y3;=zZ{YJdUw$9Tt665H!k8W#A;xs=QEte2qO+DWBiWIkbPc`s6twp3e+#m{6nC z3(o}e=~Vq0_As^V8LJwH zF*qh#MfUFkD(r2awC4BWm4raa3%|+(7d09^#Iuy!3dJ`vN-RUF1%c;Q?2;E(8^e_o ze`4mMUvVhPwA>4fnfR;vZF9t`FGJ(flf}VPr)-GsgC=1}v0#J)W-E24yB!NjwZ@g2 zcUG%{8hCUx(60CqkSB+Lw&t9eddYldCX9*m&6%mvKM+6VVOqXjbRYbLDRz2#d0_K% z50)hOD=ukti{S2Xvmqbv14)Phc*Q5XO<%LJ&c?Q0lBGj!tjaSXpMih1wokDtjrY9L z0oe$jl43=DNF5J}y;WQGZ2USrxRWQcr}qfx0DG(-FM@2^7z3K?FOqSc#?n|Fz^j(i z4wjQAKb-ZsOX`w81T}-CY&MIWUfsz+_Wo&hIkz8zD7CM^E)0|?@&23C@0aC%ib@TM zG$o!)UMpCvN+$$9btdpnbm}U%A`22?4rN7!P&*(jvu7evEV@ec%N#AjL;#hclF`-1 zGF9jEdq_CL5qZ7oV=Jy5A*oFHRo&lv)(t*$-p#%i79(BPqAZ+RF-&nJTwOB2XBwB< zKoKBT$Tt`Y$lKcD;oQEDHe51!H(6-l+cIz}#p%E7|7r>nP4_XG833&27Oo2P_g4BTMv*o>Alk7?b}#oNj_DP$+@q04GCc$D@$yhU^I zourti@oy-m%&*pTmvU;XIv{vRw|_;m5;92du&AO}39qA-F;f|j@r2dWm7l{jq6u*- zqopuZSF4i^wVG;2eBI!0H$RU+q&Oi zZ1GXC9{-`pv?|jxe z+m$}kTx9)b|ILAJ{Qg;mJ+e#T%=}v?cCWYjvw*jV)t5F2)+Us9Y?Hiy$Lp*dA8x<+ zLoa$$g9=k-#F$Ldg)khY5PJd1z9$^~TBjJf7gTPQvl>tAy%hx1wNMA9g-3hkdff!i zG>%2JlOEkMtw%Kn5dK=!Y6{7V)hsp#g^Hbk#29A-obumc#!sp|8_*Ep$DjhbRk6(8 zy;?xFTNdz-csZ8kdg3)iqdgo{`d6sC7096Ux)sRO}n8+@RAlUR3?gg{E3V9 zKG_cVfaHyC<8DT5A9+wv+me{bJF2K)GqfG{t=#u-v`yEpo6P$`Rn=zf8Y9Vj3Zk!j z9EsXq<)@C}gd$x){_GZpZW2buo^E$UxNxberzqH}$cx1t-Cjt?xt2RdDEO#6#7MEL z##pXwU?%X4P%b*)?b*mXq)cNyqVVSs%}C+!FfX%2Dptb{VHni=rxflZ;D1zf)nQG( zZFGu?f{LhgOaVbjrF$yUAQIA{(u~nDaz6u*oPdDjq??I!PI8i?N4NA~BL{5b+xOS5 zYuB##-5bvx=RW5=VPO3JwockP?)rl8rM8<58x33FtY^o`>UJhoUm6f9y`c)=U$N|c znl<+KbeMK;fP9}NA}WM1C>KcJ2-Y%PJ8N0Geysd1-+#9QZr(?Xlmz<|R9hpKEjIGS zZLIs(_`*Rcz52i4PvfNHG|<5&Lg#jB69~@t&Bo9AYm@u)IqLoX(2@?36=9E;mKYq5 z;|bzO`TgqS*WGm;>o#;GTcY8Tf}b8SxeFHJT@|J-ZTPW1XpzF-lD2!DWQKH}p8s5| z>FHk3Jb{(oesZU`Z$I;o7n>=g^;LZG{{gWXc{J{lU8-ZV^=&{`b(i{(_s5yE_0~(2 zp=orx7~J}4WfeFmU1-4XXnMl)N94Iq>dH^$qXX#Pyl!s@Ls4I;ytsDuaY1asb}R3w zh2DtUzoFF+O?zE!@k4jD!*l{gm8z@_&s{@@ruEtx!qecNt7q27qz3#XE;(Npxv$%y zV4JN-xsTugxm^J61uNj0(33t^(%oBIVj>Q)b+}vTBV?Eq&gHywdNwVq*y~_k$G^WT z^3M+KPwu4k(OmY?H4E-t(9_jVU;O0};jL_ANU488w_d!fb+{{OAGN!2gM0O}ji%p%8`{lV(AqnEb=owFKQHsf1eeFO>RHuL*-uq^v5YlJTJ+mO#-oxP z>SOEykDSn~Muu^nbt@k?ZGNJvWN_|_Ts)`E&-m+dCuc2rXIUZZcT{Qu_fp_L4&s&@ z3A!bg8Ff1WtU8>YOdO?r@dOx-m8Y_?8Qa_<#XH6uZdA+IMDAm$Z8QdsT*&b^cS zsPz0^$s9@RfTKUoW@q#M$Lq1z#hCTD20~`eb)Y6o7#s$lpv@)JmF8BwjA1CNOVeo# zX`i)H6>aZkF_UN&SbPc$XwTBP)_R2;>E`y1oXuaAFu{6PNj!jyUjEzp@=ZFK2XK}7 zdJl=r20pd-$=zQP3r=(Afc64kCE>B4DuFg&MCiS`dUxhcWo8R-cLdq4fTAZo>g*;? zr(pZ@t_fET)WV1eu9$V29*0MAbeiidd}zGkRm>VMUN$*3yNFE>aEy8}E%%`KPVb;#Kn!YA+I3HZj{Gs&XR*Z>EA#r&=Am?1st#l4P zPud=(&~4}u=)3f1b!5yoMXTKG$YcAvtOoMIKi#B;KSxXmO>?k2up;em2LRb?)oqODi9==*OZ){-)O;Q<^y5m(Y6v$ z6N*++7VowD9#5$?eh`dHkfpiYnn%T?!bw-)Xiyb-0$Hdo=rYg19Spv&kYE3W3(0J< zrj&=S&@(>;sf<4&h;__~y@~(aTw>Og5Xd#P4iXA=K)veOFIvhT3|1Cm6WraQ$^wk= zGG9a4?ma?HyJ8I*<9g})9EBg$#Hy{i>)-?S(zZxq=dWDY!{*}Bi_NRIk&4tTrh3wO z{nB03cs$_{sZ-aiy?5Q^X-}=j$lHSt!9P}w1s+CtVnT)fgGHPHKa&C+krr*jEQ}UP z-~G

Yf+vXfZ5xTYN z4D+pgT|o~_37Az*TC}MOAyJ+XbdR>0q*`iSc|h&rT{A$d*)wblhvXk)+aKMS-d6Be zzByLuz|@eI>iBu03YGi|ijpeY6@3_|UZ3{we#WYrjiP}%fnNIwUqVP}fBe)`C&(vu z-|+&#<$k_ua<3Gc1-=!ywqa%I{A(c~e&F&;nLf^hd%qB>rRI==#3E=MsB+6t#nb0z zP}UfOgb?4mcfNyx4vijOR23YhX?C!c8OFARsxd&*cBl9KSWc4~E6+*&5|0xRjE#f6 zuyp?z{G=#Y70Zl%T_>Tj-DqZsi(}f=yZ|U-hk$G-2W?~&HZb^1IUgc${F2^c^C{N{ z)>pAFQ~oS+r?yo@w1aN(65oFKP#d*|)*i{x{< z1D~$qDbk9OH z-vz*d0E@8enT>1m2&5cAk;8K~OIF?R?CGtEXDx&90C&#p-1CV+qieJ?|H0Tv!{j19 zUh9$XTqU&$us^LHu>8Ii`9_)0(^9kQ_<6d~1 z`X3BELd@%T^?!a3dY&L<36ei4J;q(oxpK$_V7O*8jhB@|PxO^f`uh0fA*+7Mm|IEW zuE+A>&tZ$sd8rzFZLDuafZqvf%I(wRsbBUD0{=`7xs+?RF90_$0B*qW>A~;JM-_@f zo)3H5pjkdNl!kuU_j}g22!`s3ohSxh4hy}b;XdZ<+k-dDZ|`p(J!`sTqg+r@UR{93ZTb&IjTHnF~0kV(LFuphz*B81z~KBg>Ibt(drCO?AhhY+;?4d+u*`uSfIwDPlY4nek#%)@Fl1^DPkotZ~x|QT=!+bjz7p{oLF}Q#tlwLtS2|Dya6iS#44Q!%q9_k z&3|gY`(m#@sg{RMFu}@X)4QX6TLvgD_9a2n{oKo$AW=loiy2P?0(S-SM@tDypQ zy$I>s$B{y7u7&*n#!hy3(-auKUI5tAmpLptc+#Oh8#G#llt9mqr|cG$z@$$LMTSwfA_Y zP57>I8(yKC<)xnR7RvuU$zs}YN69B(boo!85sH`n^D?g`;X z8$4@Mx^L&B6o?%I{!Z!E)m}Nl@xqqNKWN+FoFxZRuum;cc`P-r-~Ih~+7dVY{4`QY zXt^_m%mtpm04%0^yOVPqj6V#;-rf;sXzuK}<}qgy3T5eSlg%whOvIloF0>6e9(z`P zQV>zbb`%$H>UFs{Je#5%w+$VXOx{+2catRDF1G*iG%~BXo>t3-)&4s_Jj1@IzEuY}p(QXx2 zPmN3J5a!$(X^x8#USYFFj{|QXrenUgijPobnCSJbwDEQ`vvyAP6j+i)R@-M_d+Qh; z(>6gyKzQ)|y;WOp5IlyA2h3)z2b>oGiA|df6kZi4gho`$;2wcG$p&)5!3v9i&ACKu zuhC1jrew!9_VEj+9th20V77#6YmXIx3tAe+@2E>4()&4DU-jTnz}Gc9c0`t! z!RwhB4s?yjDsEDbMuw=^L`B-e*43;~+hLgC0rD&8x6Nl|G3SI+zv!RVmJ;H#n?KdQ z9E>wqlNii5r{&(&aoa`njR3{f++NlZ;GHI#v|)SAQu&h$x?n!(#!v45?ar2=Mp_$_ z1@bXJj(wR6*Qs^Olw%j7D!k4wEv(9gn`8XLe)hPC`fl{m3o6?n+0FCMo{c=onecLx zn_E+{>)@eU{T08f)Z94beK-(`aq&wIeKIU6Ij1(r(Z{io#A zpB%p)Q@%kNnUMEg&-7ffGI%0J0{OPj(^Gn`?&hMMfvSa z6BJDA`1EI5r}#}%hfe>q<+inT_ePsfj*9YrQgu`{#R}i4+dbW&JB!+uWcvrTbr)nd zR=B+qUyZlGZ@>MR8!SJD>-zABsn7P~e%i^KnsBP3jpx(+6QhP2P0xl^0oiqL07C_B z;9m8B^-9OSOD%pjVB$<3q2%zUw4td=5@RDvGXQ! zS3O6IIM0kqmrUJU^P+BCMWqHehK(?yaACI>BPPEI<=oH9#PApg@rpZ@&!{)ohbRf} zDH3riGo>2kLbm;k-2uk4>itUK68zj+X5oUB^MwB$jYjLgW=tK5T9?>P_~L5*{u=kW zUY2m6_i3NQIlsx;OsB-n5SWD z0R22x`be0l2f;+u1SD0kneXc6NC{t`pPg8bUNx;&7-&KTFw!bPF8~7j6^=^*2h5s$ zewyM<;Q_*GdfRV;-xDL$Fh`u!g~adE8XkCn6iqg1N%w@hamyEgNo3o3x=-5kbZtKjiSWH-9d8E;3w5GQ z-dcqsUb_GIRrT$Tl1fadLu)qqBSp!K#6&IB{h)oOE%k7}=yJ>QGMxp<1kEeLMM_>% zE7Qn_-;MJ9X9GS#_~99-#h(itd_<~21~hAX(9rR{kI%N9uymg1O&R5#TTrFJ;$OQI zwotbq$Ba<6E{RU|vFY7n&j86RJFJ1^KImks=L5_#lqogH%U17AAay(uQ;Y*QfK*%+ zjd*eb(UBF;&N4NXrtTG zmR-&~_I)7cu98a1T~E(@cFM;bkuFCH=dvrH#)GAL-Up?mnuubT6NCOn-J=*8cB>=w zR9@e%u@}s52^RJpKGK@b7-cln zy%*vuV^nc=kBxk=W5Yg0l9;1<8eh{i!s;X2kk}H!f)8BQ)jmES=1SI>s&4bEm+g@U zUj~V*34I|Tcc=>{SbG8~1rgW&>MnM~F$r0=D)%UsgxB$AEEfM+UqiCB9ZYerE2lWZ zr=9obXElc5M`+*PTpv;RGXv5WNOKGa(KmXsT>_xAnYLnJjOe1fcjrsathdt&F;{{3KR z=}ab~H?3)IF6CIEZ_T%cQ-5I#sV5iOSwHnr_`-|znOFpN+}B&G^c*6;CYLL!L%%^y zOh@0Ywz3;xrQ0G*W;mXZ6v($id5IAwjaaGD=+bq^ruxi(X@&=GUg!s*a>y|Y25xW~ zW(T%t!?>MLmq|qt8)llqjpL@J9F)6InV}BBTtw%hU>{;?4iT(t%XQoWeIU-56!zL` zRk>GOZ=^yzAu(zrY+&~QSPR0d!|2e9aH4?!$JI@w@N(VO z^+r+4P>w|q4zm2%btyqf%<6qno>P5{aITz9t|OkOF_tQxA1Sw%^C-U~Cv?)X9sRX% z;JQlX&bx^eL6;O969^GzRydm;J8^NKrfK+vC8-@v{cWFTDDNN<9e_WFr@5cTjb!Z&%Q>jf-Jv(>mOSB&wpLv+~%RJJ%kz zuD6e$@uV7ZdK+RNjON_L0@$tr-uU06gn%BCH#+WEm# zYMm5FI4rT2Z)j}Af5$RX2z>GH`N)6tewI?%iHE8qhk0|HuB^bOE%gh4bu+Zr*J4e+ zEMaJ;nKQVy_3c}m!FXv1h8b!5I)Zz|cpPqG0o@9P^hD{9PYz&}#;5;lLm zqQ)9`9|$hSMDN^xj*RG%7Z|AKxFL7;S2Y094IUeHt8S5#MjrcyHS0+HwWa$zbux=P zHBn}&H}s11tx1R+u-u{->o1!FY{rZ`jA4@7;+~CE35+!*Q;jiqOYZ!do2hf@e%4AJ z5H9b>d9W4?G=yPtI;<*IGhEspGO)RqY78yiQ`T&dEBAQvJ3sL08>Xkye@^izf6?zw^Ez!=TTWC zJ285B&Y}as5^O|fJ`oc;D7stb&uu#Ik*9>Gmnv~raLamIp1yR$R^p|9hu^jnBqhg1kYEeQI_ zN(*gNtBx8N2{2HozS;KmjpoCzigrhXs=>py=P#gGwFI>~f^}m7_@l>;1N?S}G| zu4S>W^ZbOCRUtlW_K%X=y4y(-8m+dB3lEJLek$iGlc}iZqevh3DMTkDB3afa{U7)2 zO9sj(Dj9{uII}c>?m$er>rwHwaQ8bH{u*)dr7CIt`(z$f5dPt|)!}l+(U#{}`WwfY zM%A3oui_)zF$1Kxx$;+|-SOpyqw2ej$ouEfU38&tWRJmR-qd=IFxv~p1sD~ zSDXfS97r%C-o4ygD~M#QqU=ixdq?iq13Skt#fKL}axDvIPyRfS;MigBaP|J&vWglKW6*ev)nwm-98ExJYG>vb247hOez<6=4KagXX&f# zW(K^yb&yyLcFVe#XwYEBL1A2@Oh&)!3;6cL2N>BEEb$Y30F8KAH71&>J3tcheDbeO z!s|7)q*#w%E4(2w_GdsgecC^)np(cjTI%|`jg*N#`Y##8znj|S)p!y?Ey#^RN+>{O zU1g=w>UY=dWcz#{*s|?eM=sHWe%V;RxM4%Jf!!VhbetK5iq=G}s5e@CG-mxYK8_@UGO#=l`#@ouIP<7zF4={;5wR;%hLndK_tc|Q6}omKW`r{kr@7)Q9) zrXpAI7E7R2y4BRo^lbmKZlzrOOPcs55ALJ^XrmC9OuNUVb33SNOQOyGc3}I@3qYwX zWrGvzUkI@`#kvjyzj+@#I4?xFu7(KB7hW~s#AR`r%1f-;&snqsQ9}`#SCVVn*DQXP zMqZm_dw;6Dc{M$;iTC~mfZ=5&y1EQ3juZJKZCs+1{(cBQjuR>v`Gj%FH-26QnC?5db_qavL}ebBng zihYZFZneagN+xPPK7IKy5rNd}Mk$5farw0ksu76Th1kY%fD!lFhp(`{3E!mqxN`}Z z8SvxpBelK$iSOxU6!qHB*<;;uk}>{u1}gB3#vN{TPy*vc{K0-C84+G?E!>fc{`X;6 z7p7386uy5=seWY!sC7njjA>%mL%Uy@hYf*rnwo%n{92Tnean<oab@mOI9E2S)W}dH;fIDJm7AnX8;e zrTzZWA%{Q1Hng6+ob9>N$^mq+fo^sDvH4BT8?pF1VI0Z>ktJu!tT_Il_fq$DAE`Y8 zeNB_X>?|!QnXkO5tF9WW_!l)F^k0W}r06A`$Fi=)+vwNeUa_=2WAbO$RnvlmW!U}Z zT^^M|ohjEqyD`u`8Th2dQlIS_WuoCgx&ntnHE(Z{wo9@^%kCRYX>Cl)`3e?mxtru3 zDj?~geYyldD#X2fkA2d9p5=f|eb~zd9_Pz(pI{%J{%G!tjPX-YCc}?%9-Uq70`Hix zY58LiykDC8VKbi84N;~9v2lb21!VEVeq74n_C|~pwV>1XH5Qr!Kv1Bk@XGn0BPDg^ zqdjfKlt<+i5i)1FdReq$n}fqj?+T5HGS?=K>bRl(>O1--X!9Lc zgu;KcGv&cG*}+Z(P#A8w%vX+mdf{1^&%s?q(HDICNaH8rG;GFlgp9t z^nKFQOO#$(v*pS?)uDou6Ua%vckYc(aGERU`S>*q;FEI+BS zdJW!wao?p<>&&+^mQwEb@gN4!iHEL89RRzgWP&XT{>bAwJcit?Y2y<*)(VVxLG1yz zx8_x@MHo1a*?RLIzn2al@tW#jtf$#Tmj^#2cv&Hso2^?ku}XKtg?_N1?*gXs|4K1_q<&%trC$0JO7KMQI?)B&mKy zw->iaVMR7({?O5?otHhsQj1!ahi1ycP=p(Y%?-Zy4-#&?^a9ZQlidZg zbOu6`bw&Mr^=}owV%^ZtOWz(a8Qn3nx*weW^__e&^}(|fsFp4Kr^xKX_yOMQUKR$e z4U5jrFP7=G|L$RRtEw8oKb9GT_vI+!-ssV5P326j;}V(@kXZfpaLDgxZHk<>0<3@| zfub$hISZ5OA=g%H@fP!hF}^5(KkEg%SS^(*fH{Vfn;}Oah6O6vW?-q6(pJ7#^FgsW zdoj;!)z04EkCr#>*cNE3o>bNCeVim_OEp`@nFFVL2hW0J-6Vq+uDY18Yxm#pV%L_> z?PH})zC>F%k9;#UWjrYf^~hFR)RX%lKDD(FVQU?5&P8lzh+FuTkNoPMH9^O2r+=#xdBW-f`Q}S}(4+0O&>s3pJ!h+g065@Js+*vYHC!WtHPaFqT#+${cSI zbh5a}nZd70e^q4rb5)AWnf-cBeN3RLJQU%Z@>Qz^fAcC@&P&l>xRn1hLBZPe6;*v5 ze-a&ajlIF1{=x@>RBnOFK789ojxvx!55h4nD*NqX%MxjkGIW|!i_3ZW4CyjCDN>kx zlv|LB_exLzDjeOOGQxb~yrO?Ns2dzAW+VZD^mWTokvo&g1^k<2cJ*+_*4ffWYnivx z+4zpY$Y+(rN5KXoyO@EYZ?b4E_06bO`<-{Oyk%?V30Al|P*BG3QV^_$7{z!pA>4qz z0*loLql6uaVZJt36`y!6x46H7GV2}EvotxdO*2`1gWW{*COtkdih3{OR2%Yk>0Ddo zoZVSabXvh8#Hsl=6VJOpy^}BEF|LVlmuZ6hWZllf;z>io^71&+P4q#4bov*QWZaN4 zz>uHSZPDx2yP4@8(2l(=2N#CiIC*f%)l)>5aP5gCH6qxyh448Y35CmEapTvnfYBWm zjp}=j7JtcaY2prb7}`+;=afo{aHi1Tj}_Mbx?c#EMRQotE=)J>MOr_@zb;-j-61Kt zv~na&%u6=8SD%)8duTEUgPYdf8eD%qAh*Q@PeYV~RebOMeW{i5M6i;p^S^`b%>frg zn0DbDxxfj6LN#H#xW&|q81J7~@~VUxUHUa|VmzBLhO zSSw_V`1jdmMmuN6p^|gpiE^RXxvGmXfuv6m_o(7}sb60!u5J5`R&G%K1UU?wY%*vZ z2RU5;zRE*7vy9>HIc8cVzpwY-Yu>+V&%0**W9HNC-9Ycty7N~;F9^|{2G@U1OFg-v z5CD=X4>>xgpP!nBwP}$Iux=*q|8?+uQ>>$=o)6kI-7i0>a3}4bc1doi_3h-2bZA*m zXtowcQKmw#Q(HU%3cdbPjs2%5zBV^IU_5^qlC_=vD+vi+FTSqE*j38X079W8Yj?2~ zff(A8n!Y{q;{<9u5P7b>xljWlF)MSyUfGCs96Ow-jfi)}OiVQT_J=wNR5duF_2zH- zBcmu17N^EJ%Xa6Y3&D0Px5xREh&iWrt$}#Y-bhx6j`fP-ep)f_?qu6^b%bW=$TZaCZA7%&gjJcu z`ieDiXP3>iM__xS^wunxast`LMs4>NVk#__G2s^3=PL1g{k9hXdop-9Uyo-+Ct?BN za7Q@D zrw3km{6usAKRG@28S?L54Sdk&j~OPJB%v|fB0=VBcAfn| zdUsin+a}eQWsi!3I5>;kpPN+~Iu_WiC1VDheSTUhOQdXYQe?~W zgQmFN5Pc_4>?EPls8gEd zDhRQ7*_}G(ip*QT0055XHKr?T^%AAM zG53Sg?0>r$q!_N*#4gm#r{dx5*0f`*!!_C)B#XR?Trj6sck3&M$CfuFvS83$ssmnU zFqNZn^)Xh|bz1B;(XT_@;mV^f+*CvSjg2>=!}Y!#AKA8U>7KubT0^LoP=5#tRjqOx4$-`?FnGl1&M;)d8Yy z(A@gt-wY<{-)!f)f<)au(=Z7{6kBFsn?v(a5zP0s^SL+{-TfkrCATMT&uZ^uGgINF zKVwT4g7deuvhyrw(1BI{alEgGy`jhN$8fq|nZ<01HxU)`{uS^NE$w;#XGuPY#z6Fr z)zV)I5Nh~#80R*9Cif_Or| z1e^=D=E1LJmTgvTF>jCB|9g!#K4Y}D!CNfJd_G)=wq_V}%^~#rGlHcP zEb==i5;~eSuFo1verOH}=?XN#4RC0$faw1wT)hB1M@vjT5@wWSdViS3hTfox=iQ?cv&fp$2^;be3&;c zRoh8j7l=TnN-a#y%ev*kbqOMwqmCYN>w0PP?-(B3Y5jn|&Neqw$OfYBzkp3%F&bG( zO&?4f(Gz~qC(UWMa&|1WmPquIc8JfYdU`7vB=dk1Sl}6z!OfhhrL}EP`PTk*=|AOQ zP!%QjwqjqO9Q@=118I`spLN~24AcSB#{C!~XqGmw_BUUJ)0T#H@%9D4 zd}|qBUfM$s!pD$W7*;G%UDXuKkn)s6Dg9)B866iY<)~uv-4boTcZh@ciPL=ckqQt6*$&9BR^X;* z?3h>hw1>tKUp4s(j0XZz$?HM=d+*)P_bxZ@p#$%Hj|Y9F$p}rd zpwN9+L`}!0!WTx$$Km#m{ym`7BQw##j5c#N@!Lc5dYD5s>L`9|qmLRJBtL$xJ+)9G z6k3J>|D!#S!>}GO#M;8TL9DPzzw<(Ju%>l1wPQM{KY+QyuC9BL8b2+Y#;{w3XhZL0`tJ?A1D}Blo{TwvKX{0TaykMxw8C%5}=u@D<;;-bAGZ!t-Y$H8JR|O3JTk-Qi+)FHzo9UVS)=;Doi=Qwp=12 zAz|n*VhJx__CW+IF|3chtt2;Xyt8)jNH${0TzNsOM8Kz4VjKbkEgHS?#JBFBDAyZ; z+;O;<6Qq%kV1E8;bc_A~_M%ST8bBE-Ru#x-7aieYS>cE6NetgA0p z&9`REbv15iM{Dar2v+ zHTTHUBAyGap}6OyuyIn6xzl}8hXaUnr;am|(kbhL`~XkB6JvteRB?T3L8ANLODhYA z*GVg_VhxlLf)bS}n(mr%_vwrmueBv2MKK;@PVnYb`Tq7gN!vJ(l~YD!M)@hZEYoDE zu3cJWAJ#6zJy*DgoWgm4URXbL3|LoCvvnF)xBk?!K%u+0^n=1?K~lm`TdA?U99M+8=}ykwAikr?-O2ICgIQt>l-n*N<2pe zu27`rgH;K^)=c<`>>|xC7OdvBnE3}DpuHT1B_ouhl4B>3DU{KCdpaz?rnoUF`Pq#R zzXgNS2cecn=wK|@xq;<VI?~S}j%o=nZGpj&1ym9ERy5-fhB(!ViV~VlvVHV)wI$qbu-sPS#~& z@e+cLOTE)SEo2a0p#FhJV{+Z4By-Ym&IS9WnQc+&UU#5{8L@F$`|s{@kQ1Bqjn%uN z*UZ#8Ggih-GF8~Fzt0GoZ5#tCT5y~jY##3u_c~4tghf2voz|WX_D&v1{GsIv0Vn<% zW6F1+Lp86iJq^SLoU3hE-xwp==gdJ%JF(VWt>J^OYNH;rP1uPg%U$odRFAV9%v{2R z&s*CP#&;rsq9Oq(bdR~;K=>M%lxn*ZV>wtMUbIpqfWmJ zasUXkqLq`|ki7sX*X&qx;T=EC1^4OQwbId8{gLW@0f;k_r<$CBXo+7m!54rVOvm^u z`48(v_*vHYZe>2b4X!W?lP}|Q z?tq%6V!VJ2A)qUV9J{3<4n)|BdxR%35Xj|xe7qfqb)nWuVBd#wKpvg@jU=)W`w%vm zcZtfTUQ7RJM(lVQrzG!~K9IaGcVpIcg^mgmC1?IoJmeDU&5bfP+2r>f%Y9sEy9VTT zezsY#Ce>ac|JmBgxu`lzx;DZ)WF;|6h-!tzNun7#a+=g7D74iI&IND(P*W#U20V;G zza7I?2(M&7_{LUFh0e}^%f~a`p&;kAmcZ0lj>3>Z7pO~=0uu#jUOxK~K7ShVR~BhZ zbS@Mi2`}8L+q*VXqXgXudJDWw!JKiY`f^s1-r!wJTJ%NVZyz2sVnY^f!W&;FEze2s zgV>$qzbB9EnLbDZi5q_YbE${)v3zSyl8e7NW`eu0`ZXGZsHQ|TEf+@GwK+i4j9Ha5 zTyQWjFya`&2y=mW>$X1RP4AEqgULMVIRL&FCv!J^=37ds*kD zJrs#bn;43u^}-Bx7B=IJ<3$~Uum_raO?xi^5I+m5oZCkOsivQ z2BRt^96-_PR+AN7=U3saaTkDaI(&aisA90h3MK0S@Z3|K^i(FCmV`xyQ**8R9U!?r zM@mvC;S3?eX2$V(e?|7B1f%fUquI6?nO{i$*es5wXrFXEeKv}rCba~?XXlrp#*b(G z(R<-S`w0xw3=4 z8jQwEw6mi=RWEQtZM&u;zDixMC@()eukYMw=lK%wQtiX-*QTBpY6kx%2Mf@XR1>g5 zmQZDUq=BaMDw`7eFBq%%ghN%*H82g2VFQ2bSl(lp?|W=2;>P}ROC{!RCi;qP{c?*k zdIcHw0BT2>h=fg^D=v^!ef7(4mmO@Wg}_Z4?ZMY(PQDY;Fk`DJMtHYuPS+gc+DwxN z+`+YIE}?kp-WzQRoqpTH-dHlb8;sC9Fp6>l(oHu$k2QRO%DgW^;n`lU3GoImdRmonAH)V%+VQzKZPd+|t zo4L;YD)MHB&}7KqxIxGAL@c!*FmD&#l{vk;Ppe2uO0x8o6V+(ZE(|MfQX4-!u&zs1 z?BR0*U|{9eqpiTbSaXWz%`&ftuOqg=AT1(l@tm<%$y_<3%3=UPzKj$=gXqi;$W;uA zu%dJ-@R6@3xrUV-?^javGiNO8$qmeJeYAYXB&H8N#t-~3$ zeOV%)dCyrw)Uq> zq!djq0KCa97XX_S#Mv7m(FL-3i0XY4Wd`D_v01nL6KbyV%B2Q90}ANuMo!e>lO3HC zq1g)ndW5=vCR5QWb#L2RDpy*7Y87Mt82N;XC1z#u5yOasgx_w5_-+ihWML@dZG^Uu%_q9d+IUEF91K4 z?i<$}Tj#0!SdOArx%?0H{xLbKP!IWwO#R=^!>R#B6m0Q~VX<3iv?29-pasyOT5DSJT3;oAoz)k%^ zt;c1&m1TZxpaqhk6^NpoOLEF7ut*fa{8VJQ0Pu*1G8LXbg!OEasQ0Kci-DXLB{Z5# zk`QbHmVoY{R+in*h}0KE#eA9X0$|3nq{_5DI%`OL^_i@`L}B6s5l$J#V9%)EqL;1n z?sQRFfoE$A{rV6}A>}IBDW@@?q~>S1q32I&P)@pX0oX#ZKQ1mL85z|HJ1-E9QG3V( frcToH3&6L1>I1t^sn)lIB2S#oemm`AE~fqmv?3Qy literal 0 HcmV?d00001 diff --git a/utils/external/SSD.TensorFlow/demo/demo3.jpg b/utils/external/SSD.TensorFlow/demo/demo3.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d486a47fde1a9ba826f3db372f3589d61bd20567 GIT binary patch literal 35044 zcmbTdbx>SS6eceLvVL@x53?E1|59) z?c3T{_5OOh=XO`!Tethx`EKhu=j-Q%=XJnqB{@Yo01^@s;M>aucwPd?0A3*>{g?ii zAfqDxCuk@r$f)RO=;;4t3`}fH3=Av`baX6SEG%rC7eU9w!^g$J`|tk0kNj8tzgjN| z2Lm1BzY_mv^4tL+zyS0ku^}VT0bUUxArl}y_W)=C03_5GZU0mFe+kknWE51i7nQKE zUkcQ|eo-G8`PGZEs3<5erF~xB15gN13Ey!`p%JN@pwl_N;rSJpg+VV}*-5N131i?j zbqc`5A|WLsr+ClE^nsa$k6%DgNLWPXldPP)f}+w_O)YI5T|Ip>a|=r=Ya3f<7gx8R z?jD|jLBS!RVc`+r_=LnH$e-kt?3~=Z{DQ)w;;QPJ+PeCN#-^_Bp5DIxfx)4v>6zKN z`GrO3#^%=c&hFm+!6E$o;_~YH2621$A1)*S^8W+te zSIGW1u>Thq6o7+_^wN091OQ3Eqr?U8gJe33N(ZGMOPE6AF-Wk-S7Jhp(Bug^cj-fW zA~o{r?poAOh*8jPRqoiHy#FTHM>7}PEGD7DWNzotHgmWQ-0Ju5e;8ZECp&ru5T!ai zajvHMN+3|40V5~t@HNRLfwfTh&B@W%lXuHTV}>#R7~v;U^$V1ky@^`%!1I!Ra#~e5YdgSA1q>6rK&`bqKHBC#kCKZ5$M5c%5 zbf~y2HyxI+zGt}_9fS)HBiCnIDaY;u*_-05Uz-rFVUBwJ@7AUx#3~=5!hs2PDC=KaYA^o=8!Np$sj;^fVI>m3f8&hKjCb)kty;j{ zc@&|@O>vHKlIBv3r~aYvBp+C4mWTls23W8-mEud|etsvnxBB-HU z*3lS3{Q;?_lM-Lai$lbFmP!y)ES`nSt=^9vzOXKM{`QHl9H3d4Uu98j+8iRH?R~dW zO7Wl%{M`OMW2IbO5GPRsUA6TBq2&zx!Ggax;Q z^suA4va{!CAUXed}vK6T` zza62Xy>y~oIr6sld@&okGv>C=l7tf75+Ln9e|<37!s$_{zqGJy=o7FLa9vt{@?=&{`U-DWgxhp$jd|L577R+@R8c+Dw}Envv6h^}0b*S^;yeG0-* z4D>wjQ+kok*`Nby+asNM)q*U!XkATJbxS_#@{j?XX05CQVp=CcSKq%@fNxPjy?uXhFA=0L(igf$cErPSd!-09~#f-wzL8Qny+m!QP4+sU(J!Ut3H*O#UU=FZ`t~OiuL9@$LJ0 z^tiVjy&fL5whb5;rZ$ReCJ_m)Qv?181u%v}Olo+Gm}ux4r-8N5i;6f?1ImHg1+r%h6$*R+x8`IpC6knl#yyM0a zo0?J#qgNd>hT2ooQptOmm4o!PCXW2of!#Y#9&UUCisc6kRI#?3?U`J!;mxjNT;&G4c2l@L*6Qa*BiucW~BXd zj~&*X9tI1awdo9eamrm!{O0*BgG*f80RJ_cQj5(KBcZmNq>}`a?<5TctfK4e&7saJ zUE?f$C7fu~$Iq*I{I?;hMcfTbhEfC0P--1*qAE$#8inti1TE~dezzKr;$x^Yeu_HY zQi-0Sp9iV1iR2pEtWH{@WVj2eC zezhUHV;gqbdos!ILk`bb!)|m}aHFYI$iMZP%P|Pu$}JOOf2auLQwzC~Ac75usAl11 z7Rxt;zi(ydvg;k_6kfXTLviisb#6oEaVP0|26VPB)!p!b?DlMbe$8Z5X?)v_eWo2p z09%qe?16pcf8&0UtPGV~TF|NzsRYl`7K@0?41$nYhNs`xcI~;G;cVh5OS}T0l zB$)rfO9d$;!^Ol4^;*!ddsBFcSOxE@aBd0w$Gt>V2-jvldkx?-FR!61Nm``4&rlii;P@Qka8hGC9`+pm6(k_+x7 z(BU(QHSKO~od}6{)@_)xEF(axC_ACyeR@Mu5%p& z2fHhGh+LH?n_T;m^39jvH2+<*e2XS%{*8*aT22dTIqR2Q4vRPKbtsDq;hw3IDj(i9oL#I~p~rv}e4!8*qOdBRPhC~b zQf+lgG$2GP?~Z-F4l|B)J$rp{egst-%`u>hrP~^#q*iyrn^3|;L45a_z(huC%a8F2 zEL_%-bzPH*T^O%Cd?IMPcL@D%4jtar?|Ogx_fMymAoDPXvahU3VAVcbfIDdl|AN zYd#Yg)?n>I_)6A%>&*yO5WkX=qX?~EVP z06Imtv-lH}_V(}n@st=(vRS#=w^74qZpEL(1#XUqUaAqPiZ~ujj_Ogb`ceG>^SplWi!ZRu zK5-(=>B|F?`sx|FxpyfGw-CZXO4d)W2>)hs2VAs#WBo4F2?xRHd*-7l)N8&jpFE8_ z1}3!09|`v3FMU;m3C0r1*h!hz#S9%%-ma&tco#kN69l%u2ZF<-)pxq5<`bt3F(^?!P0aeb0nu5)3?In) zNO!0kwr^`2s&Y%lxOEf{DrBs8xD(!_j~#AZ{|zucZk5rRqflY5$XG=(9hf#?!=SNp z{hAoXmiqIXf49F|rLM2Iw~9lgU-dJf2m2X-+g)O)SNG?LF!ysOk;B<44-!ssL|)xC z4$fwqBRn)%g8WE=X5tDT<&cJD(p$E$rfRsrJyRp#bV#5+!c1Z@yH4lp?#E*P(+d>D za`T&ZS(yImxnyWN5xaSa!d7+LBr!iO*79=RSt>)h5ekj60*(XGdZJS`j=$4~;f9}> z^d3?k+u!?Z zHjHEF$`I?_^YNilLZ)s24_Vau{FL!Oi~g_{^m|(C6p5_7BYmCl{I+I-Zbu6S9qrr| zNO#|o(wWY{z&6!-#NLDWJ;UOID?$q{*+an@#@D=t@69{8^re-e4&qg?YzQ61=HXM9 zXUM~pa51;~urFgq*Y@kNt_28hmWI!kwMU#3}WhSQt91o zEw;%%{6L-RRBVt|#G)XMm1g07eIOs1OMbs5b$2 zCR5u{9$)hc{^UB}2pI7A%ef<0wDTicRjGSw@0(_Jrnh1bC$Eg5n0?b4_sPMV#&|&Y z{yt}25UEld@r ztX_58fFvD!DzYd3v7qcM*wtb2qW7zTJ4m}4k@{lEQ>XVumwbXmI$Wa#k|y;bh25~0 zYJ>}L-}*2A#hrEGjBK(`FX*SKr>W$>L|R->qL<#TQ&iMaU4xu%$Nb%G74blG3=J}( ztQ+io2AuTcI0|W}rp#QbA^DE1;kWQ^nb${bY0ZG8-mnld^n{H7hF`h;J5PHpEE*{PT;YBf6h~pcy5Wcj`&>ph)c_w@r#J%I@8u?=eYF%=f3-fM$W1MVH zgY0bvk5lKMsVW}Y+<2a+Ks_vQeYM$knV~JL>#6+4`108+dO4XKLaq+nilX%dRVr%|=N+$k8)6hj&4wws>|JtGp>P{t z0q$$wc3#`nfBz_UJrh5{~Mj9oM|h?ma}?gyv`qp@jpmtg3CiZWRCNcDQkVG z@V=0^W$P+=GyX2qlXup7@453gTQTJh6+4bGjtm{5Tjz6 zFy7?Am{LD{O=q^I7Ub645MA11u^E;9?i~8*hj8aNfq?B-9d`j{BF}&T{*ZrT^*&~} ztwsKPcZD&^a)9E{FRk)qOfhJp@sqZ7^>O_-;bU&d!LRHuk<)piNI^h zs)QL_y^#m{6M^Z97iDV7i$jOpkY%d}3HZruNZGD|JfGvj$yZi7?+ekU1|3SzU^yWJ zj&XlW2;`CCr6{YXY#4#O`M8?6bp%w%ZAw{v2B==J-&8Bq9lp3_Md{UBorHEK zGKxXHzCwshEky99a8}Fwn?of-MgcmhMM6?Mp0UYl*dnFoE6?qFxq4XsjV_!j(=1wd zbuk{0Xhe4N%DW@?$`uEg_CroyY1j2^5nXrYa$tATBw$xL@>7IOk~S-z`WG`%J%8qR zQt91U;*mKo;9q{aVkcHzKx)fNM6I4y{f^U__j#Xa%|EX(6g3`W_1&DdJc zJSJ?zMR;u~x9Z>Mw8NPqU6NZksmn?+*?Xc+s5^k>Usizw|2{^2<@oB!IOo7^u!hqj zMFVkgD6kedvhp);Bpk@l6beej6QWwbBBN|3X1H^YQo&Ls7F3fd=X2zWvR^AO8*8zF z@7Rlxdpx#oMtb?-*a#A~Od2gyhGaSyC#ly2UJ&_r<47T+^B?D2~e2OG1xjPZW-`N6`>&KI_KM#opw^gR@P zS@|GmBWu6!(+{H74&_O!Q<5wFyTFgBfLv@dv7 zc({J$r~kn*iwj+%q*?cZ18>&I7!dsT91oumQTBYJ@=Ep7-yirLb5~C4mCx#I^_^6l zv6F69+P?g?_%Jn2QTas}yrob(`!vx0J@{c8)PV8cd*ZM(m-*iRA?aStXdsAYTDnvvR~D~1K7=&OW7Qrnnked>CeG-b6dN=k{ox2Mn*Ok z;pgiBrAu+vk#?KNXTWhf8eDI;;g&e!!+1R9TSA`oQ?_CS*1!hBkEUFr`=%B;3iVcd z3**G0Sy%>RtTOu*A!{&Al~0HDuMeRShY%tusxQ*VMPeE?sy!#jqLI#lX&Cc3swh81L{rwV`rsOwo_a;k>hw2 z6M9h~1>b3-S=Q^jfuzipy#rbi-I{FEigMu$d3uy$k{cOhIL6TI0W;Wbsc95AI9%>U z9r*JDS>KEV>v(Lr8N1;#6>df0>1>vjrs(weklcKZ0hZYwD%g2vJpJ z;KGIuvb`KRV#XZt(ql9G{>lfnsnlMmc;I0jysJL4sSiY!ByW3O@GD>h z>7BEI9i+ofS!wQ=ACwPVa!Wiu$yO;GqfdL&VAgCKupsK_FK~`oKEG+zbC2*A<=;u^ z&MSM$K*mIhF6+aF@6_{HeC!Ampm7IU1Og%cOFJ@lJ;d{?d7*R~M(HzAOyJjEU zVKfAtdWF?oT*UH9SS8PTra3Qo?M+;+En0Qnm9jfu2ii^x>=M}CxE<~RtgQV$5@_i&>#wdpe#U`X4{o5n?< zN-=zstOGER8-#;&d?g9>9G9BS3e}jp1o}y=F7jhAHa7p_XN+clNakY1#(W0I{;KOB zz(4{JT|D$vmiMLiqmf#o=9^+Q3EkLL0s}~WVSBrHm0ZPJKBhH7j;*r zfZC%H-Q4Qkkuj$bnAZN^yj3RjE-XN^wilVEGd1zU&*_97!7h(bIh zGv7c3#5TT3R<<3ZEJUN26E=VoLC7OR_nWNv6*f!6Ha|Y0x9appkdeyfs7Bv78gWz5 ziu8acb(a2X-8VlJG<ebsXktW9HNi;eaA*{N8rkwS*V^?)wFTQ^U1a&FAtE&MsB$v+-*?Tu= zLm44&f5U@KYEb!ygEn!*b)J?mn)fTz${xTX?j@bYzq2zG-#~BjY$p*K^NK2~V|Q2j zH_Egdf+t@$6(!Bmzjg{p;Mz$aX~PqY=%@Hero-K-7JX6d*+;_#3(2&zg41f& zPPNFo78;Mc7TL~g80z7&be1z*qzFaIhuLFickiCskmJcX7y8nGJej%;@z~A!nob+$ z;D;i?=`}PQGjBy&bG7Gg)P6b;0m=R@VWT++sBu3O;DikzLET5njV|)21>BM{l;$Mx zj7S~%MIz!0hSuaoe5m3NFC(KGGX@!#6sat3dZ{(6?d+~NgPZaK4lcs=Xk}6i@D!s; z>-@iM6J1!b2=_@H^~XK~ti=5`^kp}V(T*|*=2R=(98TFn9VKX zdCe7M?7EpL8DK91d4;IA$v4o0BhLe+skeS9gY_8@Mv>U8s~5TV4^$E^TNqA9^p1g5 zH0bskb-+-P!kws_)QEf1h0+iIa8)wI?mY*|boUeA%$a?_wse)p&5`{PPDB|?H-USi zrMz*%w(B?B9UkztP2Zgjv%HUIGyU*ri{vQRH-!FFG`m=FgS`JI3=s?#|b~kB|o)AJ2AJ+tmrX-7a_q zw=dAitWAU7z5L72@f(0r!Z%F2<=Zl#hj6a1fU+*Q{NtNNg6)2L`NtFffTSrDw6P^Kt-hK5^3LnWxKkG7gp`ipQ zY;O}CDAf^yZ&(#QXVk7F1Q%u?y$%m3my9najGMO@T=FG+f&*Q6n$gEX(A?G|l9YJxL-QRqLY>ivTx=EW9?3vPd6e@#bxH<$gsU&<^#a;G}!Mn$gYexMCX@VQxyr1iY zBYO)OPlmFXB_koO3>mG2?Vwh?wyh_OMELZ^L(+VTTKE&isv&lv#v~nFWp_R~jlqv$ zE-HwGKIAPGKvk8s648Ax@-%JyT1_-F5Nw&D_t&KX z%*i?Y6GHE7r)8XKO?JB`Q}@ze12E_(g;@>h z?xFBR2_3K{O6@<+gOs!GN5LM6w+(?kGz1fzSK~E$92M_(TfOHPlpQ@DbBg+TR|e@7 zsq6}*r-)dn!2^{F63%-qcOmceTgirJIVWiqBVfv;LADFMHioRSAP`eQ%|2A^9ttw+ z^L=~h%ZHFH@}Oai?WPDy)hA~!PN z8w1boYje-$xEySVWsQ9Cy9>$c0t|e zCsr4rN07VaIYAnku%Lg#^iuT2$$_8a&Sb$DoeTq(rR<*zh_zGP%~NgoM}1c6#E#dx z(tKY;9T+7LZ+;{VM z4AA%7NnjjA=Qgk`_Ok{qF$sxIcuOne8i~uDDZn_&-$o}aC_205FR3c={HlZM!rw9^ z3RCJbo^(*$Q!EpKRSY+t4ip}wV|*g>%P2DAC87}F_byPx(#Cf;eWQZ%eP6n%U5M7C zE7@gasX@B*+)Yc^J-)I zz7ZJEago(~SIaerYrKz&-wE{#xEwC|^(HdXWBU(tcu3O;gV5Oe-#)F2!~wgtL&P`C z(Gtd465WSkhu3O&8?7b{|K9h>OAq{|0qo0HiqU+RN3zhe>5FB{j-Q&EKdTlwa1`Lc zPhLEje?m4+_PRcS@k9FkFb{4dP~$f)Y?{3%pP(R8DDU>B%>~bfaN$}%bpl-K+l12&2=vKiP16lp!<|#96CVRcqR_x+Y zrxAHu&9paf0q8X%-f)J!wDi0E6*hH~<%ytj!jUQ>w7zOIik}J)!45`gd)2eNT%U`b@7OFdai60}q z=|P*ZF{YHUcn?wt){Uk6VNg1w;_(BgLm^1}8n$R~aE=-E zCH&#R6PDN>=E8|xp6dAaUhL@{&U58piRr?SRV^gE7s-@?=}hVtEy zeesF?U6@qP4f%&==T$h8HN7eOzV`Ry!EK2=1z8*tZAMF+R)^+RPdEza5cBW}weq+F;|hzEhdG4$61z*>`o8n#t(|{6 zalF}5=RkNcxzbJ7O)(Hluei$>9GHZT6eb1+lyi0)y@tKdul>6SqCtj@WqNplh?DXm z(5sVNH=S3zb`XM~#@~b2VV1#kGIYmYjENJUbcZj-`y?Ys{IDfh-9u$F#y5_2lImaR zA31!2kk=i8J_F?xrYD5If;p7U-nE(P9wnFyTS~AZ5r4>CPHl3?2iNEf8|@9ke`1+A zQL&EL&hDx7F+1hf?#}8vD3`6doLHvbW%jhwv~U*6x;ofZSo2EvGA(m%gpS$;ZDHLp8IzZ`55maBLo8kXTF@w}z561A zR~i;wUQyNy0^x@^Fm~8o-}z>jF|V#~*RAxUP@Qy8N=E)DJ&S$@;90U%)0f2WnUeOi&!#^}Cf0JE%h%q#1S&J6Na+r+rSve(m zkAIbhBmQ>+JQ#ywKM(XWVI|)3R)z9@G^wR}B_DW!@;Tk_pqNM=GBcIp3RlRCZT9@E zO6-jDPpEhFQ{9s}azE2kcpvm&cUy9>7Hv<4X}cMMj#tKPNP(fZ&fw7ybaW(zfzh9| z*tNBjE#x38&-!pj+ic!zhz-MAr21WpLCPOUV(Z#8?Hg-E?wLmM8gu;PBtFgCbur}M zv+AQ^=BWOkut-7wUhy;_$)P%VG952&;&MPgHOAV&&*&>f&K93d<;VkoVh(s?^+V*V$u`XHtYthA{~w+> z?)VWdYMfh=VfLxR(D{p?r6nG2;k_cp*2s}bh;wteeN)oq?SDcH9kv>mF5j$ON?yfNRiWE`?a`giu&?u>&^?X_+yuKtI29-lP& zM@Vr5Zjf@mI|ruhJqh|wrf|lzaqkP%UAHm4IP@9}h1j6ZP~h@FE}wTjUXWOXcLx<} zIE3C&lCvk-fQGQv(=Djc@Zs`d-0Bw`tapu-Xza;R#%c_daIMJN*B3k?FBXh;;P}qw zqCInbW>>LOTQ@m#+NKw~csj>V?vc3j{rgb%(BhY5;{MeK&8Y_vYVUEpia>T5+O2oa z8xSE~mw)e0fH5oiGNpgP+cbi$Yu|Tp((se3GA~`&Q-#AfXvCjd(R7wCRo}It2R{bN z;u@l?;-Um(zq^BUQdH=B1aYZjU%xYXUqv={eYA7_2dSPJNkN?J2k3&$7)K)*Rd`7M zeE(rt>lwg%CllIYJ#uInpQnr~$HHAwYQ}J)N|^BVaci=HwT!PVv~0hf^sI$4|KQ{r zG;r+FSYiL#@PL;OhhsBlY+IN)0L%a)N7-VNbK2vCTvywfc7f2ErngD4Cc4x4gX(^h zW%)13IVOg6qo}WM`xl@)^U`9+oNkJfK}*~Bi3A}G>IO}H%K?JSZ;ATk{bo#N z%<^I59)bvR5etT2sG#J9ho5tL?$y)!Y-YpA+j;*#o7OiW+kA#aPZP*y8 zhuayNFZ?96mwd)uVy}xrzt(~~En&-!r|~ZMd2H5myB8<w7rJZJ?9ykCQx41pt|_F3!33EML3AgXuyAq0q0-Uh6D|-Yw%p(M*;f*uje`|ks!ylAneS?CRf|Yx>e}{E zO>xSX^>BK%t7NOyOhfyRG`pqp{->;D`~=*Vf7Ryj&Y{@mD6p3y?dJ8bLZ4z>&=Vo9 z>hAg0sJg=@<%JVfAk{FP!R=1+?b92xzMn`JPPF*X0882^NiuJ7hsRLZUQbF=QcM@+ z{mNTtzY{gjzGb?)IXY@Y2z?_;`fPiFU#Fymu-<}Ve)oEFq+wA!h@&>Yd-oF+QMGHI z0NUGc4}X922&wO@h4^IGq?JAcnrWXtB7XM%K`ofEehERp^sQSjku%b#3AdkXFU*Ee zU)mL$rrP|}^>m#tbI~j3#EDM}9fL&p)qN4g1HBRkzxzcaz+5%yR$VD&vm>z1w;?nn zB=dGz4I0a4yZ$75Ke`x&$t71U3XXyBuOFjp9*x8WeVe%^j!*XBxQYD`xBd4Dd0eBM zkU!^ACjRArXt~Nb7oP!7m^+&eVO@9UXFVCXKoug*#z}kJ@*=8>C8Na=n36y+c2ucG z5suh^W%Ms~x<>dKW^t{poV`ll@bNQ%#cy!m7ZM0pCQaD+L!sv?;jb$7KfwI$Yf!EtohM?g@j$2Ydp zWFi)G3%w90AyW||qm|qjDk$}XUfwW0=A~kAC+;6_6$UV~0e*rQ!G|#JpnO0J^k{;_rTOx63g_ahDLOKn;!UG~-9Fm?NzU=5)0; z&UXpR)COWXAU-&;+sT>s(y&?B)HC4wiP@%3^mmu!ws-CjBhXyzP`%gv@97eW$vQn= z8p_{f0Z3Hqjz(FVPVQr#Vtz#v(B+~`IVazrS>h>Kt|s40dG7%+p{S*TR|+SAE9(aLQ{kvyXW~6J`B9@! zRd);emuVnHK~|siGN!WmxY@uht(0}g8MrQsDpziS!>aA963EYbyT;C|T=di>r6QL7 z8wfL|@~?is59ke?^BUAs74BHm6R~B=MQ@9GD+k!;wo*fsm~9N+_;smdFV)1g;F&Oz zfSKv7b7k&Ff~^-P0z0+eM(Lo#@i$cF@*D~d)ej)>?)(F{-;XuPuNL@h7Yz}b8iM_e zFdCE)-vd&nVvhu;m7cFaJ~}>G@ed_tXYJZX?G&wu`p2eL{M;7d4L3yK`e3ag z$Hm9W`dAGw*BRGz-+r~K=ufs!3iqG>otd;M#!=eS+Ao9EBpo+!mAZ97UgP0w$d@H_ z*nTz!wyzF58BQ=8{xvkxcyTBd8fj7feLElKt~&oj;ugH=7Y02|Z5vi^6p;^gDcUv- z+5WwgYV%6#mxT?LKk;5dqrpH|`B6QI?`wpE2VSOw9fR9lVvv)402N=~NKH99q~_PV zj-e#vsEqq(K#86Yb3R0{mDn-|BoMF9Fjg$V1m_UM71o4Q8ud9x1|13;aoh8FPlJF0 zrv(L;lgFgI`=0M}6dmEs>(tW@x)0If{ByrUUA0-8dUb3|Na%mE1mUC?d#dn$LOA<* zkY{e)bk2cQ+MQMBwUYf7SFGld1N%RUI2tjn8;3(_D)QNE)*WzH4qu<%e~RMVy7=-9 zZ4nm==zzI_8-$^)uZ~hkDmI7VuDJ(I+Nt&(@~G*Mlu2fq^FfBF)se9UG63L>4?j*M zGN$v_tT)$_wRg)8nbez1Z*&3*BKl?uYith`1)6fF=XGwX4}3(|Ukn>tFE8T9ED8L_ zDho8-GxmxT@2K_i88Fv)+)9*tKV@Uz=-ILOCN7PkU{3}+~lXHE1W@ctiF&4hDS&XwH%-xE?Tknp#-I&_g48;e-XNGj3-d z?|(#c{^4J_fj{pDlgSKg(&;k}$lyhoZ34AsE|ZNmv71RvV_geYifjG~YCRrDA)$Ye z+ZY(N!3o7QD$C^4hfcfQuI|2hkRIo{Ox1`=sA(zcJDSXM=-%~VzvuNGHo}C5_L@(4 zt8AS`wNW zq+zbq=(25X#wT`e*kPY5r5GYGNQRUM)7CErHrwprlEGq9rHJP4rk%MQN5O~|q_V+v z2MQVI_Ip9~wyNlIwptH6%`J=5OuCj3F#&^<`w_Kl_ZKH?KMnh1YKuxSup)%*fmb53 zSP=3UT&TayN5?!TRk!)cdePqz66K1H!@BECl_B&~-8wN<4S4LOhfijy8)ZbSR5nvU z{C*fW4P4(nE8w<7wUgEfuo}0VRPUP$(gLTIHro#7!?EN_+Xdt)WnLBp zmKgF1{#Rsd|KwEt<{tWQwS)6B@iGv)IbWAP@P1Wv-NEgY^GaLyt*zYq-7#v1x2({z zUK2Z?c6pf#(a`ow)Qm|+C(ldunc^j}`+IJv6g%@(x)0q*)STdx#z$YyL%8nn)~E?s z*RlcFSYK#PC%CGL@iGH@i9e}B%MIMUWDzNH5cX-cYjV2A6ro&hF*e@XdS+@&VXAei zG}j7JM5-ew`@s_ae#SFs!YIGYNrvS|;D|kUGm5i8M)ih|+83D0UexiDM*WPKs4w`o z6sw;_#+b_OAXNWGf@E^(O>ZyE!#T0uvRgmd_;^QyNk#BIag%}m~}WCLb%kD z_B!vEg({vf)9#V6$ROOqEYvzex+TiPAIFP2zup(c8yK&H=-f!9<>02e*^lYHn3q`g zVI3di)xYAZrAgZH5sNv+?Nl#FUr#z)(s%|;ZTf(G?oDirvH5Ui^y{bc8d|1Mc;2_f z*4oGZA(-H4^8mPRiKY3=I!lnj#{-`@4oHpICbk+T`5v7d`FEqe%u{|%V~rH933rW9 zeELBUvHpP}reN<3;PHzqD`FK^mIMA*TD z)Zc*$tM{|H0yiWO9GqY2$a0aC92exGaucdkf3S5VI1oT(nB||gcg?W2_Kka7aXtMp zz2L{IBR&(tF~;C5>vOG-zxz%o0)URJ8S>gC;DE@+^?Qb_6UvF(ml*u4C2TAqWy_Xg zHYTH9f$i9Ym3X(ngPdtNbQ?*Fe2|y8=i5m)n05Mv_A4*De)K{^BD|2DD_hBPopMd^ zXJ`Yy9spYy@jxihobu13;Bh%A;FpYp8*kf-q0Yz1zVBjaFDeTI!Oj?&HgKU zB+a02&cFK^h=vJPL77-5-Zt>Baxlrj%d<+|!=|(!z~wc$k1>o%LMBPOa$OsLa;sdP zcv?`gFEt+%@m@*Phy0nm znTp!{hU&?>K#2^LfBLzlT4uO@oJc#rgC3!~^4~hZn6Ol}7kXW6xLtv}{Za4g2e{?m zxKIBC{>+p9Bme>#6bkbn)4?AIbhc57<Qu#SP#OoWw~};Ch;bgcs;FpVetO|wgx7+?OP4lx+ET*b zkicz?=rZ;1+1?rEL|Y_{;CPP*02qj7qUOijm=KN|v0+c#Dd!i;vt?IyNP_=d)0fVr?9;g~_40Im!)Dm6U=`&Vkw-lGYVu2<1)J)!-ucQuh#c znw_^1n^U_s>;`=5Ltyz`234}sZqXb$jXZ^Q%e|hn5F?7m)YD5nZ&}Au8?a&V^1^3x zufkIr9zkO65RJngdt?9!)whU?my>**ZQStFoZ{{+P;dtPn+Kr^$}A@RYYP(A5X20F7!y9_p;!cC)5B@Szu()U1lgf(MhD@ zBrxpE{4-7o_u(50GZXpkI6YbUbRRS{%$MxJbmDSyQ@OSZj#?S`d4|R-oS|lhZh72} zCAlk8fS$5H2BW@l8Ssf|&v=n(%dG(B*L7~2BA%u{7XOyy#;J9Fp>I)%rP-W2TK%Ds zMj9*E)Y7gPyP(Ni{>3L@9JNY1!xslTr4-kYkMUBThl{@~(^f0}Z^4-4SQUSMb?C(Y z4~nibpb2gbqlD6;QW8Tc>F$`wCn4gb8&pb|q%?yG2nf;$NQ-n%y1Tn;^hU#ABL|Fm zcmM9s&dyG}?^DMw;+cJ{d-%Zh>&3Pw`9^Yli)MI>SpvE343GhZoIY1k%?igqCNP{= z@7H3QXd02@;ic>C3{|V2{Mzzu9_a~Jt2=IQ0DUOGnfV$}{W8!%c#LSZE*~Qx%r_K1 zUN?yB?=1oWr9Ms9HyYvbK4rn%%nF<>Lg>qzL{P(w z54=oP7>+&{k8d-5t|z`nhEOx0ug~vAoO(D|18dBZ8m`e4e5^wcX&v*pn6( zLJ|v=6il8F57pZezuWVW{5a7{h|s8;NUqWDDs!WcyV~zAAQ1L9pa>DCytIvW@0!jJ z+X~9t+Fjl+-m)C7ucMXcu*Asox++Y5w?)wU}o$ zg+3o#xnjZe3V&XsW3(;7#c%kHca#G_SpOU!yYb%D`zDh9k`X@-Vj7wgtS<^EWbI{* zlY;;9JqbWa7&o^@j7(e$_WCGb$xN31<1N2uqwtA3*N<)8m+?a3FA>)lV-&sH1v7wmtJp)YCFjh>~5 zH?MXc)T&D7hd&77+k&G-4CMJ77gdN@xLD{lMpc!{oqiNA3NOw{|A)bCz5 zC}d8aJ#yR^0SHUms)@N1tKZx%b~*4+ohX@yC>TbWbMM$Ugy=ms<)U|DTU;|2XMwG; zF?^nn(=M*DvWJ&EC`?m08vLm0tWzJA)&xZBoNXt$+as^kvM-kWqspKTi8i08L=^7*EMfTX<$6=thR1jWkQAr+X8AM;%47&>Zb=6B88MDcFli~3T$aU4r<12L>JOgc@Ss!X4Y&^x%7Xgd+4 za&c?_O}$3QcrE;(V|lxRC30)3?An$pGEq-6#M`7G+V!es3WIu-{anAXtR?8yEx8X( zTJf~fHUq&{6cBZ6Z!R{s$BNP=_8Usk?8BeEv}FgOY70jh!+LCWdyK|pkNC4?HFld+ zChiQSmz4lX9|=4`hltVYT%)OV(z3$DhQFUxVP3a!eJe$mO_=l1ofF5gOYyxugW{%ZOW$?6%&bR`(8j?%@9BQZ& zEEyt~0j~dPYFWF#gYhhrP`xNOK};7cI$V6O6>vDLV0iVggZixS`DAhytU#yLFCYK-ixqUyJQI<8Qtl%FiXN?#r;;70mYw zJX-O-6BMHrYo%w&!(J1RVU?gD5vhB9>w+0RwL5Fd|7~N}sJ6o?PR4cPRH)qg@cGFl zpAtM$g8*hD)6|#IOM>XIUxwZT~6Z|3d2MV5a+Y1hd8W=gseQD1)g0L#_Uu zRu4RY%z=Z~=b1=N>dR1%M1U)7LO6U-6&NADD1H81BL{?c{wT6axzre6+5x@h(g7p zJI?P+>R84KtpU4>^?Jh_Aq?ZX7th+<4?v|GfaYo-N`x(XJJrUprVd-|FJYK8Lp?v0 z|5It@BIF%N24l|R`%p+v4iPC#DK4Z!M-}hLYTnxYs5NySa(BAHObTn8Eeh}SPC%Sy zo?^QT_UU$|5Sc*CBwMZb=us1d>quJY_=8Mz(VE$99y6LtknFG?hn8i% zo{z^*w^KXe$$Y#m(NSJawX=Pdws*pgG_7wGC$364YmDD%as8kQGB_pPP~pMMTGkkTs$$+y-=2*H`rg!OR6Xv0#?ocfUxm) z2QX^RDL=%1=}VhAc7I!1YPHfg#hcCLU(a_23d~9!-^E(%YXh`7wEdYp z&C(;1nFiPxhYjPyKUcg3f1mIKhWgQAYWs55*@Tadbtfx6Llm5D)2Wh817?Ds8$haqHWCc0d_fA*<&lHT+9%)vDTgl>MtAD4sC4J?-SOX{yIgC zm5^C~5nvk&A!4=h8e|purg-PKcAkZ5aeuzLFIV})0TCgK4<{!~Ne@%tTEef371@4g z(boDLBN=;O>^Oz=ZkcQ*WdSP(qgR&x0=H^>hugNcPgC#manbA2xKqTwMqglAN|#qw zRzL_POT-aE-LAoBJ+)JCv>}DN%H+qx`*K`6Qk(CQ(fULt(8%sA_6AL!@V9Zkb}uhX-}q&?HrN?xS9p!YviGl=yk?*nAf&dqo0mNj zjJEy*e_iFQ;*_uIco35Hy~N#Z3Gk~V5l4p6%UnbMa_nqm_IWX&q^6%$`*vr&Fe*zL zgw^6rlfJTeA{5lf3a|uMNZQqIDndV@hM55$d@KVvFJM@Dz_%7vQ4w`%NavLnn+oL&+e2O?pLv-!#{oQ!j3j!rk;F=T{V4z zKDpk)<3#b=W9wzvZ#-kQ{a?a?qYNvT61k)4P@=-No$mIq+t-C-Xw+Fl6R9NoX}Flx zMvVT_ia-Iv>Jm37V`=vp#Vx_BwCyR~DfPOzY0IlS)U}&!v2ON4`Q2ZoBY0h9>CrBS zS{p3Klqa`B1gvv}T3=bh|k{ zWvECGyv(E?opERX4L6GkZVRiGo^n#ik{-?T$flhFg#&AW-z9mX*noW#(NIvmPFy|J zYfH5d=4~5r%6Xzb(>ksN*1z@xPkEbzBSjLJ)^-}4(?;tTL`+C7sKUW7;Ag2XN7lSY z7`;1{xMWe4nZ|e2v;1jWqoVb8mvlDp#Sctg$|_q^wkldW{kH{tl>WAQaX@)OW@V7; z8~q7r5k))ttcWhbE1|bt8K++H2}+|ZxZB1V0^!ZO9v^l`M-8IN=2i<~j_v4o*^ChJX@V?|+6VY% z;;6=oWv}W$4XIo6@J;24TJiUEu}W(u&QwO~VCl5ewXgq(?gi50Trn|~Waain4i6Uh zqRUnsZ8V(pV9;a zS=;Io-$ZX#XEA+DP<=M~H8rk8RFK)uzM8H)5`wr-BEo`C#50V|6v&Ud()SXf&6cpYd|R!6UDrKi-|xH21U(T$u? zFICBuoXNpm1w;@jz3JMV>!^zqw;au!RfoD{p+B`)t&ZH^5 zFCuwpC2^?s_H0|t0Y^{_At3T&kxnqFaXLw%=(&_98Hq1@JUS?2$6A?&{D7)k@X%Or zb_am`^UNWB+_KlEjyh<%;r7)>pUClz)fAyfW9f8Cp-p$+i5_D0uw`RwY`9K8Gm^d+ zKMr^dVMcfN)?cpjPvB&bluT_0E`&vt_1aU-*CY%L{}gXy~&dAu+4xT6D_ZG;foHXu2p|eH2RPq}C$pt?1jh z99Fq7GBHvj3=6GEhjk}wrUM0VaI`Xhu8Wk2C}&5wd&|Hg^_@`~7S^$qwSd#VjHw{oa^KyZlDq&2JdKFs}UF)hmpYZU~b^^Qr_{+h8p>PyRI|3P0~ zM}FC}nMKHLYl6D8KkyMse^WIH;bd_rztC9pFm9PU8GJF;f$KpH$`10*)qjSr=K>$H zFWq}Ly&mi46u*ZliT%A@my=TeRvEGA+%4PJ4x4Mfao@Y$4&dg+JyzSdt{dN&kA^es zeEx3jt{hngHD$b*!4$NMcNajoF%lA|H)}PY```Q{DtZS(TFI&31p1sfq^UJe+kW(h zT+p!shAKM$5s}`(?7!r?u}kCB$!uT5*tWLS!Y&{wxaL?qiSL?u+bPst`TGxKeQ&wx zh$i92QjX6rr6O$%-pIQ(mD;)8Z?(lxBIZhIGazDxFxmb@iWnss$JcIHqftF*r;4J_ z(a73Dg1abt$8#N2*gfFuiD}?^Zk9}K-NIk2vnUM|xwQW+F!9#c%^pJx@9trK># zJBR{J&on)M6j+phfb^!^8jA|c9liOn5Ce|8{e4`qgXKi@fE9{1EQAdOPhiA31G;{G zuHfv)$7=*S4UUGPCR}(69i}$6kxG7pqDf~OQ10wX zVf}W1(mDXk3+n`YgG-u1Ljxb}n%eyeEsS9SNeq6(RfnEZWG*oC%9Z;QOpzO-zn=!u zVV^k-sYv+vP1${zI2iy*@pfiMt zsZRdPTpEv9Ws8EqXTS&FH#-QND|k8D;Whu%bU|V7SXA+7ms_=|P8$VHI+DA zixog`uiR7khz`rhJvnidYoqXaHu4b=s(e4nPb5@O>>tseP|uWX2AnWKywh_6xItLq zc}lW>DANxT!Z^=>mpud*>%^z+lv8ovx6Z)eEXu|d7o0#l3A{ggY6QY-Gko_Sk#BJK zB{;JctFylJ>lYwg`M&wXXl8V6?9l?Nc-#lpkhs94l@-W4^hkV-y~U5=pH7|xSEfI% z0r+}!hEPX1BMj{@r0uCshviJf`wKJ$aN~{*Ll)gXPXmnBpb9Vb;zN75jbQ9HHxvnO7>oy+AJ)3cyTn z0ISyjh*U~+=S?V+g!?0W!Y`DtCVpBnU4T1%{l!1J>;7hsa?}0(7(n;~C)qs+oOljd| znHrspMA_#8`BL=p1;d;&7uM=Vcf99`Wwa;PJf*dRr@dU7!w2b(V;fc0*93!^%Bg~= zU}%y~edG|fTBOs!(evZ%;Y~pRk4$&Ve(n943u>Om#J1#9@A_3kEWlnWugzNfS()vB zv00C^o7(lVhgy>@oGKA%(Ty{bz(yOlq(e(0$C9k0QLeq9z7AKmd#tY6R4 zegm+Wji^gUES8md6djcdZTZaWMEY#hW^mTngm4P5%8T}qv-k&-U&PWuBniJWHjEPH z6HG0kTYIA~ozMyPvT6K2f;Sxud(l9nr(otrCDwohK#k-(7qw>%6Q)+}cf^O?#8RqV z!Abk}5dO+=LI?8&K}&Lw#WE{J`GX}fcmE>6M5$(;rkHI?^Ff%p9Y|P%&DIb;5;NRr zU_8EljOBv;XuA(qW@|$$N5cCeuV<51E@BaUWjz>~EDkg5LT78*)apJ$=h26-J{zbg z)0(&bR1Ox9ClXZy0${`S+R~P^%`EO~UVxRJTyHZSbp#tw9ZfDJc^g_YUcXkGbn?`w zm&#tc=_aU3zU4Mxq*5Y0p^qi2Bg$BG;yvtD9{o!CXu`k|IwWQo8uK8`URDZy)&2lJ zgC-<`WLx!IpyiZ~`7e$32>9-};(Q=`H%&S6n21mxw9`oqf5%w|6tlF`erk9A;Iflj zs;&TNRQ$Am#r!GH_I*zkt;B6b%PTbUA{kBAx&dX|Qx-xm2eL9s*ZKW?V*X$i>&~r^ z$rGKY|KNA{NCNqN1F3T84Tvg0A4Pyk(#vWpFBE(@uQG#6GB67^ zS)%u)HrneAVv}yf2LE(_DRN8!Josj6n=r_lu)0A03_sTzCmRs<*vCv@s~Xj^{c3?2 zf@c)%e=h&9sL^t;S3zLqT=_GnB`1ef^;S&)})xiZ57+>>|ybNlQ0^_9dlA%Ae5{%9e;ljp;-&W^16fy9p%P@!jGj0 z5^B*L7w(j^)5^VGzuA>;$Lv~$WYbp$R#eL&r#sHVh;h)tT$>AXcritauJ);b!_M}2 z*+xdcU@?D*tU};+A+9C4?WoiiT+v(+zSX*ed)SXCfC(m=s}yN-#n>dM=B+jgeo5vX z-pt6zD$Mp9Zy@Lp-{9UxzB)M+E*uyX%CqdgqJ56{ypiJzO?2mrrzKeria~~$oLW?- z=2BHT;pu@uOU*;2K2mS~h1yyWE(suWUKURXX{IFfyj4IJD?GUgMt83Muy7Oa833g%Waxt{-7~K0rqRM65hkH zx62aK9xcysN^#{>YZT*3akd$cE4ugv+p)q@(3acO|08mTVx_G%#|JGG@Z<*=Gf;Dz z-Ueyq)R6%drq{`D(PuGI!nyvCIb3JP$Y33|(B5w<-CDTXpR)>Li=L5r`o_7yY|w{C#^>}r3DG}iq8s@lZ_Z^4ghc%2Zn>KXi{OaT!-$yRQ&p+Q@L$!m&n~k^nibC*g%>j<;vF=M;gC zmnSUWBGpqjziN1Y{#Y5u?5@(O7&%`{n0OE93#n5GaRd|3@Tq)hAGw^}&)(M|WuuAQO@Mw${WjVNSb# z|J*2Y7$XUE2s%{`E?a%jHQmT58+{Z{D-ano@RymqScDsbGMt?nLj z+a4GiAXC!P$G|`i=JhnK%RtA#iyrq*mE8fn)96@F`=!O0pxyRJUrQ7G8` z;B5PrDmU~`6FW8G<|Cw}ptXMkKDEwQ=to7Z^yTycTPk6DrqXMFce-VB{}IL6ZZ7V8 zu^NLW-m&Zz1j*g>E-_uX4ZDg%yCZ`)GDH_fmcM+5bQLO*v)z_5TMM+;3`ai+i1*T{ z{p?oq30*CG7iu@mtLSy}0+l5SqzOH6DA6z*nNMwVPM*_wDVD9N*o}QMMJK3BUD}v_ zOyufL3wojE_m4<8)a;#$1X!LwBcb=l*L!RM?fLe65--aFS(|+3N9LrwVu?dGg(u{{ z>eBTG)(+t(0+}MG*JUDYZ@kJSgwqAYThGeE35x?X5n=(k?t1ZqtNcENtL9j*-1^Zb z>!a9?d1lN%BGW`2ECBIWx<0jFp!vrRhscMecIDYHetegc7c~CJbsQHX`lHnpe4M6U z4tZLI1tOrN{Ts*3K1h`b&s|;S+z_4m=ElY#_UFsmtQiIOnN9}zUERrE`AOhNu>xiJ z-$pSEb0-LvxXRIw)ty}Y^q7_+J6oZs6hn^2&y}lI8?7|seO640uXtM5o1LC)d^|aP zWAqoUI51K4Jy@8&GPA{9g-rmi$n0|SZqMHcgl5n7W?#02NNij&;bnEfv%X5?dQqE+ zUKWb6(vVlNIffp)hCi)Gd2LpbmKLtTh8VlVVjp*U{0tfvBBjl+{Ex`Sl=(kHZw}=_ z>qnedj%@nB2j2DcT&+vF5>`T8d)=Q?>n1~m5&!>k(tYd#^jx zEotj%#y?uxRxJLoo++=!?+y(eSkL-OQ`@-)d2w4(^P}-;J=V&m?l_E`aQbd{x??xv zejFyt6#tV@654POmqtVczK46I)=!|u6!U#nG`8DZHF&gPjSazpPgu(x=CELVT)iy)5S z?x6E*(v5gCPBENhXjp*T79ot&s4|UF?C28sKw4y^8}NFl4Kr=gd(!MA*9XTajDjxW zTxB3Ef4%PIVC~tVXBKw*%Dh-yd1@QwUimrpbI5$ucWmL|CTo9{BRgj%T%DT3R<4tc zu-tr7`cq6l2Rgp;cJDK=629N}!a8_i;OfOr>`vrK>G5~KlV_>Ppb+`zItgFUSlEe* z@q9KNo?m8IhENdF!ahPbD3c@(Iyy>{p0t#|TD&%2qSU1;W!8_CG~CllCTRI*JONSH z$>n(!33Y4scCFM@DV1fy#@$bZ}j`#xr$ z;ge8`eya*wuzB4T-1nirmE2F^UDKD(-gkb4yFD~F0~Fi%}dLJc>~jZQ%=dNMg|D3s6+jcS8+sT9eC)0 zrSR;NIW5C!r;Xb9$Y01h4!UL5`aScR`m~tbqLRhMH}iS~QF?^*cu%T5NkP5fA?|2mA{Bv#yX3HO z=;E3wCM-5z`UgV5irPZ`F{-4Oz0kQnac@%ab+m+{wGCZpteo$or%W-Tw6BokOUN$Z z{WeN$j1y30vZB@vEMc&=2bU#QHj)g zycOtiyponyh^j~LAVXIQyV@z19Uoy!H(IKsYmPq`+ji-CxDXQv4=mx5p48 z)pE;rzc`#)s9%_f+D1S7uPY{m!#+U?EN~%JJ)6O0pzq)E#f2iTyBF(snGs~DQJ^dU|PPi^78Z2lp-UW}sSCT<=jwzzYZ;n@F)^=p~ zvXtJ+&)0W9mWI|C#&H^a15lgCjXoF&N71=6a%A}s5inhR=PJvTq7WJY0vnrDC@&TD zwk|SrP(odUUhFl#q`J&F8BIlq@-j@=Hj(Su!b~-*#%0T|YaX zN0$w^(`%Mm)=0})bf^BJ?w%*gpb?wdu8~AhENOn%bZaxotg`v|x&l8 zs6IOF_v|(k;b^GPk{tWBgF!FjwEv1;B#A}6HZ^Y<+C0zud;B+9x^gb-d(=wC5fjmS ze81z9`hN(m-OS8h%m8M)2wGi2|Mo$Y;BG8k;5$R?dWHk2WP#~TE|j>wo4g1 z{*T7Z3F_|b*QEj87w^j@NDMhk;uBBpnXf7Gd1KDZJeQR73%W*6sps#P6&A?H2Cn#D z{H8W5PLo~|9+<6;Zhd^30DVTVZs%v^qPa9O^(Dm1Y%o=$z{rFrQNW3XoBk9QV~GkZ?IXj4#@? zv|;At3QTlJdzBni*ll09belG|NaEV}d*dC|^Kl3fnLko#&%gD}zI5iFKx@jI0%n7n zD3K8@&CIVzW+HztB)rfo*;bRQ!PbzO)T<9RMyr#hG|pLA4qo(m%f@zpww6*sQm2Ir zy664h`$ruLM(r%ue1+qspX8|~DJ>|RUVa)*YM23Vuv#mdX67o9faDT~BC{>QV|b-iP2Nrrec)EkspJ*01Z`@p z+<1k=@r04G5FP=(ft}blb^}jjyP)%M=(Ra06GDuMHNbLWz{VCd*JiTwI=?E&h}CR9 z64Qhe$EfRAx#0*kN^k`>y&LYKvpkx=B93cJp3WYjvy$Cis|Ez&m3)SdMr#kUsj$_4 zBa?j5b9>x6{2G-Cys2Mr-u4afyUW1{-U%EXu+K=kOB;xgPL444a;*!(3I^taKa!+52cz~UU*7?M#2Z_|cuM*r3+34ypR{qCm?Z<~+E_Ad=oITQ43~ty=A%!A-aEpmK@hE)EA^N*hM|$W!A<4x z%QLW!o^J-E=9)bnGmWQ&7D2eNV1krX`<`lx{Aa7?not{RPl*TdeahC+V?&LdM33U$ zk~-i=0Zat191L!?VTI$qp8p<5fs;-&tTwx4d>PU)Pta6sy1$NBW80klXnd(yx$*0y zIj^sy49|&1;>op(zyygRDEJH-!Tt1I>4S-{$fOicis3R8NyzrZ?K+xmbo*u*;~pqJynIW|yCz=+e zLi_#j(4|cP^0cdQ$Ie?H>7vAN4V)bH# z$JZ&()2ynwxBZuObe$Syj>KyXcTGXHt#^@iiaNe72><_L{Mjb@4m%$nR$#aiG@rYy zuycu5)$4twNP5py%*XH0yKh%MUU|aU4}|+8WDdtS+G$pNn#Ma!PY&j;o`fY9m@h0{ z4dtYX@HlLHUNMO$ays8ZGk&<&Tf2tkyFDBDvO!=vaRkhkBpYWCpmO7hAHQ5Ru%ao8_XSSD2Dib1{2J*=5dPu$SUPy( zIF7R1LQa2cCGDiM{OwvFflj@-wM}JILE{2WN`;|k{FpUH{DOQmuF7s)E81O+&Y1W* zO$xa11wlYxxchOOZddQCp*3OaSOx;dxvhw19S=Wp-+T|g-9QO`aIeq^=s8M z&d9;rjccC$9oo%`!lAFT7{r5s=e9lz`7l~gcSqh2miTKh3w1l;hGGg z;eq^n@)n3?hMA8E48nv&a1JEwLsozS#n>#+o-L3#!c&M{)OSRDP=L1Mc?m;1#UVt7 zH?w3dpz`BYrZLlpJ~eN5<4w1qV6bUCO!lO1iu-kj#gtOJ`fWZc@F7Ha8r}~Af1Beu zwLbM0h!sMtJrh$%e)-!ek-HV`s%&`+Ym?r$VyZ1oAO0ISTVm0Q>J&7|hfr0E$ys*{ z9=U_{5Gg%gn8NQlpO9w$X2uU!=N@0_a11~W-JqX7lwZ8&JUA3@CI>l1XK8i zZz(5YVBOMC8mVC=_f;oNXDLm)dG@1ik*aYgPsJ(;ep128pG4v($=(M_K{$jbgv?thm9Xig7*6}60=$lN+ec>zXk)kc6)ln?;O2xC%4PxW{%?>c(~}GAM!)GckPAFH?B5zw zWBw_kxDh+ovU=M8B)L)`;Lq6pe`Cb-@zuvs*1${;0J&##ETzxfxM6*LQ~0~TJQ@-# z&uOC@_F|hx&zafw4X`c5ogd&ik%<&1;7%1P;I|EY3%sN;-hCB;9Y z6GS~eZrrr?wgl(tL~e!ElG?H}e<-|3T>tx^SEHSP(Z?)qiu+Xeu%yd<@E6vFSICO`2bimRd&96}_ z_c7WJ#SB+UmK* zNU(SpkjW>AXJ9#SP7_oS5O9h$O1x)g@^;JV@VC>d4`Z8jsgfjN$!$#=_Cszth&uq)E{-86R1<4;$dm^WsY|@v|=| zW84;q_pC_q@a$UXkNn%5+H#q$z-Q^NCUFYYS_Wb3CCkH1BjaD?BiIP4oI1;2u%AB@ zQyOZNo)-hZpMafoay8}{Vx8~%N*|4>kOXDFcq@Op#GwxoOz5!4i|Rv?iLUXB{I-<1 zN)Q{h_EF!c?*V*wp)}b~^DBL0tm%*^-_zLfo?Y|x-wuu_8FM;|+xo*vhYb=l7yzP< zxkL7p({zDL8pb2{e(9rdcTrcIFV6AGzuMVlOI^Xi#q$efvTL5*n)O@6on^Hnub{zu zY@l~v4Ds?1#(CPqHKdU9UisOLKO4V^F8t^{Z>E9Yv(S}>kos+b=oT6p`&pdXh6okc z-0Gu=^U^-I$_lA+wG&lN>a~+W8={j;wy%A_*0fXF*fev3hqF7=U$@CZShKvGwTJG` ziIwx)IZl4xy^QnmXN{GRtUhpg_NKqN*vM%VPxi@0Ha1FKwed16)+N-7LUxQ z6bV^%lyZuR-6znWrr>}N@#RO4m^VMf2X!*@l7uYNS!O<&(AQHBiTQ@F zRH}!j?!aAetX3GX`Xu2!5|=L=1MckmXNf`kR~%msB4c*&G=kXt;*ovD4+8urJ!@}d z`#`IDz5X&N=nW0Nhg|^O7S}`@Q$rixw6&?}##8Q6OlcwQ;YxE{ImXNk{eJ(=TkohK ztCY2x><~p}bSaYlvOD`eL~R1kftA)eu&TqBzD3`dTrSD{tP0EU{iNX{tjZeBwERM9 zQmw`jqL3l=R8~RrQ((d_u>KgOhmlaJoDn=j`Q2#G2<0|cHzev;oovS#P(}SQB5@`K z$#PL$JjUGU2hPl^0~chR1_lE3H2TEViP{xoyE!9IsQvMja|IMhNvZ z1`g-Tt*?z_;(YhLWNFjw+pBjJXFJ3S>Zv|UcuI*P@B@uBP)M1OdFIijU;47Z1BGY( z%z(>DMf59oIV4NX$2KQ6;Nfub8R`<{O=ym*2tAO>$z?$Ajb)TchhkhyGBIa-f2$ji z9=cute4yKER!0pval#WXyIAC*k(mVBHNYfr9*BMiWQGJ11k@}p zk5|kcrp-Cmx*y%gPlX&6-_b__eTrqdguet?iTl|1R+=xH?N)z!Vy>&@@!PV0JO8%X zndoui$Wmu#r}=v?#W|Jglo2~S_YpNs>r?yr`F}*^rgZ!AAxRI-hx_mRqTckl>2N+& znEgWNE1#J*bME%umek5j3g*n$-jv?#c6UpjuE|`hsM(&iAv^8>}OWgJC+3TrGUw$gD%ayz{<|H zr9y-1s>jDP_*npXAPZJA=nA0VGWZ-G&?CT>TTGr$9wb5{NZrg{ZAV&m1te?D=#|5 ztiOc5gAWKl(*_%15STiN8ita9+A8|Q`_FWQjB-#mg|Va)y!Y1jiDZp^%J6^SerINkW;4^0%|_#T{qhqwIZo zyXMTPnX7cG91^d}WR_gRk3Aukf1z}G+}~^8S{t&|6)`Kd{v!fR)s%T`HF(*8?T_*&bP@ zVqDoxYP5+lFzkpWh&FVKVZz*H`=+6)%PwU6;+AcpOseD{p z=~sv?V)H$NEm>i?We#Tv96j`gW3R3$Iorhr3Hp8)NOyp@aSte%<9mD?Ji}_kfP8F?+t$E&e=ufJkNx zL)#XHp^zG%Bh`~qWufi%)R&U7b%?#t(ay_R1j6VJy@$4(@&&P{5gS4*0kM0&S zUHw7!$+L<(HO>C6Pkliq6~_Tm31yPqi|_2ucm1$;(=u%5F?gl~?zO7%YDXw33Wq=T z(@qz@|1Cr^62xK=?)sMO9y4bSOeyWaD$CkaI=x6H^=6Q}CH223+R+xqcug+W7~l2C znWkA!KF;?g#b3RO;{YOxo@2|2F)cQpO8-gE=}~4&jMRH~yn2@twodOy4O_f+5 z!tZQZoy%QRV#Y&OKRChR&?US_Ki5!dPvNzY4G@cIg>YLbih_|BG1w#D( zgj2`!RDr(3tOlxptI-eG#B~BoFe-2epZ}gSb2exF)U5pO;vYe=)30Xt6u%lJ(0)e2 z)Iv0?BG-4U1vGcUe=@$}lv@Y<+7u?IMvc=;UULo>OAK=O{=8F}JE&l`ebu0N7exG5 zsbM{tjP{kkU$z<+_GBqBRG`)B==uH)jt!JSfD9D6#RMM0KTlM}#_b_Q-%(MCq>;%o z^)n6b3WO4h|6o0c42BJNnP^;=_{L<-W}ZJs4W!~q?|KJ$bc1LcN__FEJ{BS%ALc-3 zr(jHavbMdK;Cg3QLrLA#cS=TuQ_=lpacw@24hwbTI?B2h|6Zbjo$yY?YgxZIEH5GW z-9-yGnDWpt`K*YL2BJiZH-uO-;vR-<__E~ITi;@}ioN z#+wm2@6CnhmV(Wg_`C19$dLr?%9ULB#9m#A;3%xNMa~oIaaRPg{M#Au$^_HWj@AN( z?MCn}FdOtMK=BHyhIe;tuZS{Bil#4b9wsJ8~JqM8w#pH~|zP z!Pc~&_4_cxltIU;g&8ji@nnoFk_-PVKUO_vgU(v0)|>~g$8>oRr*4^XCu39phEPJE zo>*ZYw~qkl2C+CC57|8eSN#o&V_X1cgeb-DN5F|y+jcB0&b_S4~iM0ZoJqTT4Nu1*OzANely zqy!A8+%IsLV4tG6_H!#92Y0 z9}M@fYwvfY*hVaIlJ0$K#HhK4^2L>sfn#$vsbnBxxDhpHE> zNRpfF+A`>RCcOLeX^jZwQUMJmxPI!A2+g5f$aZaC1grOyT0TK({WMAOwi5l@N258w z;p(bD9HTAuJBB-U&{Ij%fJLKoF{?X@ov5FQAmT~?0>n{3aq%e^cusu4B;jcQKLD=i z>h3gQAl#XGlP&Ckn0=Y9XKPlHPV?TQTOsG9ZUdVJHzHWx&t3;@CkJZvF-oHxwF&!Q zn3$|Pm&O$EciZa1-$XGpe~V$+iT9Erb(dx0#5NGKhQt(!CGh`G$PQsb9yk1W(7rVb!13n%e|w}6Se;<^Ph!%GQvD_}cjB#imN5rF zOO-x(Ls&TUJ&05jU(5xv9JNaUxEFjYyQy{G<07CtA%kvdBXEIfpee z!oW+pVWAKi#|A;bjWEfFn2jO(D33AUopO)4YtvEo*@n}{cULXv-lIA>5+s(-i7s}S zbEcjHzpy+vCmndf+;{hceD4@F;ZdHEH=7RiJzXyT9QoI*JpOi>ZmHrNUfI6|V;&=u zf4`2p_V}aaQqSO2rED?J6U69 zY_c)SV7$E?#@Z%l6?+w z#?#Myik?+#6&7>4M}~DNnmMkuo8_5QsEQ?lqE>CB=jA1~@&j;iHy&H%-9oV3tFttV zg3b$t1N+(NI47|BROxCZMBN_U;DA*^u|Hh&0|TCTuSorsejeKR@8X`TBafovw z7ma?+0}?(_98AX<$a!V;=Ek+*Egnl|xV@Tr0>sU6HPz^j7)VlDMs{EnD9J2%z$Z1+ zc+cV$zNc{|=A|~CB;M`kukP82$vHjv=nqQypZ4we`Qd*E{?FRwjn1Jptez+fqQr8z zo;hQAXH5Bs6<*#MQ*k%~Q-I7s8>$j+#~fySz2gaNekc4!*E~wm-&)#gml9oBM|d7K zwXw?!Xv#M#!UcxdQ~9&6DfX^!|0}Lkm-u$r;qG8}5B6 za;qMjBR82N`CE1d+`MG;I0Oxct~nm~sggD@H~N*e+WN+*ZEiJ*H26bDd1f1XXEHewE^u*#Br+U=NXYIoHHMI38>W)r{iZ$ca=3WGC!bz0 zMn-Ubc%q7@6>Enxy@{_NO(*ZnlPrwmVGg+@V}pvb70;Tke|sXUZd7ohHs652 zsOk4e&PgNEiYVnpM5A+iEyQwpcLv^Nk%^hVc&&rTA%QvT+a8!C(=?qt=&+d9YmG{L zlJZ^LJNb+TH(>BHfN(trJPZ_4aaJ6*B}rZ>T4@*e*9CWx^C{eL4Ce79nx4u0`BqHs zkTGq%^SHN6k~qj5W82n>DLAB2Bv1>PV@8o%%y1D|HQC8xSG95jymfN{#+bbg;cMJoJ@Nfq>#s^Nk)2*QM5q|W3 zU=PEVRs;e%_vb%=2cAhq9IDqON#y?kMUh%K+84y91nwsxvyPv8w-mGLk-`++Zrb0( zGmuHb34UrL(YKeV?l?ad1~DH|2! zu^eXuu{{S)b43+gs06&c0kI+S*LGMDxCLKeF_X_e{pszPBg~P^W5^@-!*5O7Z~!5@ z?j28V)KOW>Mg$V=5wts*RtISI?%Y8qcH(yY#Be#}`idfDkZzC5#EFf-nt6vs8eSP3B2x%S{({3iTlISM2s{?#Sz>9zz=aD z>P`wPvo5C>Hzso9B-Bz}&pe;TI{ny=KegQ5rtP4-+lXK?7i%123`TM~cj;dte%^jJ z)Gz)nFNQB9xcg^?t<|pW%z#H6vZBnOU=?B^c)>X%=5-#VqP^@qS25*qz6 q%S8oQWEoTptL@Wr select_threshold + + select_mask = tf.cast(select_mask, tf.float32) + selected_bboxes[class_ind] = tf.multiply(bboxes_pred, tf.expand_dims(select_mask, axis=-1)) + selected_scores[class_ind] = tf.multiply(class_scores, select_mask) + + return selected_bboxes, selected_scores + +def clip_bboxes(ymin, xmin, ymax, xmax, name): + with tf.name_scope(name, 'clip_bboxes', [ymin, xmin, ymax, xmax]): + ymin = tf.maximum(ymin, 0.) + xmin = tf.maximum(xmin, 0.) + ymax = tf.minimum(ymax, 1.) + xmax = tf.minimum(xmax, 1.) + + ymin = tf.minimum(ymin, ymax) + xmin = tf.minimum(xmin, xmax) + + return ymin, xmin, ymax, xmax + +def filter_bboxes(scores_pred, ymin, xmin, ymax, xmax, min_size, name): + with tf.name_scope(name, 'filter_bboxes', [scores_pred, ymin, xmin, ymax, xmax]): + width = xmax - xmin + height = ymax - ymin + + filter_mask = tf.logical_and(width > min_size, height > min_size) + + filter_mask = tf.cast(filter_mask, tf.float32) + return tf.multiply(ymin, filter_mask), tf.multiply(xmin, filter_mask), \ + tf.multiply(ymax, filter_mask), tf.multiply(xmax, filter_mask), tf.multiply(scores_pred, filter_mask) + +def sort_bboxes(scores_pred, ymin, xmin, ymax, xmax, keep_topk, name): + with tf.name_scope(name, 'sort_bboxes', [scores_pred, ymin, xmin, ymax, xmax]): + cur_bboxes = tf.shape(scores_pred)[0] + scores, idxes = tf.nn.top_k(scores_pred, k=tf.minimum(keep_topk, cur_bboxes), sorted=True) + + ymin, xmin, ymax, xmax = tf.gather(ymin, idxes), tf.gather(xmin, idxes), tf.gather(ymax, idxes), tf.gather(xmax, idxes) + + paddings_scores = tf.expand_dims(tf.stack([0, tf.maximum(keep_topk-cur_bboxes, 0)], axis=0), axis=0) + + return tf.pad(ymin, paddings_scores, "CONSTANT"), tf.pad(xmin, paddings_scores, "CONSTANT"),\ + tf.pad(ymax, paddings_scores, "CONSTANT"), tf.pad(xmax, paddings_scores, "CONSTANT"),\ + tf.pad(scores, paddings_scores, "CONSTANT") + +def nms_bboxes(scores_pred, bboxes_pred, nms_topk, nms_threshold, name): + with tf.name_scope(name, 'nms_bboxes', [scores_pred, bboxes_pred]): + idxes = tf.image.non_max_suppression(bboxes_pred, scores_pred, nms_topk, nms_threshold) + return tf.gather(scores_pred, idxes), tf.gather(bboxes_pred, idxes) + +def parse_by_class(cls_pred, bboxes_pred, num_classes, select_threshold, min_size, keep_topk, nms_topk, nms_threshold): + with tf.name_scope('select_bboxes', [cls_pred, bboxes_pred]): + scores_pred = tf.nn.softmax(cls_pred) + selected_bboxes, selected_scores = select_bboxes(scores_pred, bboxes_pred, num_classes, select_threshold) + for class_ind in range(1, num_classes): + ymin, xmin, ymax, xmax = tf.unstack(selected_bboxes[class_ind], 4, axis=-1) + #ymin, xmin, ymax, xmax = tf.split(selected_bboxes[class_ind], 4, axis=-1) + #ymin, xmin, ymax, xmax = tf.squeeze(ymin), tf.squeeze(xmin), tf.squeeze(ymax), tf.squeeze(xmax) + ymin, xmin, ymax, xmax = clip_bboxes(ymin, xmin, ymax, xmax, 'clip_bboxes_{}'.format(class_ind)) + ymin, xmin, ymax, xmax, selected_scores[class_ind] = filter_bboxes(selected_scores[class_ind], + ymin, xmin, ymax, xmax, min_size, 'filter_bboxes_{}'.format(class_ind)) + ymin, xmin, ymax, xmax, selected_scores[class_ind] = sort_bboxes(selected_scores[class_ind], + ymin, xmin, ymax, xmax, keep_topk, 'sort_bboxes_{}'.format(class_ind)) + selected_bboxes[class_ind] = tf.stack([ymin, xmin, ymax, xmax], axis=-1) + selected_scores[class_ind], selected_bboxes[class_ind] = nms_bboxes(selected_scores[class_ind], selected_bboxes[class_ind], nms_topk, nms_threshold, 'nms_bboxes_{}'.format(class_ind)) + + return selected_bboxes, selected_scores + +def ssd_model_fn(features, labels, mode, params): + """model_fn for SSD to be used with our Estimator.""" + filename = features['filename'] + shape = features['shape'] + loc_targets = features['loc_targets'] + cls_targets = features['cls_targets'] + match_scores = features['match_scores'] + features = features['image'] + + global global_anchor_info + decode_fn = global_anchor_info['decode_fn'] + num_anchors_per_layer = global_anchor_info['num_anchors_per_layer'] + all_num_anchors_depth = global_anchor_info['all_num_anchors_depth'] + + with tf.variable_scope(params['model_scope'], default_name=None, values=[features], reuse=tf.AUTO_REUSE): + backbone = ssd_net.VGG16Backbone(params['data_format']) + feature_layers = backbone.forward(features, training=(mode == tf.estimator.ModeKeys.TRAIN)) + #print(feature_layers) + location_pred, cls_pred = ssd_net.multibox_head(feature_layers, params['num_classes'], all_num_anchors_depth, data_format=params['data_format']) + if params['data_format'] == 'channels_first': + cls_pred = [tf.transpose(pred, [0, 2, 3, 1]) for pred in cls_pred] + location_pred = [tf.transpose(pred, [0, 2, 3, 1]) for pred in location_pred] + + cls_pred = [tf.reshape(pred, [tf.shape(features)[0], -1, params['num_classes']]) for pred in cls_pred] + location_pred = [tf.reshape(pred, [tf.shape(features)[0], -1, 4]) for pred in location_pred] + + cls_pred = tf.concat(cls_pred, axis=1) + location_pred = tf.concat(location_pred, axis=1) + + cls_pred = tf.reshape(cls_pred, [-1, params['num_classes']]) + location_pred = tf.reshape(location_pred, [-1, 4]) + + with tf.device('/cpu:0'): + bboxes_pred = decode_fn(location_pred) + bboxes_pred = tf.concat(bboxes_pred, axis=0) + selected_bboxes, selected_scores = parse_by_class(cls_pred, bboxes_pred, + params['num_classes'], params['select_threshold'], params['min_size'], + params['keep_topk'], params['nms_topk'], params['nms_threshold']) + + predictions = {'filename': filename, 'shape': shape } + for class_ind in range(1, params['num_classes']): + predictions['scores_{}'.format(class_ind)] = tf.expand_dims(selected_scores[class_ind], axis=0) + predictions['bboxes_{}'.format(class_ind)] = tf.expand_dims(selected_bboxes[class_ind], axis=0) + + flaten_cls_targets = tf.reshape(cls_targets, [-1]) + flaten_match_scores = tf.reshape(match_scores, [-1]) + flaten_loc_targets = tf.reshape(loc_targets, [-1, 4]) + + # each positive examples has one label + positive_mask = flaten_cls_targets > 0 + n_positives = tf.count_nonzero(positive_mask) + + batch_n_positives = tf.count_nonzero(cls_targets, -1) + + batch_negtive_mask = tf.equal(cls_targets, 0)#tf.logical_and(tf.equal(cls_targets, 0), match_scores > 0.) + batch_n_negtives = tf.count_nonzero(batch_negtive_mask, -1) + + batch_n_neg_select = tf.cast(params['negative_ratio'] * tf.cast(batch_n_positives, tf.float32), tf.int32) + batch_n_neg_select = tf.minimum(batch_n_neg_select, tf.cast(batch_n_negtives, tf.int32)) + + # hard negative mining for classification + predictions_for_bg = tf.nn.softmax(tf.reshape(cls_pred, [tf.shape(features)[0], -1, params['num_classes']]))[:, :, 0] + prob_for_negtives = tf.where(batch_negtive_mask, + 0. - predictions_for_bg, + # ignore all the positives + 0. - tf.ones_like(predictions_for_bg)) + topk_prob_for_bg, _ = tf.nn.top_k(prob_for_negtives, k=tf.shape(prob_for_negtives)[1]) + score_at_k = tf.gather_nd(topk_prob_for_bg, tf.stack([tf.range(tf.shape(features)[0]), batch_n_neg_select - 1], axis=-1)) + + selected_neg_mask = prob_for_negtives >= tf.expand_dims(score_at_k, axis=-1) + + # include both selected negtive and all positive examples + final_mask = tf.stop_gradient(tf.logical_or(tf.reshape(tf.logical_and(batch_negtive_mask, selected_neg_mask), [-1]), positive_mask)) + total_examples = tf.count_nonzero(final_mask) + + cls_pred = tf.boolean_mask(cls_pred, final_mask) + location_pred = tf.boolean_mask(location_pred, tf.stop_gradient(positive_mask)) + flaten_cls_targets = tf.boolean_mask(tf.clip_by_value(flaten_cls_targets, 0, params['num_classes']), final_mask) + flaten_loc_targets = tf.stop_gradient(tf.boolean_mask(flaten_loc_targets, positive_mask)) + + # Calculate loss, which includes softmax cross entropy and L2 regularization. + #cross_entropy = (params['negative_ratio'] + 1.) * tf.cond(n_positives > 0, lambda: tf.losses.sparse_softmax_cross_entropy(labels=glabels, logits=cls_pred), lambda: 0.) + cross_entropy = tf.losses.sparse_softmax_cross_entropy(labels=flaten_cls_targets, logits=cls_pred) * (params['negative_ratio'] + 1.) + # Create a tensor named cross_entropy for logging purposes. + tf.identity(cross_entropy, name='cross_entropy_loss') + tf.summary.scalar('cross_entropy_loss', cross_entropy) + + #loc_loss = tf.cond(n_positives > 0, lambda: modified_smooth_l1(location_pred, tf.stop_gradient(flaten_loc_targets), sigma=1.), lambda: tf.zeros_like(location_pred)) + loc_loss = modified_smooth_l1(location_pred, flaten_loc_targets, sigma=1.) + loc_loss = tf.reduce_mean(tf.reduce_sum(loc_loss, axis=-1), name='location_loss') + tf.summary.scalar('location_loss', loc_loss) + tf.losses.add_loss(loc_loss) + + # Add weight decay to the loss. We exclude the batch norm variables because + # doing so leads to a small improvement in accuracy. + total_loss = tf.add(cross_entropy, loc_loss, name='total_loss') + + cls_accuracy = tf.metrics.accuracy(flaten_cls_targets, tf.argmax(cls_pred, axis=-1)) + + # Create a tensor named train_accuracy for logging purposes. + tf.identity(cls_accuracy[1], name='cls_accuracy') + tf.summary.scalar('cls_accuracy', cls_accuracy[1]) + + summary_hook = tf.train.SummarySaverHook(save_steps=params['save_summary_steps'], + output_dir=params['summary_dir'], + summary_op=tf.summary.merge_all()) + if mode == tf.estimator.ModeKeys.PREDICT: + return tf.estimator.EstimatorSpec( + mode=mode, + predictions=predictions, + prediction_hooks=[summary_hook], + loss=None, train_op=None) + else: + raise ValueError('This script only support "PREDICT" mode!') + +def parse_comma_list(args): + return [float(s.strip()) for s in args.split(',')] + +def main(_): + # Using the Winograd non-fused algorithms provides a small performance boost. + os.environ['TF_ENABLE_WINOGRAD_NONFUSED'] = '1' + + gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction) + config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False, intra_op_parallelism_threads=FLAGS.num_cpu_threads, inter_op_parallelism_threads=FLAGS.num_cpu_threads, gpu_options=gpu_options) + + # Set up a RunConfig to only save checkpoints once per training cycle. + run_config = tf.estimator.RunConfig().replace( + save_checkpoints_secs=None).replace( + save_checkpoints_steps=None).replace( + save_summary_steps=FLAGS.save_summary_steps).replace( + keep_checkpoint_max=5).replace( + log_step_count_steps=FLAGS.log_every_n_steps).replace( + session_config=config) + + summary_dir = os.path.join(FLAGS.model_dir, 'predict') + + ssd_detector = tf.estimator.Estimator( + model_fn=ssd_model_fn, model_dir=FLAGS.model_dir, config=run_config, + params={ + 'select_threshold': FLAGS.select_threshold, + 'min_size': FLAGS.min_size, + 'nms_threshold': FLAGS.nms_threshold, + 'nms_topk': FLAGS.nms_topk, + 'keep_topk': FLAGS.keep_topk, + 'data_format': FLAGS.data_format, + 'batch_size': FLAGS.batch_size, + 'model_scope': FLAGS.model_scope, + 'save_summary_steps': FLAGS.save_summary_steps, + 'summary_dir': summary_dir, + 'num_classes': FLAGS.num_classes, + 'negative_ratio': FLAGS.negative_ratio, + 'match_threshold': FLAGS.match_threshold, + 'neg_threshold': FLAGS.neg_threshold, + 'weight_decay': FLAGS.weight_decay, + }) + tensors_to_log = { + 'ce': 'cross_entropy_loss', + 'loc': 'location_loss', + 'loss': 'total_loss', + 'acc': 'cls_accuracy', + } + logging_hook = tf.train.LoggingTensorHook(tensors=tensors_to_log, every_n_iter=FLAGS.log_every_n_steps, + formatter=lambda dicts: (', '.join(['%s=%.6f' % (k, v) for k, v in dicts.items()]))) + + print('Starting a predict cycle.') + pred_results = ssd_detector.predict(input_fn=input_pipeline(dataset_pattern='val-*', is_training=False, batch_size=FLAGS.batch_size), + hooks=[logging_hook], checkpoint_path=get_checkpoint())#, yield_single_examples=False) + + det_results = list(pred_results) + #print(list(det_results)) + + #[{'bboxes_1': array([[0. , 0. , 0.28459054, 0.5679505 ], [0.3158835 , 0.34792888, 0.7312541 , 1. ]], dtype=float32), 'scores_17': array([0.01333667, 0.01152573], dtype=float32), 'filename': b'000703.jpg', 'shape': array([334, 500, 3])}] + for class_ind in range(1, FLAGS.num_classes): + with open(os.path.join(summary_dir, 'results_{}.txt'.format(class_ind)), 'wt') as f: + for image_ind, pred in enumerate(det_results): + filename = pred['filename'] + shape = pred['shape'] + scores = pred['scores_{}'.format(class_ind)] + bboxes = pred['bboxes_{}'.format(class_ind)] + bboxes[:, 0] = (bboxes[:, 0] * shape[0]).astype(np.int32, copy=False) + 1 + bboxes[:, 1] = (bboxes[:, 1] * shape[1]).astype(np.int32, copy=False) + 1 + bboxes[:, 2] = (bboxes[:, 2] * shape[0]).astype(np.int32, copy=False) + 1 + bboxes[:, 3] = (bboxes[:, 3] * shape[1]).astype(np.int32, copy=False) + 1 + + valid_mask = np.logical_and((bboxes[:, 2] - bboxes[:, 0] > 0), (bboxes[:, 3] - bboxes[:, 1] > 0)) + + for det_ind in range(valid_mask.shape[0]): + if not valid_mask[det_ind]: + continue + f.write('{:s} {:.3f} {:.1f} {:.1f} {:.1f} {:.1f}\n'. + format(filename.decode('utf8')[:-4], scores[det_ind], + bboxes[det_ind, 1], bboxes[det_ind, 0], + bboxes[det_ind, 3], bboxes[det_ind, 2])) + + +if __name__ == '__main__': + tf.logging.set_verbosity(tf.logging.INFO) + tf.app.run() diff --git a/utils/external/SSD.TensorFlow/net/ssd_net.py b/utils/external/SSD.TensorFlow/net/ssd_net.py new file mode 100644 index 0000000..e3ea283 --- /dev/null +++ b/utils/external/SSD.TensorFlow/net/ssd_net.py @@ -0,0 +1,255 @@ +# Copyright 2018 Changan Wang + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +_BATCH_NORM_DECAY = 0.9 +_BATCH_NORM_EPSILON = 1e-5 +_USE_FUSED_BN = True + +# vgg_16/conv2/conv2_1/biases +# vgg_16/conv4/conv4_3/biases +# vgg_16/conv1/conv1_1/biases +# vgg_16/fc6/weights +# vgg_16/conv3/conv3_2/biases +# vgg_16/conv5/conv5_3/biases +# vgg_16/conv3/conv3_1/weights +# vgg_16/conv4/conv4_2/weights +# vgg_16/conv1/conv1_1/weights +# vgg_16/conv5/conv5_3/weights +# vgg_16/conv4/conv4_1/weights +# vgg_16/conv3/conv3_3/weights +# vgg_16/conv5/conv5_2/biases +# vgg_16/conv3/conv3_2/weights +# vgg_16/conv4/conv4_2/biases +# vgg_16/conv5/conv5_2/weights +# vgg_16/conv3/conv3_1/biases +# vgg_16/conv2/conv2_2/weights +# vgg_16/fc7/weights +# vgg_16/conv5/conv5_1/biases +# vgg_16/conv1/conv1_2/biases +# vgg_16/conv2/conv2_2/biases +# vgg_16/conv4/conv4_1/biases +# vgg_16/fc7/biases +# vgg_16/fc6/biases +# vgg_16/conv4/conv4_3/weights +# vgg_16/conv2/conv2_1/weights +# vgg_16/conv5/conv5_1/weights +# vgg_16/conv3/conv3_3/biases +# vgg_16/conv1/conv1_2/weights + +class ReLuLayer(tf.layers.Layer): + def __init__(self, name, **kwargs): + super(ReLuLayer, self).__init__(name=name, trainable=trainable, **kwargs) + self._name = name + def build(self, input_shape): + self._relu = lambda x : tf.nn.relu(x, name=self._name) + self.built = True + + def call(self, inputs): + return self._relu(inputs) + + def compute_output_shape(self, input_shape): + return tf.TensorShape(input_shape) + +def forward_module(m, inputs, training=False): + if isinstance(m, tf.layers.BatchNormalization) or isinstance(m, tf.layers.Dropout): + return m.apply(inputs, training=training) + return m.apply(inputs) + +class VGG16Backbone(object): + def __init__(self, data_format='channels_first'): + super(VGG16Backbone, self).__init__() + self._data_format = data_format + self._bn_axis = -1 if data_format == 'channels_last' else 1 + #initializer = tf.glorot_uniform_initializer glorot_normal_initializer + self._conv_initializer = tf.glorot_uniform_initializer + self._conv_bn_initializer = tf.glorot_uniform_initializer#lambda : tf.truncated_normal_initializer(mean=0.0, stddev=0.005) + # VGG layers + self._conv1_block = self.conv_block(2, 64, 3, (1, 1), 'conv1') + self._pool1 = tf.layers.MaxPooling2D(2, 2, padding='same', data_format=self._data_format, name='pool1') + self._conv2_block = self.conv_block(2, 128, 3, (1, 1), 'conv2') + self._pool2 = tf.layers.MaxPooling2D(2, 2, padding='same', data_format=self._data_format, name='pool2') + self._conv3_block = self.conv_block(3, 256, 3, (1, 1), 'conv3') + self._pool3 = tf.layers.MaxPooling2D(2, 2, padding='same', data_format=self._data_format, name='pool3') + self._conv4_block = self.conv_block(3, 512, 3, (1, 1), 'conv4') + self._pool4 = tf.layers.MaxPooling2D(2, 2, padding='same', data_format=self._data_format, name='pool4') + self._conv5_block = self.conv_block(3, 512, 3, (1, 1), 'conv5') + self._pool5 = tf.layers.MaxPooling2D(3, 1, padding='same', data_format=self._data_format, name='pool5') + self._conv6 = tf.layers.Conv2D(filters=1024, kernel_size=3, strides=1, padding='same', dilation_rate=6, + data_format=self._data_format, activation=tf.nn.relu, use_bias=True, + kernel_initializer=self._conv_initializer(), + bias_initializer=tf.zeros_initializer(), + name='fc6', _scope='fc6', _reuse=None) + self._conv7 = tf.layers.Conv2D(filters=1024, kernel_size=1, strides=1, padding='same', + data_format=self._data_format, activation=tf.nn.relu, use_bias=True, + kernel_initializer=self._conv_initializer(), + bias_initializer=tf.zeros_initializer(), + name='fc7', _scope='fc7', _reuse=None) + # SSD layers + with tf.variable_scope('additional_layers') as scope: + self._conv8_block = self.ssd_conv_block(256, 2, 'conv8') + self._conv9_block = self.ssd_conv_block(128, 2, 'conv9') + self._conv10_block = self.ssd_conv_block(128, 1, 'conv10', padding='valid') + self._conv11_block = self.ssd_conv_block(128, 1, 'conv11', padding='valid') + + def l2_normalize(self, x, name): + with tf.name_scope(name, "l2_normalize", [x]) as name: + axis = -1 if self._data_format == 'channels_last' else 1 + square_sum = tf.reduce_sum(tf.square(x), axis, keep_dims=True) + x_inv_norm = tf.rsqrt(tf.maximum(square_sum, 1e-10)) + return tf.multiply(x, x_inv_norm, name=name) + + def forward(self, inputs, training=False): + # inputs should in BGR + feature_layers = [] + # forward vgg layers + for conv in self._conv1_block: + inputs = forward_module(conv, inputs, training=training) + inputs = self._pool1.apply(inputs) + for conv in self._conv2_block: + inputs = forward_module(conv, inputs, training=training) + inputs = self._pool2.apply(inputs) + for conv in self._conv3_block: + inputs = forward_module(conv, inputs, training=training) + inputs = self._pool3.apply(inputs) + for conv in self._conv4_block: + inputs = forward_module(conv, inputs, training=training) + # conv4_3 + with tf.variable_scope('conv4_3_scale') as scope: + weight_scale = tf.Variable([20.] * 512, trainable=training, name='weights') + if self._data_format == 'channels_last': + weight_scale = tf.reshape(weight_scale, [1, 1, 1, -1], name='reshape') + else: + weight_scale = tf.reshape(weight_scale, [1, -1, 1, 1], name='reshape') + + feature_layers.append(tf.multiply(weight_scale, self.l2_normalize(inputs, name='norm'), name='rescale') + ) + inputs = self._pool4.apply(inputs) + for conv in self._conv5_block: + inputs = forward_module(conv, inputs, training=training) + inputs = self._pool5.apply(inputs) + # forward fc layers + inputs = self._conv6.apply(inputs) + inputs = self._conv7.apply(inputs) + # fc7 + feature_layers.append(inputs) + # forward ssd layers + for layer in self._conv8_block: + inputs = forward_module(layer, inputs, training=training) + # conv8 + feature_layers.append(inputs) + for layer in self._conv9_block: + inputs = forward_module(layer, inputs, training=training) + # conv9 + feature_layers.append(inputs) + for layer in self._conv10_block: + inputs = forward_module(layer, inputs, training=training) + # conv10 + feature_layers.append(inputs) + for layer in self._conv11_block: + inputs = forward_module(layer, inputs, training=training) + # conv11 + feature_layers.append(inputs) + + return feature_layers + + def conv_block(self, num_blocks, filters, kernel_size, strides, name, reuse=None): + with tf.variable_scope(name): + conv_blocks = [] + for ind in range(1, num_blocks + 1): + conv_blocks.append( + tf.layers.Conv2D(filters=filters, kernel_size=kernel_size, strides=strides, padding='same', + data_format=self._data_format, activation=tf.nn.relu, use_bias=True, + kernel_initializer=self._conv_initializer(), + bias_initializer=tf.zeros_initializer(), + name='{}_{}'.format(name, ind), _scope='{}_{}'.format(name, ind), _reuse=None) + ) + return conv_blocks + + def ssd_conv_block(self, filters, strides, name, padding='same', reuse=None): + with tf.variable_scope(name): + conv_blocks = [] + conv_blocks.append( + tf.layers.Conv2D(filters=filters, kernel_size=1, strides=1, padding=padding, + data_format=self._data_format, activation=tf.nn.relu, use_bias=True, + kernel_initializer=self._conv_initializer(), + bias_initializer=tf.zeros_initializer(), + name='{}_1'.format(name), _scope='{}_1'.format(name), _reuse=None) + ) + conv_blocks.append( + tf.layers.Conv2D(filters=filters * 2, kernel_size=3, strides=strides, padding=padding, + data_format=self._data_format, activation=tf.nn.relu, use_bias=True, + kernel_initializer=self._conv_initializer(), + bias_initializer=tf.zeros_initializer(), + name='{}_2'.format(name), _scope='{}_2'.format(name), _reuse=None) + ) + return conv_blocks + + def ssd_conv_bn_block(self, filters, strides, name, reuse=None): + with tf.variable_scope(name): + conv_bn_blocks = [] + conv_bn_blocks.append( + tf.layers.Conv2D(filters=filters, kernel_size=1, strides=1, padding='same', + data_format=self._data_format, activation=None, use_bias=False, + kernel_initializer=self._conv_bn_initializer(), + bias_initializer=None, + name='{}_1'.format(name), _scope='{}_1'.format(name), _reuse=None) + ) + conv_bn_blocks.append( + tf.layers.BatchNormalization(axis=self._bn_axis, momentum=BN_MOMENTUM, epsilon=BN_EPSILON, fused=USE_FUSED_BN, + name='{}_bn1'.format(name), _scope='{}_bn1'.format(name), _reuse=None) + ) + conv_bn_blocks.append( + ReLuLayer('{}_relu1'.format(name), _scope='{}_relu1'.format(name), _reuse=None) + ) + conv_bn_blocks.append( + tf.layers.Conv2D(filters=filters * 2, kernel_size=3, strides=strides, padding='same', + data_format=self._data_format, activation=None, use_bias=False, + kernel_initializer=self._conv_bn_initializer(), + bias_initializer=None, + name='{}_2'.format(name), _scope='{}_2'.format(name), _reuse=None) + ) + conv_bn_blocks.append( + tf.layers.BatchNormalization(axis=self._bn_axis, momentum=BN_MOMENTUM, epsilon=BN_EPSILON, fused=USE_FUSED_BN, + name='{}_bn2'.format(name), _scope='{}_bn2'.format(name), _reuse=None) + ) + conv_bn_blocks.append( + ReLuLayer('{}_relu2'.format(name), _scope='{}_relu2'.format(name), _reuse=None) + ) + return conv_bn_blocks + +def multibox_head(feature_layers, num_classes, num_anchors_depth_per_layer, data_format='channels_first'): + with tf.variable_scope('multibox_head'): + cls_preds = [] + loc_preds = [] + for ind, feat in enumerate(feature_layers): + loc_preds.append(tf.layers.conv2d(feat, num_anchors_depth_per_layer[ind] * 4, (3, 3), use_bias=True, + name='loc_{}'.format(ind), strides=(1, 1), + padding='same', data_format=data_format, activation=None, + kernel_initializer=tf.glorot_uniform_initializer(), + bias_initializer=tf.zeros_initializer())) + cls_preds.append(tf.layers.conv2d(feat, num_anchors_depth_per_layer[ind] * num_classes, (3, 3), use_bias=True, + name='cls_{}'.format(ind), strides=(1, 1), + padding='same', data_format=data_format, activation=None, + kernel_initializer=tf.glorot_uniform_initializer(), + bias_initializer=tf.zeros_initializer())) + + return loc_preds, cls_preds + + diff --git a/utils/external/SSD.TensorFlow/preprocessing/preprocessing_unittest.py b/utils/external/SSD.TensorFlow/preprocessing/preprocessing_unittest.py new file mode 100644 index 0000000..92e4167 --- /dev/null +++ b/utils/external/SSD.TensorFlow/preprocessing/preprocessing_unittest.py @@ -0,0 +1,131 @@ +# Copyright 2018 Changan Wang + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +import tensorflow as tf +from scipy.misc import imread, imsave, imshow, imresize +import numpy as np +import sys; sys.path.insert(0, ".") +from utility import draw_toolbox +import ssd_preprocessing + +slim = tf.contrib.slim + +def save_image_with_bbox(image, labels_, scores_, bboxes_): + if not hasattr(save_image_with_bbox, "counter"): + save_image_with_bbox.counter = 0 # it doesn't exist yet, so initialize it + save_image_with_bbox.counter += 1 + + img_to_draw = np.copy(image) + + img_to_draw = draw_toolbox.bboxes_draw_on_img(img_to_draw, labels_, scores_, bboxes_, thickness=2) + imsave(os.path.join('./debug/{}.jpg').format(save_image_with_bbox.counter), img_to_draw) + return save_image_with_bbox.counter + +def slim_get_split(file_pattern='{}_????'): + # Features in Pascal VOC TFRecords. + keys_to_features = { + 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), + 'image/format': tf.FixedLenFeature((), tf.string, default_value='jpeg'), + 'image/filename': tf.FixedLenFeature((), tf.string, default_value=''), + 'image/height': tf.FixedLenFeature([1], tf.int64), + 'image/width': tf.FixedLenFeature([1], tf.int64), + 'image/channels': tf.FixedLenFeature([1], tf.int64), + 'image/shape': tf.FixedLenFeature([3], tf.int64), + 'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/label': tf.VarLenFeature(dtype=tf.int64), + 'image/object/bbox/difficult': tf.VarLenFeature(dtype=tf.int64), + 'image/object/bbox/truncated': tf.VarLenFeature(dtype=tf.int64), + } + items_to_handlers = { + 'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'), + 'filename': slim.tfexample_decoder.Tensor('image/filename'), + 'shape': slim.tfexample_decoder.Tensor('image/shape'), + 'object/bbox': slim.tfexample_decoder.BoundingBox( + ['ymin', 'xmin', 'ymax', 'xmax'], 'image/object/bbox/'), + 'object/label': slim.tfexample_decoder.Tensor('image/object/bbox/label'), + 'object/difficult': slim.tfexample_decoder.Tensor('image/object/bbox/difficult'), + 'object/truncated': slim.tfexample_decoder.Tensor('image/object/bbox/truncated'), + } + decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) + + dataset = slim.dataset.Dataset( + data_sources=file_pattern, + reader=tf.TFRecordReader, + decoder=decoder, + num_samples=100, + items_to_descriptions=None, + num_classes=21, + labels_to_names=None) + + with tf.name_scope('dataset_data_provider'): + provider = slim.dataset_data_provider.DatasetDataProvider( + dataset, + num_readers=2, + common_queue_capacity=32, + common_queue_min=8, + shuffle=True, + num_epochs=1) + + [org_image, filename, shape, glabels_raw, gbboxes_raw, isdifficult] = provider.get(['image', 'filename', 'shape', + 'object/label', + 'object/bbox', + 'object/difficult']) + image, glabels, gbboxes = ssd_preprocessing.preprocess_image(org_image, glabels_raw, gbboxes_raw, [300, 300], is_training=True, data_format='channels_first', output_rgb=True) + + image = tf.transpose(image, perm=(1, 2, 0)) + save_image_op = tf.py_func(save_image_with_bbox, + [ssd_preprocessing.unwhiten_image(image), + tf.clip_by_value(glabels, 0, tf.int64.max), + tf.ones_like(glabels), + gbboxes], + tf.int64, stateful=True) + return save_image_op + +if __name__ == '__main__': + save_image_op = slim_get_split('/media/rs/7A0EE8880EE83EAF/Detections/SSD/dataset/tfrecords/*') + # Create the graph, etc. + init_op = tf.group([tf.local_variables_initializer(), tf.local_variables_initializer(), tf.tables_initializer()]) + + # Create a session for running operations in the Graph. + sess = tf.Session() + # Initialize the variables (like the epoch counter). + sess.run(init_op) + + # Start input enqueue threads. + coord = tf.train.Coordinator() + threads = tf.train.start_queue_runners(sess=sess, coord=coord) + + try: + while not coord.should_stop(): + # Run training steps or whatever + print(sess.run(save_image_op)) + + except tf.errors.OutOfRangeError: + print('Done training -- epoch limit reached') + finally: + # When done, ask the threads to stop. + coord.request_stop() + + # Wait for threads to finish. + coord.join(threads) + sess.close() diff --git a/utils/external/SSD.TensorFlow/preprocessing/ssd_preprocessing.py b/utils/external/SSD.TensorFlow/preprocessing/ssd_preprocessing.py new file mode 100644 index 0000000..3ab8dcc --- /dev/null +++ b/utils/external/SSD.TensorFlow/preprocessing/ssd_preprocessing.py @@ -0,0 +1,521 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Provides utilities to preprocess images. + +The preprocessing steps for VGG were introduced in the following technical +report: + + Very Deep Convolutional Networks For Large-Scale Image Recognition + Karen Simonyan and Andrew Zisserman + arXiv technical report, 2015 + PDF: http://arxiv.org/pdf/1409.1556.pdf + ILSVRC 2014 Slides: http://www.robots.ox.ac.uk/~karen/pdf/ILSVRC_2014.pdf + CC-BY-4.0 + +More information can be obtained from the VGG website: +www.robots.ox.ac.uk/~vgg/research/very_deep/ +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +from tensorflow.python.ops import control_flow_ops + +slim = tf.contrib.slim + +_R_MEAN = 123.68 +_G_MEAN = 116.78 +_B_MEAN = 103.94 + +def _ImageDimensions(image, rank = 3): + """Returns the dimensions of an image tensor. + + Args: + image: A rank-D Tensor. For 3-D of shape: `[height, width, channels]`. + rank: The expected rank of the image + + Returns: + A list of corresponding to the dimensions of the + input image. Dimensions that are statically known are python integers, + otherwise they are integer scalar tensors. + """ + if image.get_shape().is_fully_defined(): + return image.get_shape().as_list() + else: + static_shape = image.get_shape().with_rank(rank).as_list() + dynamic_shape = tf.unstack(tf.shape(image), rank) + return [s if s is not None else d + for s, d in zip(static_shape, dynamic_shape)] + +def apply_with_random_selector(x, func, num_cases): + """Computes func(x, sel), with sel sampled from [0...num_cases-1]. + + Args: + x: input Tensor. + func: Python function to apply. + num_cases: Python int32, number of cases to sample sel from. + + Returns: + The result of func(x, sel), where func receives the value of the + selector as a python integer, but sel is sampled dynamically. + """ + sel = tf.random_uniform([], maxval=num_cases, dtype=tf.int32) + # Pass the real x only to one of the func calls. + return control_flow_ops.merge([ + func(control_flow_ops.switch(x, tf.equal(sel, case))[1], case) + for case in range(num_cases)])[0] + + +def distort_color(image, color_ordering=0, fast_mode=True, scope=None): + """Distort the color of a Tensor image. + + Each color distortion is non-commutative and thus ordering of the color ops + matters. Ideally we would randomly permute the ordering of the color ops. + Rather then adding that level of complication, we select a distinct ordering + of color ops for each preprocessing thread. + + Args: + image: 3-D Tensor containing single image in [0, 1]. + color_ordering: Python int, a type of distortion (valid values: 0-3). + fast_mode: Avoids slower ops (random_hue and random_contrast) + scope: Optional scope for name_scope. + Returns: + 3-D Tensor color-distorted image on range [0, 1] + Raises: + ValueError: if color_ordering not in [0, 3] + """ + with tf.name_scope(scope, 'distort_color', [image]): + if fast_mode: + if color_ordering == 0: + image = tf.image.random_brightness(image, max_delta=32. / 255.) + image = tf.image.random_saturation(image, lower=0.5, upper=1.5) + else: + image = tf.image.random_saturation(image, lower=0.5, upper=1.5) + image = tf.image.random_brightness(image, max_delta=32. / 255.) + else: + if color_ordering == 0: + image = tf.image.random_brightness(image, max_delta=32. / 255.) + image = tf.image.random_saturation(image, lower=0.5, upper=1.5) + image = tf.image.random_hue(image, max_delta=0.2) + image = tf.image.random_contrast(image, lower=0.5, upper=1.5) + elif color_ordering == 1: + image = tf.image.random_saturation(image, lower=0.5, upper=1.5) + image = tf.image.random_brightness(image, max_delta=32. / 255.) + image = tf.image.random_contrast(image, lower=0.5, upper=1.5) + image = tf.image.random_hue(image, max_delta=0.2) + elif color_ordering == 2: + image = tf.image.random_contrast(image, lower=0.5, upper=1.5) + image = tf.image.random_hue(image, max_delta=0.2) + image = tf.image.random_brightness(image, max_delta=32. / 255.) + image = tf.image.random_saturation(image, lower=0.5, upper=1.5) + elif color_ordering == 3: + image = tf.image.random_hue(image, max_delta=0.2) + image = tf.image.random_saturation(image, lower=0.5, upper=1.5) + image = tf.image.random_contrast(image, lower=0.5, upper=1.5) + image = tf.image.random_brightness(image, max_delta=32. / 255.) + else: + raise ValueError('color_ordering must be in [0, 3]') + + # The random_* ops do not necessarily clamp. + return tf.clip_by_value(image, 0.0, 1.0) + +def ssd_random_sample_patch(image, labels, bboxes, ratio_list=[0.1, 0.3, 0.5, 0.7, 0.9, 1.], name=None): + '''ssd_random_sample_patch. + select one min_iou + sample _width and _height from [0-width] and [0-height] + check if the aspect ratio between 0.5-2. + select left_top point from (width - _width, height - _height) + check if this bbox has a min_iou with all ground_truth bboxes + keep ground_truth those center is in this sampled patch, if none then try again + ''' + def sample_width_height(width, height): + with tf.name_scope('sample_width_height'): + index = 0 + max_attempt = 10 + sampled_width, sampled_height = width, height + + def condition(index, sampled_width, sampled_height, width, height): + return tf.logical_or(tf.logical_and(tf.logical_or(tf.greater(sampled_width, sampled_height * 2), + tf.greater(sampled_height, sampled_width * 2)), + tf.less(index, max_attempt)), + tf.less(index, 1)) + + def body(index, sampled_width, sampled_height, width, height): + sampled_width = tf.random_uniform([1], minval=0.3, maxval=0.999, dtype=tf.float32)[0] * width + sampled_height = tf.random_uniform([1], minval=0.3, maxval=0.999, dtype=tf.float32)[0] *height + + return index+1, sampled_width, sampled_height, width, height + + [index, sampled_width, sampled_height, _, _] = tf.while_loop(condition, body, + [index, sampled_width, sampled_height, width, height], parallel_iterations=4, back_prop=False, swap_memory=True) + + return tf.cast(sampled_width, tf.int32), tf.cast(sampled_height, tf.int32) + + def jaccard_with_anchors(roi, bboxes): + with tf.name_scope('jaccard_with_anchors'): + int_ymin = tf.maximum(roi[0], bboxes[:, 0]) + int_xmin = tf.maximum(roi[1], bboxes[:, 1]) + int_ymax = tf.minimum(roi[2], bboxes[:, 2]) + int_xmax = tf.minimum(roi[3], bboxes[:, 3]) + h = tf.maximum(int_ymax - int_ymin, 0.) + w = tf.maximum(int_xmax - int_xmin, 0.) + inter_vol = h * w + union_vol = (roi[3] - roi[1]) * (roi[2] - roi[0]) + ((bboxes[:, 2] - bboxes[:, 0]) * (bboxes[:, 3] - bboxes[:, 1]) - inter_vol) + jaccard = tf.div(inter_vol, union_vol) + return jaccard + + def areas(bboxes): + with tf.name_scope('bboxes_areas'): + vol = (bboxes[:, 3] - bboxes[:, 1]) * (bboxes[:, 2] - bboxes[:, 0]) + return vol + + def check_roi_center(width, height, labels, bboxes): + with tf.name_scope('check_roi_center'): + index = 0 + max_attempt = 20 + roi = [0., 0., 0., 0.] + float_width = tf.cast(width, tf.float32) + float_height = tf.cast(height, tf.float32) + mask = tf.cast(tf.zeros_like(labels, dtype=tf.uint8), tf.bool) + center_x, center_y = (bboxes[:, 1] + bboxes[:, 3]) / 2, (bboxes[:, 0] + bboxes[:, 2]) / 2 + + def condition(index, roi, mask): + return tf.logical_or(tf.logical_and(tf.reduce_sum(tf.cast(mask, tf.int32)) < 1, + tf.less(index, max_attempt)), + tf.less(index, 1)) + + def body(index, roi, mask): + sampled_width, sampled_height = sample_width_height(float_width, float_height) + + x = tf.random_uniform([], minval=0, maxval=width - sampled_width, dtype=tf.int32) + y = tf.random_uniform([], minval=0, maxval=height - sampled_height, dtype=tf.int32) + + roi = [tf.cast(y, tf.float32) / float_height, + tf.cast(x, tf.float32) / float_width, + tf.cast(y + sampled_height, tf.float32) / float_height, + tf.cast(x + sampled_width, tf.float32) / float_width] + + mask_min = tf.logical_and(tf.greater(center_y, roi[0]), tf.greater(center_x, roi[1])) + mask_max = tf.logical_and(tf.less(center_y, roi[2]), tf.less(center_x, roi[3])) + mask = tf.logical_and(mask_min, mask_max) + + return index + 1, roi, mask + + [index, roi, mask] = tf.while_loop(condition, body, [index, roi, mask], parallel_iterations=10, back_prop=False, swap_memory=True) + + mask_labels = tf.boolean_mask(labels, mask) + mask_bboxes = tf.boolean_mask(bboxes, mask) + + return roi, mask_labels, mask_bboxes + def check_roi_overlap(width, height, labels, bboxes, min_iou): + with tf.name_scope('check_roi_overlap'): + index = 0 + max_attempt = 50 + roi = [0., 0., 1., 1.] + mask_labels = labels + mask_bboxes = bboxes + + def condition(index, roi, mask_labels, mask_bboxes): + return tf.logical_or(tf.logical_or(tf.logical_and(tf.reduce_sum(tf.cast(jaccard_with_anchors(roi, mask_bboxes) < min_iou, tf.int32)) > 0, + tf.less(index, max_attempt)), + tf.less(index, 1)), + tf.less(tf.shape(mask_labels)[0], 1)) + + def body(index, roi, mask_labels, mask_bboxes): + roi, mask_labels, mask_bboxes = check_roi_center(width, height, labels, bboxes) + return index+1, roi, mask_labels, mask_bboxes + + [index, roi, mask_labels, mask_bboxes] = tf.while_loop(condition, body, [index, roi, mask_labels, mask_bboxes], parallel_iterations=16, back_prop=False, swap_memory=True) + + return tf.cond(tf.greater(tf.shape(mask_labels)[0], 0), + lambda : (tf.cast([roi[0] * tf.cast(height, tf.float32), + roi[1] * tf.cast(width, tf.float32), + (roi[2] - roi[0]) * tf.cast(height, tf.float32), + (roi[3] - roi[1]) * tf.cast(width, tf.float32)], tf.int32), mask_labels, mask_bboxes), + lambda : (tf.cast([0, 0, height, width], tf.int32), labels, bboxes)) + + + def sample_patch(image, labels, bboxes, min_iou): + with tf.name_scope('sample_patch'): + height, width, depth = _ImageDimensions(image, rank=3) + + roi_slice_range, mask_labels, mask_bboxes = check_roi_overlap(width, height, labels, bboxes, min_iou) + + scale = tf.cast(tf.stack([height, width, height, width]), mask_bboxes.dtype) + mask_bboxes = mask_bboxes * scale + + # Add offset. + offset = tf.cast(tf.stack([roi_slice_range[0], roi_slice_range[1], roi_slice_range[0], roi_slice_range[1]]), mask_bboxes.dtype) + mask_bboxes = mask_bboxes - offset + + cliped_ymin = tf.maximum(0., mask_bboxes[:, 0]) + cliped_xmin = tf.maximum(0., mask_bboxes[:, 1]) + cliped_ymax = tf.minimum(tf.cast(roi_slice_range[2], tf.float32), mask_bboxes[:, 2]) + cliped_xmax = tf.minimum(tf.cast(roi_slice_range[3], tf.float32), mask_bboxes[:, 3]) + + mask_bboxes = tf.stack([cliped_ymin, cliped_xmin, cliped_ymax, cliped_xmax], axis=-1) + # Rescale to target dimension. + scale = tf.cast(tf.stack([roi_slice_range[2], roi_slice_range[3], + roi_slice_range[2], roi_slice_range[3]]), mask_bboxes.dtype) + + return tf.cond(tf.logical_or(tf.less(roi_slice_range[2], 1), tf.less(roi_slice_range[3], 1)), + lambda: (image, labels, bboxes), + lambda: (tf.slice(image, [roi_slice_range[0], roi_slice_range[1], 0], [roi_slice_range[2], roi_slice_range[3], -1]), + mask_labels, mask_bboxes / scale)) + + with tf.name_scope('ssd_random_sample_patch'): + image = tf.convert_to_tensor(image, name='image') + + min_iou_list = tf.convert_to_tensor(ratio_list) + samples_min_iou = tf.multinomial(tf.log([[1. / len(ratio_list)] * len(ratio_list)]), 1) + + sampled_min_iou = min_iou_list[tf.cast(samples_min_iou[0][0], tf.int32)] + + return tf.cond(tf.less(sampled_min_iou, 1.), lambda: sample_patch(image, labels, bboxes, sampled_min_iou), lambda: (image, labels, bboxes)) + +def ssd_random_expand(image, bboxes, ratio=2., name=None): + with tf.name_scope('ssd_random_expand'): + image = tf.convert_to_tensor(image, name='image') + if image.get_shape().ndims != 3: + raise ValueError('\'image\' must have 3 dimensions.') + + height, width, depth = _ImageDimensions(image, rank=3) + + float_height, float_width = tf.to_float(height), tf.to_float(width) + + canvas_width, canvas_height = tf.to_int32(float_width * ratio), tf.to_int32(float_height * ratio) + + mean_color_of_image = [_R_MEAN/255., _G_MEAN/255., _B_MEAN/255.]#tf.reduce_mean(tf.reshape(image, [-1, 3]), 0) + + x = tf.random_uniform([], minval=0, maxval=canvas_width - width, dtype=tf.int32) + y = tf.random_uniform([], minval=0, maxval=canvas_height - height, dtype=tf.int32) + + paddings = tf.convert_to_tensor([[y, canvas_height - height - y], [x, canvas_width - width - x]]) + + big_canvas = tf.stack([tf.pad(image[:, :, 0], paddings, "CONSTANT", constant_values = mean_color_of_image[0]), + tf.pad(image[:, :, 1], paddings, "CONSTANT", constant_values = mean_color_of_image[1]), + tf.pad(image[:, :, 2], paddings, "CONSTANT", constant_values = mean_color_of_image[2])], axis=-1) + + scale = tf.cast(tf.stack([height, width, height, width]), bboxes.dtype) + absolute_bboxes = bboxes * scale + tf.cast(tf.stack([y, x, y, x]), bboxes.dtype) + + return big_canvas, absolute_bboxes / tf.cast(tf.stack([canvas_height, canvas_width, canvas_height, canvas_width]), bboxes.dtype) + +# def ssd_random_sample_patch_wrapper(image, labels, bboxes): +# with tf.name_scope('ssd_random_sample_patch_wrapper'): +# orgi_image, orgi_labels, orgi_bboxes = image, labels, bboxes +# def check_bboxes(bboxes): +# areas = (bboxes[:, 3] - bboxes[:, 1]) * (bboxes[:, 2] - bboxes[:, 0]) +# return tf.logical_and(tf.logical_and(areas < 0.9, areas > 0.001), +# tf.logical_and((bboxes[:, 3] - bboxes[:, 1]) > 0.025, (bboxes[:, 2] - bboxes[:, 0]) > 0.025)) + +# index = 0 +# max_attempt = 3 +# def condition(index, image, labels, bboxes): +# return tf.logical_or(tf.logical_and(tf.reduce_sum(tf.cast(check_bboxes(bboxes), tf.int64)) < 1, tf.less(index, max_attempt)), tf.less(index, 1)) + +# def body(index, image, labels, bboxes): +# image, bboxes = tf.cond(tf.random_uniform([], minval=0., maxval=1., dtype=tf.float32) < 0.5, +# lambda: (image, bboxes), +# lambda: ssd_random_expand(image, bboxes, tf.random_uniform([1], minval=1.1, maxval=4., dtype=tf.float32)[0])) +# # Distort image and bounding boxes. +# random_sample_image, labels, bboxes = ssd_random_sample_patch(image, labels, bboxes, ratio_list=[-0.1, 0.1, 0.3, 0.5, 0.7, 0.9, 1.]) +# random_sample_image.set_shape([None, None, 3]) +# return index+1, random_sample_image, labels, bboxes + +# [index, image, labels, bboxes] = tf.while_loop(condition, body, [index, orgi_image, orgi_labels, orgi_bboxes], parallel_iterations=4, back_prop=False, swap_memory=True) + +# valid_mask = check_bboxes(bboxes) +# labels, bboxes = tf.boolean_mask(labels, valid_mask), tf.boolean_mask(bboxes, valid_mask) +# return tf.cond(tf.less(index, max_attempt), +# lambda : (image, labels, bboxes), +# lambda : (orgi_image, orgi_labels, orgi_bboxes)) + +def ssd_random_sample_patch_wrapper(image, labels, bboxes): + with tf.name_scope('ssd_random_sample_patch_wrapper'): + orgi_image, orgi_labels, orgi_bboxes = image, labels, bboxes + def check_bboxes(bboxes): + areas = (bboxes[:, 3] - bboxes[:, 1]) * (bboxes[:, 2] - bboxes[:, 0]) + return tf.logical_and(tf.logical_and(areas < 0.9, areas > 0.001), + tf.logical_and((bboxes[:, 3] - bboxes[:, 1]) > 0.025, (bboxes[:, 2] - bboxes[:, 0]) > 0.025)) + + index = 0 + max_attempt = 3 + def condition(index, image, labels, bboxes, orgi_image, orgi_labels, orgi_bboxes): + return tf.logical_or(tf.logical_and(tf.reduce_sum(tf.cast(check_bboxes(bboxes), tf.int64)) < 1, tf.less(index, max_attempt)), tf.less(index, 1)) + + def body(index, image, labels, bboxes, orgi_image, orgi_labels, orgi_bboxes): + image, bboxes = tf.cond(tf.random_uniform([], minval=0., maxval=1., dtype=tf.float32) < 0.5, + lambda: (orgi_image, orgi_bboxes), + lambda: ssd_random_expand(orgi_image, orgi_bboxes, tf.random_uniform([1], minval=1.1, maxval=4., dtype=tf.float32)[0])) + # Distort image and bounding boxes. + random_sample_image, labels, bboxes = ssd_random_sample_patch(image, orgi_labels, bboxes, ratio_list=[-0.1, 0.1, 0.3, 0.5, 0.7, 0.9, 1.]) + random_sample_image.set_shape([None, None, 3]) + return index+1, random_sample_image, labels, bboxes, orgi_image, orgi_labels, orgi_bboxes + + [index, image, labels, bboxes, orgi_image, orgi_labels, orgi_bboxes] = tf.while_loop(condition, body, [index, image, labels, bboxes, orgi_image, orgi_labels, orgi_bboxes], parallel_iterations=4, back_prop=False, swap_memory=True) + + valid_mask = check_bboxes(bboxes) + labels, bboxes = tf.boolean_mask(labels, valid_mask), tf.boolean_mask(bboxes, valid_mask) + return tf.cond(tf.less(index, max_attempt), + lambda : (image, labels, bboxes), + lambda : (orgi_image, orgi_labels, orgi_bboxes)) + +def _mean_image_subtraction(image, means): + """Subtracts the given means from each image channel. + + For example: + means = [123.68, 116.779, 103.939] + image = _mean_image_subtraction(image, means) + + Note that the rank of `image` must be known. + + Args: + image: a tensor of size [height, width, C]. + means: a C-vector of values to subtract from each channel. + + Returns: + the centered image. + + Raises: + ValueError: If the rank of `image` is unknown, if `image` has a rank other + than three or if the number of channels in `image` doesn't match the + number of values in `means`. + """ + if image.get_shape().ndims != 3: + raise ValueError('Input must be of size [height, width, C>0]') + num_channels = image.get_shape().as_list()[-1] + if len(means) != num_channels: + raise ValueError('len(means) must match the number of channels') + + channels = tf.split(axis=2, num_or_size_splits=num_channels, value=image) + for i in range(num_channels): + channels[i] -= means[i] + return tf.concat(axis=2, values=channels) + +def unwhiten_image(image): + means=[_R_MEAN, _G_MEAN, _B_MEAN] + num_channels = image.get_shape().as_list()[-1] + channels = tf.split(axis=2, num_or_size_splits=num_channels, value=image) + for i in range(num_channels): + channels[i] += means[i] + return tf.concat(axis=2, values=channels) + +def random_flip_left_right(image, bboxes): + with tf.name_scope('random_flip_left_right'): + uniform_random = tf.random_uniform([], 0, 1.0) + mirror_cond = tf.less(uniform_random, .5) + # Flip image. + result = tf.cond(mirror_cond, lambda: tf.image.flip_left_right(image), lambda: image) + # Flip bboxes. + mirror_bboxes = tf.stack([bboxes[:, 0], 1 - bboxes[:, 3], + bboxes[:, 2], 1 - bboxes[:, 1]], axis=-1) + bboxes = tf.cond(mirror_cond, lambda: mirror_bboxes, lambda: bboxes) + return result, bboxes + +def preprocess_for_train(image, labels, bboxes, out_shape, data_format='channels_first', scope='ssd_preprocessing_train', output_rgb=True): + """Preprocesses the given image for training. + + Args: + image: A `Tensor` representing an image of arbitrary size. + labels: A `Tensor` containing all labels for all bboxes of this image. + bboxes: A `Tensor` containing all bboxes of this image, in range [0., 1.] with shape [num_bboxes, 4]. + out_shape: The height and width of the image after preprocessing. + data_format: The data_format of the desired output image. + Returns: + A preprocessed image. + """ + with tf.name_scope(scope, 'ssd_preprocessing_train', [image, labels, bboxes]): + if image.get_shape().ndims != 3: + raise ValueError('Input must be of size [height, width, C>0]') + # Convert to float scaled [0, 1]. + orig_dtype = image.dtype + if orig_dtype != tf.float32: + image = tf.image.convert_image_dtype(image, dtype=tf.float32) + + # Randomly distort the colors. There are 4 ways to do it. + distort_image = apply_with_random_selector(image, + lambda x, ordering: distort_color(x, ordering, True), + num_cases=4) + + random_sample_image, labels, bboxes = ssd_random_sample_patch_wrapper(distort_image, labels, bboxes) + # image, bboxes = tf.cond(tf.random_uniform([1], minval=0., maxval=1., dtype=tf.float32)[0] < 0.25, + # lambda: (image, bboxes), + # lambda: ssd_random_expand(image, bboxes, tf.random_uniform([1], minval=2, maxval=4, dtype=tf.int32)[0])) + + # # Distort image and bounding boxes. + # random_sample_image, labels, bboxes = ssd_random_sample_patch(image, labels, bboxes, ratio_list=[0.1, 0.3, 0.5, 0.7, 0.9, 1.]) + + # Randomly flip the image horizontally. + random_sample_flip_image, bboxes = random_flip_left_right(random_sample_image, bboxes) + # Rescale to VGG input scale. + random_sample_flip_resized_image = tf.image.resize_images(random_sample_flip_image, out_shape, method=tf.image.ResizeMethod.BILINEAR, align_corners=False) + random_sample_flip_resized_image.set_shape([None, None, 3]) + + final_image = tf.to_float(tf.image.convert_image_dtype(random_sample_flip_resized_image, orig_dtype, saturate=True)) + final_image = _mean_image_subtraction(final_image, [_R_MEAN, _G_MEAN, _B_MEAN]) + + final_image.set_shape(out_shape + [3]) + if not output_rgb: + image_channels = tf.unstack(final_image, axis=-1, name='split_rgb') + final_image = tf.stack([image_channels[2], image_channels[1], image_channels[0]], axis=-1, name='merge_bgr') + if data_format == 'channels_first': + final_image = tf.transpose(final_image, perm=(2, 0, 1)) + return final_image, labels, bboxes + +def preprocess_for_eval(image, out_shape, data_format='channels_first', scope='ssd_preprocessing_eval', output_rgb=True): + """Preprocesses the given image for evaluation. + + Args: + image: A `Tensor` representing an image of arbitrary size. + out_shape: The height and width of the image after preprocessing. + data_format: The data_format of the desired output image. + Returns: + A preprocessed image. + """ + with tf.name_scope(scope, 'ssd_preprocessing_eval', [image]): + image = tf.to_float(image) + image = tf.image.resize_images(image, out_shape, method=tf.image.ResizeMethod.BILINEAR, align_corners=False) + image.set_shape(out_shape + [3]) + + image = _mean_image_subtraction(image, [_R_MEAN, _G_MEAN, _B_MEAN]) + if not output_rgb: + image_channels = tf.unstack(image, axis=-1, name='split_rgb') + image = tf.stack([image_channels[2], image_channels[1], image_channels[0]], axis=-1, name='merge_bgr') + # Image data format. + if data_format == 'channels_first': + image = tf.transpose(image, perm=(2, 0, 1)) + return image + +def preprocess_image(image, labels, bboxes, out_shape, is_training=False, data_format='channels_first', output_rgb=True): + """Preprocesses the given image. + + Args: + image: A `Tensor` representing an image of arbitrary size. + labels: A `Tensor` containing all labels for all bboxes of this image. + bboxes: A `Tensor` containing all bboxes of this image, in range [0., 1.] with shape [num_bboxes, 4]. + out_shape: The height and width of the image after preprocessing. + is_training: Wether we are in training phase. + data_format: The data_format of the desired output image. + + Returns: + A preprocessed image. + """ + if is_training: + return preprocess_for_train(image, labels, bboxes, out_shape, data_format=data_format, output_rgb=output_rgb) + else: + return preprocess_for_eval(image, out_shape, data_format=data_format, output_rgb=output_rgb) diff --git a/utils/external/SSD.TensorFlow/simple_ssd_demo.py b/utils/external/SSD.TensorFlow/simple_ssd_demo.py new file mode 100644 index 0000000..67540bc --- /dev/null +++ b/utils/external/SSD.TensorFlow/simple_ssd_demo.py @@ -0,0 +1,220 @@ +# Copyright 2018 Changan Wang + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import sys + +import tensorflow as tf +from scipy.misc import imread, imsave, imshow, imresize +import numpy as np + +from net import ssd_net + +from dataset import dataset_common +from preprocessing import ssd_preprocessing +from utility import anchor_manipulator +from utility import draw_toolbox + +# scaffold related configuration +tf.app.flags.DEFINE_integer( + 'num_classes', 21, 'Number of classes to use in the dataset.') +# model related configuration +tf.app.flags.DEFINE_integer( + 'train_image_size', 300, + 'The size of the input image for the model to use.') +tf.app.flags.DEFINE_string( + 'data_format', 'channels_last', # 'channels_first' or 'channels_last' + 'A flag to override the data format used in the model. channels_first ' + 'provides a performance boost on GPU but is not always compatible ' + 'with CPU. If left unspecified, the data format will be chosen ' + 'automatically based on whether TensorFlow was built for CPU or GPU.') +tf.app.flags.DEFINE_float( + 'select_threshold', 0.2, 'Class-specific confidence score threshold for selecting a box.') +tf.app.flags.DEFINE_float( + 'min_size', 0.03, 'The min size of bboxes to keep.') +tf.app.flags.DEFINE_float( + 'nms_threshold', 0.45, 'Matching threshold in NMS algorithm.') +tf.app.flags.DEFINE_integer( + 'nms_topk', 20, 'Number of total object to keep after NMS.') +tf.app.flags.DEFINE_integer( + 'keep_topk', 200, 'Number of total object to keep for each image before nms.') +# checkpoint related configuration +tf.app.flags.DEFINE_string( + 'checkpoint_path', './logs', + 'The path to a checkpoint from which to fine-tune.') +tf.app.flags.DEFINE_string( + 'model_scope', 'ssd300', + 'Model scope name used to replace the name_scope in checkpoint.') + +FLAGS = tf.app.flags.FLAGS +#CUDA_VISIBLE_DEVICES + +def get_checkpoint(): + if tf.gfile.IsDirectory(FLAGS.checkpoint_path): + checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path) + else: + checkpoint_path = FLAGS.checkpoint_path + + return checkpoint_path + +def select_bboxes(scores_pred, bboxes_pred, num_classes, select_threshold): + selected_bboxes = {} + selected_scores = {} + with tf.name_scope('select_bboxes', [scores_pred, bboxes_pred]): + for class_ind in range(1, num_classes): + class_scores = scores_pred[:, class_ind] + + select_mask = class_scores > select_threshold + select_mask = tf.cast(select_mask, tf.float32) + selected_bboxes[class_ind] = tf.multiply(bboxes_pred, tf.expand_dims(select_mask, axis=-1)) + selected_scores[class_ind] = tf.multiply(class_scores, select_mask) + + return selected_bboxes, selected_scores + +def clip_bboxes(ymin, xmin, ymax, xmax, name): + with tf.name_scope(name, 'clip_bboxes', [ymin, xmin, ymax, xmax]): + ymin = tf.maximum(ymin, 0.) + xmin = tf.maximum(xmin, 0.) + ymax = tf.minimum(ymax, 1.) + xmax = tf.minimum(xmax, 1.) + + ymin = tf.minimum(ymin, ymax) + xmin = tf.minimum(xmin, xmax) + + return ymin, xmin, ymax, xmax + +def filter_bboxes(scores_pred, ymin, xmin, ymax, xmax, min_size, name): + with tf.name_scope(name, 'filter_bboxes', [scores_pred, ymin, xmin, ymax, xmax]): + width = xmax - xmin + height = ymax - ymin + + filter_mask = tf.logical_and(width > min_size, height > min_size) + + filter_mask = tf.cast(filter_mask, tf.float32) + return tf.multiply(ymin, filter_mask), tf.multiply(xmin, filter_mask), \ + tf.multiply(ymax, filter_mask), tf.multiply(xmax, filter_mask), tf.multiply(scores_pred, filter_mask) + +def sort_bboxes(scores_pred, ymin, xmin, ymax, xmax, keep_topk, name): + with tf.name_scope(name, 'sort_bboxes', [scores_pred, ymin, xmin, ymax, xmax]): + cur_bboxes = tf.shape(scores_pred)[0] + scores, idxes = tf.nn.top_k(scores_pred, k=tf.minimum(keep_topk, cur_bboxes), sorted=True) + + ymin, xmin, ymax, xmax = tf.gather(ymin, idxes), tf.gather(xmin, idxes), tf.gather(ymax, idxes), tf.gather(xmax, idxes) + + paddings_scores = tf.expand_dims(tf.stack([0, tf.maximum(keep_topk-cur_bboxes, 0)], axis=0), axis=0) + + return tf.pad(ymin, paddings_scores, "CONSTANT"), tf.pad(xmin, paddings_scores, "CONSTANT"),\ + tf.pad(ymax, paddings_scores, "CONSTANT"), tf.pad(xmax, paddings_scores, "CONSTANT"),\ + tf.pad(scores, paddings_scores, "CONSTANT") + +def nms_bboxes(scores_pred, bboxes_pred, nms_topk, nms_threshold, name): + with tf.name_scope(name, 'nms_bboxes', [scores_pred, bboxes_pred]): + idxes = tf.image.non_max_suppression(bboxes_pred, scores_pred, nms_topk, nms_threshold) + return tf.gather(scores_pred, idxes), tf.gather(bboxes_pred, idxes) + +def parse_by_class(cls_pred, bboxes_pred, num_classes, select_threshold, min_size, keep_topk, nms_topk, nms_threshold): + with tf.name_scope('select_bboxes', [cls_pred, bboxes_pred]): + scores_pred = tf.nn.softmax(cls_pred) + selected_bboxes, selected_scores = select_bboxes(scores_pred, bboxes_pred, num_classes, select_threshold) + for class_ind in range(1, num_classes): + ymin, xmin, ymax, xmax = tf.unstack(selected_bboxes[class_ind], 4, axis=-1) + #ymin, xmin, ymax, xmax = tf.squeeze(ymin), tf.squeeze(xmin), tf.squeeze(ymax), tf.squeeze(xmax) + ymin, xmin, ymax, xmax = clip_bboxes(ymin, xmin, ymax, xmax, 'clip_bboxes_{}'.format(class_ind)) + ymin, xmin, ymax, xmax, selected_scores[class_ind] = filter_bboxes(selected_scores[class_ind], + ymin, xmin, ymax, xmax, min_size, 'filter_bboxes_{}'.format(class_ind)) + ymin, xmin, ymax, xmax, selected_scores[class_ind] = sort_bboxes(selected_scores[class_ind], + ymin, xmin, ymax, xmax, keep_topk, 'sort_bboxes_{}'.format(class_ind)) + selected_bboxes[class_ind] = tf.stack([ymin, xmin, ymax, xmax], axis=-1) + selected_scores[class_ind], selected_bboxes[class_ind] = nms_bboxes(selected_scores[class_ind], selected_bboxes[class_ind], nms_topk, nms_threshold, 'nms_bboxes_{}'.format(class_ind)) + + return selected_bboxes, selected_scores + +def main(_): + with tf.Graph().as_default(): + out_shape = [FLAGS.train_image_size] * 2 + + image_input = tf.placeholder(tf.uint8, shape=(None, None, 3)) + shape_input = tf.placeholder(tf.int32, shape=(2,)) + + features = ssd_preprocessing.preprocess_for_eval(image_input, out_shape, data_format=FLAGS.data_format, output_rgb=False) + features = tf.expand_dims(features, axis=0) + + anchor_creator = anchor_manipulator.AnchorCreator(out_shape, + layers_shapes = [(38, 38), (19, 19), (10, 10), (5, 5), (3, 3), (1, 1)], + anchor_scales = [(0.1,), (0.2,), (0.375,), (0.55,), (0.725,), (0.9,)], + extra_anchor_scales = [(0.1414,), (0.2739,), (0.4541,), (0.6315,), (0.8078,), (0.9836,)], + anchor_ratios = [(1., 2., .5), (1., 2., 3., .5, 0.3333), (1., 2., 3., .5, 0.3333), (1., 2., 3., .5, 0.3333), (1., 2., .5), (1., 2., .5)], + #anchor_ratios = [(2., .5), (2., 3., .5, 0.3333), (2., 3., .5, 0.3333), (2., 3., .5, 0.3333), (2., .5), (2., .5)], + layer_steps = [8, 16, 32, 64, 100, 300]) + all_anchors, all_num_anchors_depth, all_num_anchors_spatial = anchor_creator.get_all_anchors() + + anchor_encoder_decoder = anchor_manipulator.AnchorEncoder(allowed_borders = [1.0] * 6, + positive_threshold = None, + ignore_threshold = None, + prior_scaling=[0.1, 0.1, 0.2, 0.2]) + + decode_fn = lambda pred : anchor_encoder_decoder.ext_decode_all_anchors(pred, all_anchors, all_num_anchors_depth, all_num_anchors_spatial) + + with tf.variable_scope(FLAGS.model_scope, default_name=None, values=[features], reuse=tf.AUTO_REUSE): + backbone = ssd_net.VGG16Backbone(FLAGS.data_format) + feature_layers = backbone.forward(features, training=False) + location_pred, cls_pred = ssd_net.multibox_head(feature_layers, FLAGS.num_classes, all_num_anchors_depth, data_format=FLAGS.data_format) + if FLAGS.data_format == 'channels_first': + cls_pred = [tf.transpose(pred, [0, 2, 3, 1]) for pred in cls_pred] + location_pred = [tf.transpose(pred, [0, 2, 3, 1]) for pred in location_pred] + + cls_pred = [tf.reshape(pred, [-1, FLAGS.num_classes]) for pred in cls_pred] + location_pred = [tf.reshape(pred, [-1, 4]) for pred in location_pred] + + cls_pred = tf.concat(cls_pred, axis=0) + location_pred = tf.concat(location_pred, axis=0) + + with tf.device('/cpu:0'): + bboxes_pred = decode_fn(location_pred) + bboxes_pred = tf.concat(bboxes_pred, axis=0) + selected_bboxes, selected_scores = parse_by_class(cls_pred, bboxes_pred, + FLAGS.num_classes, FLAGS.select_threshold, FLAGS.min_size, + FLAGS.keep_topk, FLAGS.nms_topk, FLAGS.nms_threshold) + + labels_list = [] + scores_list = [] + bboxes_list = [] + for k, v in selected_scores.items(): + labels_list.append(tf.ones_like(v, tf.int32) * k) + scores_list.append(v) + bboxes_list.append(selected_bboxes[k]) + all_labels = tf.concat(labels_list, axis=0) + all_scores = tf.concat(scores_list, axis=0) + all_bboxes = tf.concat(bboxes_list, axis=0) + + saver = tf.train.Saver() + with tf.Session() as sess: + init = tf.global_variables_initializer() + sess.run(init) + + saver.restore(sess, get_checkpoint()) + + np_image = imread('./demo/test.jpg') + labels_, scores_, bboxes_ = sess.run([all_labels, all_scores, all_bboxes], feed_dict = {image_input : np_image, shape_input : np_image.shape[:-1]}) + + img_to_draw = draw_toolbox.bboxes_draw_on_img(np_image, labels_, scores_, bboxes_, thickness=2) + imsave('./demo/test_out.jpg', img_to_draw) + +if __name__ == '__main__': + tf.logging.set_verbosity(tf.logging.INFO) + tf.app.run() diff --git a/utils/external/SSD.TensorFlow/train_ssd.py b/utils/external/SSD.TensorFlow/train_ssd.py new file mode 100644 index 0000000..a6c09a8 --- /dev/null +++ b/utils/external/SSD.TensorFlow/train_ssd.py @@ -0,0 +1,498 @@ +# Copyright 2018 Changan Wang + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import sys + +import tensorflow as tf + +from net import ssd_net + +from dataset import dataset_common +from preprocessing import ssd_preprocessing +from utility import anchor_manipulator +from utility import scaffolds + +# hardware related configuration +tf.app.flags.DEFINE_integer( + 'num_readers', 8, + 'The number of parallel readers that read data from the dataset.') +tf.app.flags.DEFINE_integer( + 'num_preprocessing_threads', 24, + 'The number of threads used to create the batches.') +tf.app.flags.DEFINE_integer( + 'num_cpu_threads', 0, + 'The number of cpu cores used to train.') +tf.app.flags.DEFINE_float( + 'gpu_memory_fraction', 1., 'GPU memory fraction to use.') +# scaffold related configuration +tf.app.flags.DEFINE_string( + 'data_dir', './dataset/tfrecords', + 'The directory where the dataset input data is stored.') +tf.app.flags.DEFINE_integer( + 'num_classes', 21, 'Number of classes to use in the dataset.') +tf.app.flags.DEFINE_string( + 'model_dir', './logs/', + 'The directory where the model will be stored.') +tf.app.flags.DEFINE_integer( + 'log_every_n_steps', 10, + 'The frequency with which logs are printed.') +tf.app.flags.DEFINE_integer( + 'save_summary_steps', 500, + 'The frequency with which summaries are saved, in seconds.') +tf.app.flags.DEFINE_integer( + 'save_checkpoints_secs', 7200, + 'The frequency with which the model is saved, in seconds.') +# model related configuration +tf.app.flags.DEFINE_integer( + 'train_image_size', 300, + 'The size of the input image for the model to use.') +tf.app.flags.DEFINE_integer( + 'train_epochs', None, + 'The number of epochs to use for training.') +tf.app.flags.DEFINE_integer( + 'max_number_of_steps', 120000, + 'The max number of steps to use for training.') +tf.app.flags.DEFINE_integer( + 'batch_size', 32, + 'Batch size for training and evaluation.') +tf.app.flags.DEFINE_string( + 'data_format', 'channels_first', # 'channels_first' or 'channels_last' + 'A flag to override the data format used in the model. channels_first ' + 'provides a performance boost on GPU but is not always compatible ' + 'with CPU. If left unspecified, the data format will be chosen ' + 'automatically based on whether TensorFlow was built for CPU or GPU.') +tf.app.flags.DEFINE_float( + 'negative_ratio', 3., 'Negative ratio in the loss function.') +tf.app.flags.DEFINE_float( + 'match_threshold', 0.5, 'Matching threshold in the loss function.') +tf.app.flags.DEFINE_float( + 'neg_threshold', 0.5, 'Matching threshold for the negtive examples in the loss function.') +# optimizer related configuration +tf.app.flags.DEFINE_integer( + 'tf_random_seed', 20180503, 'Random seed for TensorFlow initializers.') +tf.app.flags.DEFINE_float( + 'weight_decay', 5e-4, 'The weight decay on the model weights.') +tf.app.flags.DEFINE_float( + 'momentum', 0.9, + 'The momentum for the MomentumOptimizer and RMSPropOptimizer.') +tf.app.flags.DEFINE_float('learning_rate', 1e-3, 'Initial learning rate.') +tf.app.flags.DEFINE_float( + 'end_learning_rate', 0.000001, + 'The minimal end learning rate used by a polynomial decay learning rate.') +# for learning rate piecewise_constant decay +tf.app.flags.DEFINE_string( + 'decay_boundaries', '500, 80000, 100000', + 'Learning rate decay boundaries by global_step (comma-separated list).') +tf.app.flags.DEFINE_string( + 'lr_decay_factors', '0.1, 1, 0.1, 0.01', + 'The values of learning_rate decay factor for each segment between boundaries (comma-separated list).') +# checkpoint related configuration +tf.app.flags.DEFINE_string( + 'checkpoint_path', './model', + 'The path to a checkpoint from which to fine-tune.') +tf.app.flags.DEFINE_string( + 'checkpoint_model_scope', 'vgg_16', + 'Model scope in the checkpoint. None if the same as the trained model.') +tf.app.flags.DEFINE_string( + 'model_scope', 'ssd300', + 'Model scope name used to replace the name_scope in checkpoint.') +tf.app.flags.DEFINE_string( + 'checkpoint_exclude_scopes', 'ssd300/multibox_head, ssd300/additional_layers, ssd300/conv4_3_scale', + 'Comma-separated list of scopes of variables to exclude when restoring from a checkpoint.') +tf.app.flags.DEFINE_boolean( + 'ignore_missing_vars', True, + 'When restoring a checkpoint would ignore missing variables.') +tf.app.flags.DEFINE_boolean( + 'multi_gpu', True, + 'Whether there is GPU to use for training.') + +FLAGS = tf.app.flags.FLAGS +#CUDA_VISIBLE_DEVICES +def validate_batch_size_for_multi_gpu(batch_size): + """For multi-gpu, batch-size must be a multiple of the number of + available GPUs. + + Note that this should eventually be handled by replicate_model_fn + directly. Multi-GPU support is currently experimental, however, + so doing the work here until that feature is in place. + """ + if FLAGS.multi_gpu: + from tensorflow.python.client import device_lib + + local_device_protos = device_lib.list_local_devices() + num_gpus = sum([1 for d in local_device_protos if d.device_type == 'GPU']) + if not num_gpus: + raise ValueError('Multi-GPU mode was specified, but no GPUs ' + 'were found. To use CPU, run --multi_gpu=False.') + + remainder = batch_size % num_gpus + if remainder: + err = ('When running with multiple GPUs, batch size ' + 'must be a multiple of the number of available GPUs. ' + 'Found {} GPUs with a batch size of {}; try --batch_size={} instead.' + ).format(num_gpus, batch_size, batch_size - remainder) + raise ValueError(err) + return num_gpus + return 0 + +def get_init_fn(): + return scaffolds.get_init_fn_for_scaffold(FLAGS.model_dir, FLAGS.checkpoint_path, + FLAGS.model_scope, FLAGS.checkpoint_model_scope, + FLAGS.checkpoint_exclude_scopes, FLAGS.ignore_missing_vars, + name_remap={'/kernel': '/weights', '/bias': '/biases'}) + +# couldn't find better way to pass params from input_fn to model_fn +# some tensors used by model_fn must be created in input_fn to ensure they are in the same graph +# but when we put these tensors to labels's dict, the replicate_model_fn will split them into each GPU +# the problem is that they shouldn't be splited +global_anchor_info = dict() + +def input_pipeline(dataset_pattern='train-*', is_training=True, batch_size=FLAGS.batch_size): + def input_fn(): + out_shape = [FLAGS.train_image_size] * 2 + anchor_creator = anchor_manipulator.AnchorCreator(out_shape, + layers_shapes = [(38, 38), (19, 19), (10, 10), (5, 5), (3, 3), (1, 1)], + anchor_scales = [(0.1,), (0.2,), (0.375,), (0.55,), (0.725,), (0.9,)], + extra_anchor_scales = [(0.1414,), (0.2739,), (0.4541,), (0.6315,), (0.8078,), (0.9836,)], + anchor_ratios = [(1., 2., .5), (1., 2., 3., .5, 0.3333), (1., 2., 3., .5, 0.3333), (1., 2., 3., .5, 0.3333), (1., 2., .5), (1., 2., .5)], + layer_steps = [8, 16, 32, 64, 100, 300]) + all_anchors, all_num_anchors_depth, all_num_anchors_spatial = anchor_creator.get_all_anchors() + + num_anchors_per_layer = [] + for ind in range(len(all_anchors)): + num_anchors_per_layer.append(all_num_anchors_depth[ind] * all_num_anchors_spatial[ind]) + + anchor_encoder_decoder = anchor_manipulator.AnchorEncoder(allowed_borders = [1.0] * 6, + positive_threshold = FLAGS.match_threshold, + ignore_threshold = FLAGS.neg_threshold, + prior_scaling=[0.1, 0.1, 0.2, 0.2]) + + image_preprocessing_fn = lambda image_, labels_, bboxes_ : ssd_preprocessing.preprocess_image(image_, labels_, bboxes_, out_shape, is_training=is_training, data_format=FLAGS.data_format, output_rgb=False) + anchor_encoder_fn = lambda glabels_, gbboxes_: anchor_encoder_decoder.encode_all_anchors(glabels_, gbboxes_, all_anchors, all_num_anchors_depth, all_num_anchors_spatial) + + image, _, shape, loc_targets, cls_targets, match_scores = dataset_common.slim_get_batch(FLAGS.num_classes, + batch_size, + ('train' if is_training else 'val'), + os.path.join(FLAGS.data_dir, dataset_pattern), + FLAGS.num_readers, + FLAGS.num_preprocessing_threads, + image_preprocessing_fn, + anchor_encoder_fn, + num_epochs=FLAGS.train_epochs, + is_training=is_training) + global global_anchor_info + global_anchor_info = {'decode_fn': lambda pred : anchor_encoder_decoder.decode_all_anchors(pred, num_anchors_per_layer), + 'num_anchors_per_layer': num_anchors_per_layer, + 'all_num_anchors_depth': all_num_anchors_depth } + + return image, {'shape': shape, 'loc_targets': loc_targets, 'cls_targets': cls_targets, 'match_scores': match_scores} + return input_fn + +def modified_smooth_l1(bbox_pred, bbox_targets, bbox_inside_weights=1., bbox_outside_weights=1., sigma=1.): + """ + ResultLoss = outside_weights * SmoothL1(inside_weights * (bbox_pred - bbox_targets)) + SmoothL1(x) = 0.5 * (sigma * x)^2, if |x| < 1 / sigma^2 + |x| - 0.5 / sigma^2, otherwise + """ + with tf.name_scope('smooth_l1', [bbox_pred, bbox_targets]): + sigma2 = sigma * sigma + + inside_mul = tf.multiply(bbox_inside_weights, tf.subtract(bbox_pred, bbox_targets)) + + smooth_l1_sign = tf.cast(tf.less(tf.abs(inside_mul), 1.0 / sigma2), tf.float32) + smooth_l1_option1 = tf.multiply(tf.multiply(inside_mul, inside_mul), 0.5 * sigma2) + smooth_l1_option2 = tf.subtract(tf.abs(inside_mul), 0.5 / sigma2) + smooth_l1_result = tf.add(tf.multiply(smooth_l1_option1, smooth_l1_sign), + tf.multiply(smooth_l1_option2, tf.abs(tf.subtract(smooth_l1_sign, 1.0)))) + + outside_mul = tf.multiply(bbox_outside_weights, smooth_l1_result) + + return outside_mul + + +# from scipy.misc import imread, imsave, imshow, imresize +# import numpy as np +# from utility import draw_toolbox + +# def save_image_with_bbox(image, labels_, scores_, bboxes_): +# if not hasattr(save_image_with_bbox, "counter"): +# save_image_with_bbox.counter = 0 # it doesn't exist yet, so initialize it +# save_image_with_bbox.counter += 1 + +# img_to_draw = np.copy(image) + +# img_to_draw = draw_toolbox.bboxes_draw_on_img(img_to_draw, labels_, scores_, bboxes_, thickness=2) +# imsave(os.path.join('./debug/{}.jpg').format(save_image_with_bbox.counter), img_to_draw) +# return save_image_with_bbox.counter + +def ssd_model_fn(features, labels, mode, params): + """model_fn for SSD to be used with our Estimator.""" + shape = labels['shape'] + loc_targets = labels['loc_targets'] + cls_targets = labels['cls_targets'] + match_scores = labels['match_scores'] + + global global_anchor_info + decode_fn = global_anchor_info['decode_fn'] + num_anchors_per_layer = global_anchor_info['num_anchors_per_layer'] + all_num_anchors_depth = global_anchor_info['all_num_anchors_depth'] + + # bboxes_pred = decode_fn(loc_targets[0]) + # bboxes_pred = [tf.reshape(preds, [-1, 4]) for preds in bboxes_pred] + # bboxes_pred = tf.concat(bboxes_pred, axis=0) + # save_image_op = tf.py_func(save_image_with_bbox, + # [ssd_preprocessing.unwhiten_image(features[0]), + # tf.clip_by_value(cls_targets[0], 0, tf.int64.max), + # match_scores[0], + # bboxes_pred], + # tf.int64, stateful=True) + # with tf.control_dependencies([save_image_op]): + + #print(all_num_anchors_depth) + with tf.variable_scope(params['model_scope'], default_name=None, values=[features], reuse=tf.AUTO_REUSE): + backbone = ssd_net.VGG16Backbone(params['data_format']) + feature_layers = backbone.forward(features, training=(mode == tf.estimator.ModeKeys.TRAIN)) + #print(feature_layers) + location_pred, cls_pred = ssd_net.multibox_head(feature_layers, params['num_classes'], all_num_anchors_depth, data_format=params['data_format']) + + if params['data_format'] == 'channels_first': + cls_pred = [tf.transpose(pred, [0, 2, 3, 1]) for pred in cls_pred] + location_pred = [tf.transpose(pred, [0, 2, 3, 1]) for pred in location_pred] + + cls_pred = [tf.reshape(pred, [tf.shape(features)[0], -1, params['num_classes']]) for pred in cls_pred] + location_pred = [tf.reshape(pred, [tf.shape(features)[0], -1, 4]) for pred in location_pred] + + cls_pred = tf.concat(cls_pred, axis=1) + location_pred = tf.concat(location_pred, axis=1) + + cls_pred = tf.reshape(cls_pred, [-1, params['num_classes']]) + location_pred = tf.reshape(location_pred, [-1, 4]) + + with tf.device('/cpu:0'): + with tf.control_dependencies([cls_pred, location_pred]): + with tf.name_scope('post_forward'): + #bboxes_pred = decode_fn(location_pred) + bboxes_pred = tf.map_fn(lambda _preds : decode_fn(_preds), + tf.reshape(location_pred, [tf.shape(features)[0], -1, 4]), + dtype=[tf.float32] * len(num_anchors_per_layer), back_prop=False) + #cls_targets = tf.Print(cls_targets, [tf.shape(bboxes_pred[0]),tf.shape(bboxes_pred[1]),tf.shape(bboxes_pred[2]),tf.shape(bboxes_pred[3])]) + bboxes_pred = [tf.reshape(preds, [-1, 4]) for preds in bboxes_pred] + bboxes_pred = tf.concat(bboxes_pred, axis=0) + + flaten_cls_targets = tf.reshape(cls_targets, [-1]) + flaten_match_scores = tf.reshape(match_scores, [-1]) + flaten_loc_targets = tf.reshape(loc_targets, [-1, 4]) + + # each positive examples has one label + positive_mask = flaten_cls_targets > 0 + n_positives = tf.count_nonzero(positive_mask) + + batch_n_positives = tf.count_nonzero(cls_targets, -1) + + batch_negtive_mask = tf.equal(cls_targets, 0)#tf.logical_and(tf.equal(cls_targets, 0), match_scores > 0.) + batch_n_negtives = tf.count_nonzero(batch_negtive_mask, -1) + + batch_n_neg_select = tf.cast(params['negative_ratio'] * tf.cast(batch_n_positives, tf.float32), tf.int32) + batch_n_neg_select = tf.minimum(batch_n_neg_select, tf.cast(batch_n_negtives, tf.int32)) + + # hard negative mining for classification + predictions_for_bg = tf.nn.softmax(tf.reshape(cls_pred, [tf.shape(features)[0], -1, params['num_classes']]))[:, :, 0] + prob_for_negtives = tf.where(batch_negtive_mask, + 0. - predictions_for_bg, + # ignore all the positives + 0. - tf.ones_like(predictions_for_bg)) + topk_prob_for_bg, _ = tf.nn.top_k(prob_for_negtives, k=tf.shape(prob_for_negtives)[1]) + score_at_k = tf.gather_nd(topk_prob_for_bg, tf.stack([tf.range(tf.shape(features)[0]), batch_n_neg_select - 1], axis=-1)) + + selected_neg_mask = prob_for_negtives >= tf.expand_dims(score_at_k, axis=-1) + + # include both selected negtive and all positive examples + final_mask = tf.stop_gradient(tf.logical_or(tf.reshape(tf.logical_and(batch_negtive_mask, selected_neg_mask), [-1]), positive_mask)) + total_examples = tf.count_nonzero(final_mask) + + cls_pred = tf.boolean_mask(cls_pred, final_mask) + location_pred = tf.boolean_mask(location_pred, tf.stop_gradient(positive_mask)) + flaten_cls_targets = tf.boolean_mask(tf.clip_by_value(flaten_cls_targets, 0, params['num_classes']), final_mask) + flaten_loc_targets = tf.stop_gradient(tf.boolean_mask(flaten_loc_targets, positive_mask)) + + predictions = { + 'classes': tf.argmax(cls_pred, axis=-1), + 'probabilities': tf.reduce_max(tf.nn.softmax(cls_pred, name='softmax_tensor'), axis=-1), + 'loc_predict': bboxes_pred } + + cls_accuracy = tf.metrics.accuracy(flaten_cls_targets, predictions['classes']) + metrics = {'cls_accuracy': cls_accuracy} + + # Create a tensor named train_accuracy for logging purposes. + tf.identity(cls_accuracy[1], name='cls_accuracy') + tf.summary.scalar('cls_accuracy', cls_accuracy[1]) + + if mode == tf.estimator.ModeKeys.PREDICT: + return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions) + + # Calculate loss, which includes softmax cross entropy and L2 regularization. + #cross_entropy = tf.cond(n_positives > 0, lambda: tf.losses.sparse_softmax_cross_entropy(labels=flaten_cls_targets, logits=cls_pred), lambda: 0.)# * (params['negative_ratio'] + 1.) + #flaten_cls_targets=tf.Print(flaten_cls_targets, [flaten_loc_targets],summarize=50000) + cross_entropy = tf.losses.sparse_softmax_cross_entropy(labels=flaten_cls_targets, logits=cls_pred) * (params['negative_ratio'] + 1.) + # Create a tensor named cross_entropy for logging purposes. + tf.identity(cross_entropy, name='cross_entropy_loss') + tf.summary.scalar('cross_entropy_loss', cross_entropy) + + #loc_loss = tf.cond(n_positives > 0, lambda: modified_smooth_l1(location_pred, tf.stop_gradient(flaten_loc_targets), sigma=1.), lambda: tf.zeros_like(location_pred)) + loc_loss = modified_smooth_l1(location_pred, flaten_loc_targets, sigma=1.) + #loc_loss = modified_smooth_l1(location_pred, tf.stop_gradient(gtargets)) + loc_loss = tf.reduce_mean(tf.reduce_sum(loc_loss, axis=-1), name='location_loss') + tf.summary.scalar('location_loss', loc_loss) + tf.losses.add_loss(loc_loss) + + l2_loss_vars = [] + for trainable_var in tf.trainable_variables(): + if '_bn' not in trainable_var.name: + if 'conv4_3_scale' not in trainable_var.name: + l2_loss_vars.append(tf.nn.l2_loss(trainable_var)) + else: + l2_loss_vars.append(tf.nn.l2_loss(trainable_var) * 0.1) + # Add weight decay to the loss. We exclude the batch norm variables because + # doing so leads to a small improvement in accuracy. + total_loss = tf.add(cross_entropy + loc_loss, tf.multiply(params['weight_decay'], tf.add_n(l2_loss_vars), name='l2_loss'), name='total_loss') + + if mode == tf.estimator.ModeKeys.TRAIN: + global_step = tf.train.get_or_create_global_step() + + lr_values = [params['learning_rate'] * decay for decay in params['lr_decay_factors']] + learning_rate = tf.train.piecewise_constant(tf.cast(global_step, tf.int32), + [int(_) for _ in params['decay_boundaries']], + lr_values) + truncated_learning_rate = tf.maximum(learning_rate, tf.constant(params['end_learning_rate'], dtype=learning_rate.dtype), name='learning_rate') + # Create a tensor named learning_rate for logging purposes. + tf.summary.scalar('learning_rate', truncated_learning_rate) + + optimizer = tf.train.MomentumOptimizer(learning_rate=truncated_learning_rate, + momentum=params['momentum']) + optimizer = tf.contrib.estimator.TowerOptimizer(optimizer) + + # Batch norm requires update_ops to be added as a train_op dependency. + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) + with tf.control_dependencies(update_ops): + train_op = optimizer.minimize(total_loss, global_step) + else: + train_op = None + + return tf.estimator.EstimatorSpec( + mode=mode, + predictions=predictions, + loss=total_loss, + train_op=train_op, + eval_metric_ops=metrics, + scaffold=tf.train.Scaffold(init_fn=get_init_fn())) + +def parse_comma_list(args): + return [float(s.strip()) for s in args.split(',')] + +def main(_): + # Using the Winograd non-fused algorithms provides a small performance boost. + os.environ['TF_ENABLE_WINOGRAD_NONFUSED'] = '1' + + gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction) + config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False, intra_op_parallelism_threads=FLAGS.num_cpu_threads, inter_op_parallelism_threads=FLAGS.num_cpu_threads, gpu_options=gpu_options) + + num_gpus = validate_batch_size_for_multi_gpu(FLAGS.batch_size) + + # Set up a RunConfig to only save checkpoints once per training cycle. + run_config = tf.estimator.RunConfig().replace( + save_checkpoints_secs=FLAGS.save_checkpoints_secs).replace( + save_checkpoints_steps=None).replace( + save_summary_steps=FLAGS.save_summary_steps).replace( + keep_checkpoint_max=5).replace( + tf_random_seed=FLAGS.tf_random_seed).replace( + log_step_count_steps=FLAGS.log_every_n_steps).replace( + session_config=config) + + replicate_ssd_model_fn = tf.contrib.estimator.replicate_model_fn(ssd_model_fn, loss_reduction=tf.losses.Reduction.MEAN) + ssd_detector = tf.estimator.Estimator( + model_fn=replicate_ssd_model_fn, model_dir=FLAGS.model_dir, config=run_config, + params={ + 'num_gpus': num_gpus, + 'data_format': FLAGS.data_format, + 'batch_size': FLAGS.batch_size, + 'model_scope': FLAGS.model_scope, + 'num_classes': FLAGS.num_classes, + 'negative_ratio': FLAGS.negative_ratio, + 'match_threshold': FLAGS.match_threshold, + 'neg_threshold': FLAGS.neg_threshold, + 'weight_decay': FLAGS.weight_decay, + 'momentum': FLAGS.momentum, + 'learning_rate': FLAGS.learning_rate, + 'end_learning_rate': FLAGS.end_learning_rate, + 'decay_boundaries': parse_comma_list(FLAGS.decay_boundaries), + 'lr_decay_factors': parse_comma_list(FLAGS.lr_decay_factors), + }) + tensors_to_log = { + 'lr': 'learning_rate', + 'ce': 'cross_entropy_loss', + 'loc': 'location_loss', + 'loss': 'total_loss', + 'l2': 'l2_loss', + 'acc': 'post_forward/cls_accuracy', + } + logging_hook = tf.train.LoggingTensorHook(tensors=tensors_to_log, every_n_iter=FLAGS.log_every_n_steps, + formatter=lambda dicts: (', '.join(['%s=%.6f' % (k, v) for k, v in dicts.items()]))) + + #hook = tf.train.ProfilerHook(save_steps=50, output_dir='.', show_memory=True) + print('Starting a training cycle.') + ssd_detector.train(input_fn=input_pipeline(dataset_pattern='train-*', is_training=True, batch_size=FLAGS.batch_size), + hooks=[logging_hook], max_steps=FLAGS.max_number_of_steps) + +if __name__ == '__main__': + tf.logging.set_verbosity(tf.logging.INFO) + tf.app.run() + + + # cls_targets = tf.reshape(cls_targets, [-1]) + # match_scores = tf.reshape(match_scores, [-1]) + # loc_targets = tf.reshape(loc_targets, [-1, 4]) + + # # each positive examples has one label + # positive_mask = cls_targets > 0 + # n_positives = tf.count_nonzero(positive_mask) + + # negtive_mask = tf.logical_and(tf.equal(cls_targets, 0), match_scores > 0.) + # n_negtives = tf.count_nonzero(negtive_mask) + + # n_neg_to_select = tf.cast(params['negative_ratio'] * tf.cast(n_positives, tf.float32), tf.int32) + # n_neg_to_select = tf.minimum(n_neg_to_select, tf.cast(n_negtives, tf.int32)) + + # # hard negative mining for classification + # predictions_for_bg = tf.nn.softmax(cls_pred)[:, 0] + + # prob_for_negtives = tf.where(negtive_mask, + # 0. - predictions_for_bg, + # # ignore all the positives + # 0. - tf.ones_like(predictions_for_bg)) + # topk_prob_for_bg, _ = tf.nn.top_k(prob_for_negtives, k=n_neg_to_select) + # selected_neg_mask = prob_for_negtives > topk_prob_for_bg[-1] + + # # include both selected negtive and all positive examples + # final_mask = tf.stop_gradient(tf.logical_or(tf.logical_and(negtive_mask, selected_neg_mask), positive_mask)) + # total_examples = tf.count_nonzero(final_mask) + + # glabels = tf.boolean_mask(tf.clip_by_value(cls_targets, 0, FLAGS.num_classes), final_mask) + # cls_pred = tf.boolean_mask(cls_pred, final_mask) + # location_pred = tf.boolean_mask(location_pred, tf.stop_gradient(positive_mask)) + # loc_targets = tf.boolean_mask(loc_targets, tf.stop_gradient(positive_mask)) diff --git a/utils/external/SSD.TensorFlow/utility/anchor_manipulator.py b/utils/external/SSD.TensorFlow/utility/anchor_manipulator.py new file mode 100644 index 0000000..a5d14c7 --- /dev/null +++ b/utils/external/SSD.TensorFlow/utility/anchor_manipulator.py @@ -0,0 +1,333 @@ +# Copyright 2018 Changan Wang + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +import math + +import tensorflow as tf +import numpy as np + +from tensorflow.contrib.image.python.ops import image_ops + +def areas(gt_bboxes): + with tf.name_scope('bboxes_areas', [gt_bboxes]): + ymin, xmin, ymax, xmax = tf.split(gt_bboxes, 4, axis=1) + return (xmax - xmin) * (ymax - ymin) + +def intersection(gt_bboxes, default_bboxes): + with tf.name_scope('bboxes_intersection', [gt_bboxes, default_bboxes]): + # num_anchors x 1 + ymin, xmin, ymax, xmax = tf.split(gt_bboxes, 4, axis=1) + # 1 x num_anchors + gt_ymin, gt_xmin, gt_ymax, gt_xmax = [tf.transpose(b, perm=[1, 0]) for b in tf.split(default_bboxes, 4, axis=1)] + # broadcast here to generate the full matrix + int_ymin = tf.maximum(ymin, gt_ymin) + int_xmin = tf.maximum(xmin, gt_xmin) + int_ymax = tf.minimum(ymax, gt_ymax) + int_xmax = tf.minimum(xmax, gt_xmax) + h = tf.maximum(int_ymax - int_ymin, 0.) + w = tf.maximum(int_xmax - int_xmin, 0.) + + return h * w +def iou_matrix(gt_bboxes, default_bboxes): + with tf.name_scope('iou_matrix', [gt_bboxes, default_bboxes]): + inter_vol = intersection(gt_bboxes, default_bboxes) + # broadcast + union_vol = areas(gt_bboxes) + tf.transpose(areas(default_bboxes), perm=[1, 0]) - inter_vol + + return tf.where(tf.equal(union_vol, 0.0), + tf.zeros_like(inter_vol), tf.truediv(inter_vol, union_vol)) + +def do_dual_max_match(overlap_matrix, low_thres, high_thres, ignore_between=True, gt_max_first=True): + ''' + overlap_matrix: num_gt * num_anchors + ''' + with tf.name_scope('dual_max_match', [overlap_matrix]): + # first match from anchors' side + anchors_to_gt = tf.argmax(overlap_matrix, axis=0) + # the matching degree + match_values = tf.reduce_max(overlap_matrix, axis=0) + + #positive_mask = tf.greater(match_values, high_thres) + less_mask = tf.less(match_values, low_thres) + between_mask = tf.logical_and(tf.less(match_values, high_thres), tf.greater_equal(match_values, low_thres)) + negative_mask = less_mask if ignore_between else between_mask + ignore_mask = between_mask if ignore_between else less_mask + # fill all negative positions with -1, all ignore positions is -2 + match_indices = tf.where(negative_mask, -1 * tf.ones_like(anchors_to_gt), anchors_to_gt) + match_indices = tf.where(ignore_mask, -2 * tf.ones_like(match_indices), match_indices) + + # negtive values has no effect in tf.one_hot, that means all zeros along that axis + # so all positive match positions in anchors_to_gt_mask is 1, all others are 0 + anchors_to_gt_mask = tf.one_hot(tf.clip_by_value(match_indices, -1, tf.cast(tf.shape(overlap_matrix)[0], tf.int64)), + tf.shape(overlap_matrix)[0], on_value=1, off_value=0, axis=0, dtype=tf.int32) + # match from ground truth's side + gt_to_anchors = tf.argmax(overlap_matrix, axis=1) + + if gt_max_first: + # the max match from ground truth's side has higher priority + left_gt_to_anchors_mask = tf.one_hot(gt_to_anchors, tf.shape(overlap_matrix)[1], on_value=1, off_value=0, axis=1, dtype=tf.int32) + else: + # the max match from anchors' side has higher priority + # use match result from ground truth's side only when the the matching degree from anchors' side is lower than position threshold + left_gt_to_anchors_mask = tf.cast(tf.logical_and(tf.reduce_max(anchors_to_gt_mask, axis=1, keep_dims=True) < 1, + tf.one_hot(gt_to_anchors, tf.shape(overlap_matrix)[1], + on_value=True, off_value=False, axis=1, dtype=tf.bool) + ), tf.int64) + # can not use left_gt_to_anchors_mask here, because there are many ground truthes match to one anchor, we should pick the highest one even when we are merging matching from ground truth side + left_gt_to_anchors_scores = overlap_matrix * tf.to_float(left_gt_to_anchors_mask) + # merge matching results from ground truth's side with the original matching results from anchors' side + # then select all the overlap score of those matching pairs + selected_scores = tf.gather_nd(overlap_matrix, tf.stack([tf.where(tf.reduce_max(left_gt_to_anchors_mask, axis=0) > 0, + tf.argmax(left_gt_to_anchors_scores, axis=0), + anchors_to_gt), + tf.range(tf.cast(tf.shape(overlap_matrix)[1], tf.int64))], axis=1)) + # return the matching results for both foreground anchors and background anchors, also with overlap scores + return tf.where(tf.reduce_max(left_gt_to_anchors_mask, axis=0) > 0, + tf.argmax(left_gt_to_anchors_scores, axis=0), + match_indices), selected_scores + +# def save_anchors(bboxes, labels, anchors_point): +# if not hasattr(save_image_with_bbox, "counter"): +# save_image_with_bbox.counter = 0 # it doesn't exist yet, so initialize it +# save_image_with_bbox.counter += 1 + +# np.save('./debug/bboxes_{}.npy'.format(save_image_with_bbox.counter), np.copy(bboxes)) +# np.save('./debug/labels_{}.npy'.format(save_image_with_bbox.counter), np.copy(labels)) +# np.save('./debug/anchors_{}.npy'.format(save_image_with_bbox.counter), np.copy(anchors_point)) +# return save_image_with_bbox.counter + +class AnchorEncoder(object): + def __init__(self, allowed_borders, positive_threshold, ignore_threshold, prior_scaling, clip=False): + super(AnchorEncoder, self).__init__() + self._all_anchors = None + self._allowed_borders = allowed_borders + self._positive_threshold = positive_threshold + self._ignore_threshold = ignore_threshold + self._prior_scaling = prior_scaling + self._clip = clip + + def center2point(self, center_y, center_x, height, width): + return center_y - height / 2., center_x - width / 2., center_y + height / 2., center_x + width / 2., + + def point2center(self, ymin, xmin, ymax, xmax): + height, width = (ymax - ymin), (xmax - xmin) + return ymin + height / 2., xmin + width / 2., height, width + + def encode_all_anchors(self, labels, bboxes, all_anchors, all_num_anchors_depth, all_num_anchors_spatial, debug=False): + # y, x, h, w are all in range [0, 1] relative to the original image size + # shape info: + # y_on_image, x_on_image: layers_shapes[0] * layers_shapes[1] + # h_on_image, w_on_image: num_anchors + assert (len(all_num_anchors_depth)==len(all_num_anchors_spatial)) and (len(all_num_anchors_depth)==len(all_anchors)), 'inconsist num layers for anchors.' + with tf.name_scope('encode_all_anchors'): + num_layers = len(all_num_anchors_depth) + list_anchors_ymin = [] + list_anchors_xmin = [] + list_anchors_ymax = [] + list_anchors_xmax = [] + tiled_allowed_borders = [] + for ind, anchor in enumerate(all_anchors): + anchors_ymin_, anchors_xmin_, anchors_ymax_, anchors_xmax_ = self.center2point(anchor[0], anchor[1], anchor[2], anchor[3]) + + list_anchors_ymin.append(tf.reshape(anchors_ymin_, [-1])) + list_anchors_xmin.append(tf.reshape(anchors_xmin_, [-1])) + list_anchors_ymax.append(tf.reshape(anchors_ymax_, [-1])) + list_anchors_xmax.append(tf.reshape(anchors_xmax_, [-1])) + + tiled_allowed_borders.extend([self._allowed_borders[ind]] * all_num_anchors_depth[ind] * all_num_anchors_spatial[ind]) + + anchors_ymin = tf.concat(list_anchors_ymin, 0, name='concat_ymin') + anchors_xmin = tf.concat(list_anchors_xmin, 0, name='concat_xmin') + anchors_ymax = tf.concat(list_anchors_ymax, 0, name='concat_ymax') + anchors_xmax = tf.concat(list_anchors_xmax, 0, name='concat_xmax') + + if self._clip: + anchors_ymin = tf.clip_by_value(anchors_ymin, 0., 1.) + anchors_xmin = tf.clip_by_value(anchors_xmin, 0., 1.) + anchors_ymax = tf.clip_by_value(anchors_ymax, 0., 1.) + anchors_xmax = tf.clip_by_value(anchors_xmax, 0., 1.) + + anchor_allowed_borders = tf.stack(tiled_allowed_borders, 0, name='concat_allowed_borders') + + inside_mask = tf.logical_and(tf.logical_and(anchors_ymin > -anchor_allowed_borders * 1., + anchors_xmin > -anchor_allowed_borders * 1.), + tf.logical_and(anchors_ymax < (1. + anchor_allowed_borders * 1.), + anchors_xmax < (1. + anchor_allowed_borders * 1.))) + + anchors_point = tf.stack([anchors_ymin, anchors_xmin, anchors_ymax, anchors_xmax], axis=-1) + + # save_anchors_op = tf.py_func(save_anchors, + # [bboxes, + # labels, + # anchors_point], + # tf.int64, stateful=True) + + # with tf.control_dependencies([save_anchors_op]): + overlap_matrix = iou_matrix(bboxes, anchors_point) * tf.cast(tf.expand_dims(inside_mask, 0), tf.float32) + matched_gt, gt_scores = do_dual_max_match(overlap_matrix, self._ignore_threshold, self._positive_threshold) + # get all positive matching positions + matched_gt_mask = matched_gt > -1 + matched_indices = tf.clip_by_value(matched_gt, 0, tf.int64.max) + # the labels here maybe chaos at those non-positive positions + gt_labels = tf.gather(labels, matched_indices) + # filter the invalid labels + gt_labels = gt_labels * tf.cast(matched_gt_mask, tf.int64) + # set those ignored positions to -1 + gt_labels = gt_labels + (-1 * tf.cast(matched_gt < -1, tf.int64)) + + gt_ymin, gt_xmin, gt_ymax, gt_xmax = tf.unstack(tf.gather(bboxes, matched_indices), 4, axis=-1) + + # transform to center / size. + gt_cy, gt_cx, gt_h, gt_w = self.point2center(gt_ymin, gt_xmin, gt_ymax, gt_xmax) + anchor_cy, anchor_cx, anchor_h, anchor_w = self.point2center(anchors_ymin, anchors_xmin, anchors_ymax, anchors_xmax) + # encode features. + # the prior_scaling (in fact is 5 and 10) is use for balance the regression loss of center and with(or height) + gt_cy = (gt_cy - anchor_cy) / anchor_h / self._prior_scaling[0] + gt_cx = (gt_cx - anchor_cx) / anchor_w / self._prior_scaling[1] + gt_h = tf.log(gt_h / anchor_h) / self._prior_scaling[2] + gt_w = tf.log(gt_w / anchor_w) / self._prior_scaling[3] + # now gt_localizations is our regression object, but also maybe chaos at those non-positive positions + if debug: + gt_targets = tf.stack([anchors_ymin, anchors_xmin, anchors_ymax, anchors_xmax], axis=-1) + else: + gt_targets = tf.stack([gt_cy, gt_cx, gt_h, gt_w], axis=-1) + # set all targets of non-positive positions to 0 + gt_targets = tf.expand_dims(tf.cast(matched_gt_mask, tf.float32), -1) * gt_targets + self._all_anchors = (anchor_cy, anchor_cx, anchor_h, anchor_w) + return gt_targets, gt_labels, gt_scores + + # return a list, of which each is: + # shape: [feature_h, feature_w, num_anchors, 4] + # order: ymin, xmin, ymax, xmax + def decode_all_anchors(self, pred_location, num_anchors_per_layer): + assert self._all_anchors is not None, 'no anchors to decode.' + with tf.name_scope('decode_all_anchors', [pred_location]): + anchor_cy, anchor_cx, anchor_h, anchor_w = self._all_anchors + + pred_h = tf.exp(pred_location[:, -2] * self._prior_scaling[2]) * anchor_h + pred_w = tf.exp(pred_location[:, -1] * self._prior_scaling[3]) * anchor_w + pred_cy = pred_location[:, 0] * self._prior_scaling[0] * anchor_h + anchor_cy + pred_cx = pred_location[:, 1] * self._prior_scaling[1] * anchor_w + anchor_cx + + return tf.split(tf.stack(self.center2point(pred_cy, pred_cx, pred_h, pred_w), axis=-1), num_anchors_per_layer, axis=0) + + def ext_decode_all_anchors(self, pred_location, all_anchors, all_num_anchors_depth, all_num_anchors_spatial): + assert (len(all_num_anchors_depth)==len(all_num_anchors_spatial)) and (len(all_num_anchors_depth)==len(all_anchors)), 'inconsist num layers for anchors.' + with tf.name_scope('ext_decode_all_anchors', [pred_location]): + num_anchors_per_layer = [] + for ind in range(len(all_anchors)): + num_anchors_per_layer.append(all_num_anchors_depth[ind] * all_num_anchors_spatial[ind]) + + num_layers = len(all_num_anchors_depth) + list_anchors_ymin = [] + list_anchors_xmin = [] + list_anchors_ymax = [] + list_anchors_xmax = [] + tiled_allowed_borders = [] + for ind, anchor in enumerate(all_anchors): + anchors_ymin_, anchors_xmin_, anchors_ymax_, anchors_xmax_ = self.center2point(anchor[0], anchor[1], anchor[2], anchor[3]) + + list_anchors_ymin.append(tf.reshape(anchors_ymin_, [-1])) + list_anchors_xmin.append(tf.reshape(anchors_xmin_, [-1])) + list_anchors_ymax.append(tf.reshape(anchors_ymax_, [-1])) + list_anchors_xmax.append(tf.reshape(anchors_xmax_, [-1])) + + anchors_ymin = tf.concat(list_anchors_ymin, 0, name='concat_ymin') + anchors_xmin = tf.concat(list_anchors_xmin, 0, name='concat_xmin') + anchors_ymax = tf.concat(list_anchors_ymax, 0, name='concat_ymax') + anchors_xmax = tf.concat(list_anchors_xmax, 0, name='concat_xmax') + + anchor_cy, anchor_cx, anchor_h, anchor_w = self.point2center(anchors_ymin, anchors_xmin, anchors_ymax, anchors_xmax) + + pred_h = tf.exp(pred_location[:,-2] * self._prior_scaling[2]) * anchor_h + pred_w = tf.exp(pred_location[:, -1] * self._prior_scaling[3]) * anchor_w + pred_cy = pred_location[:, 0] * self._prior_scaling[0] * anchor_h + anchor_cy + pred_cx = pred_location[:, 1] * self._prior_scaling[1] * anchor_w + anchor_cx + + return tf.split(tf.stack(self.center2point(pred_cy, pred_cx, pred_h, pred_w), axis=-1), num_anchors_per_layer, axis=0) + +class AnchorCreator(object): + def __init__(self, img_shape, layers_shapes, anchor_scales, extra_anchor_scales, anchor_ratios, layer_steps): + super(AnchorCreator, self).__init__() + # img_shape -> (height, width) + self._img_shape = img_shape + self._layers_shapes = layers_shapes + self._anchor_scales = anchor_scales + self._extra_anchor_scales = extra_anchor_scales + self._anchor_ratios = anchor_ratios + self._layer_steps = layer_steps + self._anchor_offset = [0.5] * len(self._layers_shapes) + + def get_layer_anchors(self, layer_shape, anchor_scale, extra_anchor_scale, anchor_ratio, layer_step, offset = 0.5): + ''' assume layer_shape[0] = 6, layer_shape[1] = 5 + x_on_layer = [[0, 1, 2, 3, 4], + [0, 1, 2, 3, 4], + [0, 1, 2, 3, 4], + [0, 1, 2, 3, 4], + [0, 1, 2, 3, 4], + [0, 1, 2, 3, 4]] + y_on_layer = [[0, 0, 0, 0, 0], + [1, 1, 1, 1, 1], + [2, 2, 2, 2, 2], + [3, 3, 3, 3, 3], + [4, 4, 4, 4, 4], + [5, 5, 5, 5, 5]] + ''' + with tf.name_scope('get_layer_anchors'): + x_on_layer, y_on_layer = tf.meshgrid(tf.range(layer_shape[1]), tf.range(layer_shape[0])) + + y_on_image = (tf.cast(y_on_layer, tf.float32) + offset) * layer_step / self._img_shape[0] + x_on_image = (tf.cast(x_on_layer, tf.float32) + offset) * layer_step / self._img_shape[1] + + num_anchors_along_depth = len(anchor_scale) * len(anchor_ratio) + len(extra_anchor_scale) + num_anchors_along_spatial = layer_shape[1] * layer_shape[0] + + list_h_on_image = [] + list_w_on_image = [] + + global_index = 0 + # for square anchors + for _, scale in enumerate(extra_anchor_scale): + list_h_on_image.append(scale) + list_w_on_image.append(scale) + global_index += 1 + # for other aspect ratio anchors + for scale_index, scale in enumerate(anchor_scale): + for ratio_index, ratio in enumerate(anchor_ratio): + list_h_on_image.append(scale / math.sqrt(ratio)) + list_w_on_image.append(scale * math.sqrt(ratio)) + global_index += 1 + # shape info: + # y_on_image, x_on_image: layers_shapes[0] * layers_shapes[1] + # h_on_image, w_on_image: num_anchors_along_depth + return tf.expand_dims(y_on_image, axis=-1), tf.expand_dims(x_on_image, axis=-1), \ + tf.constant(list_h_on_image, dtype=tf.float32), \ + tf.constant(list_w_on_image, dtype=tf.float32), num_anchors_along_depth, num_anchors_along_spatial + + def get_all_anchors(self): + all_anchors = [] + all_num_anchors_depth = [] + all_num_anchors_spatial = [] + for layer_index, layer_shape in enumerate(self._layers_shapes): + anchors_this_layer = self.get_layer_anchors(layer_shape, + self._anchor_scales[layer_index], + self._extra_anchor_scales[layer_index], + self._anchor_ratios[layer_index], + self._layer_steps[layer_index], + self._anchor_offset[layer_index]) + all_anchors.append(anchors_this_layer[:-2]) + all_num_anchors_depth.append(anchors_this_layer[-2]) + all_num_anchors_spatial.append(anchors_this_layer[-1]) + return all_anchors, all_num_anchors_depth, all_num_anchors_spatial + diff --git a/utils/external/SSD.TensorFlow/utility/anchor_manipulator_unittest.py b/utils/external/SSD.TensorFlow/utility/anchor_manipulator_unittest.py new file mode 100644 index 0000000..bbacc64 --- /dev/null +++ b/utils/external/SSD.TensorFlow/utility/anchor_manipulator_unittest.py @@ -0,0 +1,156 @@ +# Copyright 2018 Changan Wang + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +import tensorflow as tf +from scipy.misc import imread, imsave, imshow, imresize +import numpy as np +import sys; sys.path.insert(0, ".") +from utility import draw_toolbox +from utility import anchor_manipulator +from preprocessing import ssd_preprocessing + +slim = tf.contrib.slim + +def save_image_with_bbox(image, labels_, scores_, bboxes_): + if not hasattr(save_image_with_bbox, "counter"): + save_image_with_bbox.counter = 0 # it doesn't exist yet, so initialize it + save_image_with_bbox.counter += 1 + + img_to_draw = np.copy(image) + + img_to_draw = draw_toolbox.bboxes_draw_on_img(img_to_draw, labels_, scores_, bboxes_, thickness=2) + imsave(os.path.join('./debug/{}.jpg').format(save_image_with_bbox.counter), img_to_draw) + return save_image_with_bbox.counter + +def slim_get_split(file_pattern='{}_????'): + # Features in Pascal VOC TFRecords. + keys_to_features = { + 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), + 'image/format': tf.FixedLenFeature((), tf.string, default_value='jpeg'), + 'image/height': tf.FixedLenFeature([1], tf.int64), + 'image/width': tf.FixedLenFeature([1], tf.int64), + 'image/channels': tf.FixedLenFeature([1], tf.int64), + 'image/shape': tf.FixedLenFeature([3], tf.int64), + 'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/label': tf.VarLenFeature(dtype=tf.int64), + 'image/object/bbox/difficult': tf.VarLenFeature(dtype=tf.int64), + 'image/object/bbox/truncated': tf.VarLenFeature(dtype=tf.int64), + } + items_to_handlers = { + 'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'), + 'shape': slim.tfexample_decoder.Tensor('image/shape'), + 'object/bbox': slim.tfexample_decoder.BoundingBox( + ['ymin', 'xmin', 'ymax', 'xmax'], 'image/object/bbox/'), + 'object/label': slim.tfexample_decoder.Tensor('image/object/bbox/label'), + 'object/difficult': slim.tfexample_decoder.Tensor('image/object/bbox/difficult'), + 'object/truncated': slim.tfexample_decoder.Tensor('image/object/bbox/truncated'), + } + decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) + + dataset = slim.dataset.Dataset( + data_sources=file_pattern, + reader=tf.TFRecordReader, + decoder=decoder, + num_samples=100, + items_to_descriptions=None, + num_classes=21, + labels_to_names=None) + + with tf.name_scope('dataset_data_provider'): + provider = slim.dataset_data_provider.DatasetDataProvider( + dataset, + num_readers=2, + common_queue_capacity=32, + common_queue_min=8, + shuffle=True, + num_epochs=1) + + [org_image, shape, glabels_raw, gbboxes_raw, isdifficult] = provider.get(['image', 'shape', + 'object/label', + 'object/bbox', + 'object/difficult']) + image, glabels, gbboxes = ssd_preprocessing.preprocess_image(org_image, glabels_raw, gbboxes_raw, [300, 300], is_training=True, data_format='channels_last', output_rgb=True) + + anchor_creator = anchor_manipulator.AnchorCreator([300] * 2, + layers_shapes = [(38, 38), (19, 19), (10, 10), (5, 5), (3, 3), (1, 1)], + anchor_scales = [(0.1,), (0.2,), (0.375,), (0.55,), (0.725,), (0.9,)], + extra_anchor_scales = [(0.1414,), (0.2739,), (0.4541,), (0.6315,), (0.8078,), (0.9836,)], + anchor_ratios = [(2., .5), (2., 3., .5, 0.3333), (2., 3., .5, 0.3333), (2., 3., .5, 0.3333), (2., .5), (2., .5)], + layer_steps = [8, 16, 32, 64, 100, 300]) + + all_anchors, all_num_anchors_depth, all_num_anchors_spatial = anchor_creator.get_all_anchors() + + num_anchors_per_layer = [] + for ind in range(len(all_anchors)): + num_anchors_per_layer.append(all_num_anchors_depth[ind] * all_num_anchors_spatial[ind]) + + anchor_encoder_decoder = anchor_manipulator.AnchorEncoder(allowed_borders=[1.0] * 6, + positive_threshold = 0.5, + ignore_threshold = 0.5, + prior_scaling=[0.1, 0.1, 0.2, 0.2]) + + gt_targets, gt_labels, gt_scores = anchor_encoder_decoder.encode_all_anchors(glabels, gbboxes, all_anchors, all_num_anchors_depth, all_num_anchors_spatial, True) + + anchors = anchor_encoder_decoder._all_anchors + # split by layers + gt_targets, gt_labels, gt_scores, anchors = tf.split(gt_targets, num_anchors_per_layer, axis=0),\ + tf.split(gt_labels, num_anchors_per_layer, axis=0),\ + tf.split(gt_scores, num_anchors_per_layer, axis=0),\ + [tf.split(anchor, num_anchors_per_layer, axis=0) for anchor in anchors] + + save_image_op = tf.py_func(save_image_with_bbox, + [ssd_preprocessing.unwhiten_image(image), + tf.clip_by_value(tf.concat(gt_labels, axis=0), 0, tf.int64.max), + tf.concat(gt_scores, axis=0), + tf.concat(gt_targets, axis=0)], + tf.int64, stateful=True) + return save_image_op + +if __name__ == '__main__': + save_image_op = slim_get_split('/media/rs/7A0EE8880EE83EAF/Detections/SSD/dataset/tfrecords/train*') + # Create the graph, etc. + init_op = tf.group([tf.local_variables_initializer(), tf.local_variables_initializer(), tf.tables_initializer()]) + + # Create a session for running operations in the Graph. + sess = tf.Session() + # Initialize the variables (like the epoch counter). + sess.run(init_op) + + # Start input enqueue threads. + coord = tf.train.Coordinator() + threads = tf.train.start_queue_runners(sess=sess, coord=coord) + + try: + while not coord.should_stop(): + # Run training steps or whatever + print(sess.run(save_image_op)) + + except tf.errors.OutOfRangeError: + print('Done training -- epoch limit reached') + finally: + # When done, ask the threads to stop. + coord.request_stop() + + # Wait for threads to finish. + coord.join(threads) + sess.close() diff --git a/utils/external/SSD.TensorFlow/utility/checkpint_inspect.py b/utils/external/SSD.TensorFlow/utility/checkpint_inspect.py new file mode 100644 index 0000000..2979e88 --- /dev/null +++ b/utils/external/SSD.TensorFlow/utility/checkpint_inspect.py @@ -0,0 +1,55 @@ +# Copyright 2018 Changan Wang + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python import pywrap_tensorflow + +def print_tensors_in_checkpoint_file(file_name, tensor_name, all_tensors): + try: + reader = pywrap_tensorflow.NewCheckpointReader(file_name) + if all_tensors: + var_to_shape_map = reader.get_variable_to_shape_map() + for key in var_to_shape_map: + print("tensor_name: ", key) + print(reader.get_tensor(key)) + elif not tensor_name: + print(reader.debug_string().decode("utf-8")) + else: + print("tensor_name: ", tensor_name) + print(reader.get_tensor(tensor_name)) + except Exception as e: # pylint: disable=broad-except + print(str(e)) + if "corrupted compressed block contents" in str(e): + print("It's likely that your checkpoint file has been compressed " + "with SNAPPY.") + +def print_all_tensors_name(file_name): + try: + reader = pywrap_tensorflow.NewCheckpointReader(file_name) + var_to_shape_map = reader.get_variable_to_shape_map() + for key in var_to_shape_map: + print(key) + except Exception as e: # pylint: disable=broad-except + print(str(e)) + if "corrupted compressed block contents" in str(e): + print("It's likely that your checkpoint file has been compressed " + "with SNAPPY.") + +if __name__ == "__main__": + print_all_tensors_name('./model/vgg16_reducedfc.ckpt') diff --git a/utils/external/SSD.TensorFlow/utility/draw_toolbox.py b/utils/external/SSD.TensorFlow/utility/draw_toolbox.py new file mode 100644 index 0000000..a72ae50 --- /dev/null +++ b/utils/external/SSD.TensorFlow/utility/draw_toolbox.py @@ -0,0 +1,73 @@ +# Copyright 2018 Changan Wang + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +import cv2 +import matplotlib.cm as mpcm + +from dataset import dataset_common + +def gain_translate_table(): + label2name_table = {} + for class_name, labels_pair in dataset_common.VOC_LABELS.items(): + label2name_table[labels_pair[0]] = class_name + return label2name_table + +label2name_table = gain_translate_table() + +def colors_subselect(colors, num_classes=21): + dt = len(colors) // num_classes + sub_colors = [] + for i in range(num_classes): + color = colors[i*dt] + if isinstance(color[0], float): + sub_colors.append([int(c * 255) for c in color]) + else: + sub_colors.append([c for c in color]) + return sub_colors + +colors = colors_subselect(mpcm.plasma.colors, num_classes=21) +colors_tableau = [(255, 255, 255), (31, 119, 180), (174, 199, 232), (255, 127, 14), (255, 187, 120), + (44, 160, 44), (152, 223, 138), (214, 39, 40), (255, 152, 150), + (148, 103, 189), (197, 176, 213), (140, 86, 75), (196, 156, 148), + (227, 119, 194), (247, 182, 210), (127, 127, 127), (199, 199, 199), + (188, 189, 34), (219, 219, 141), (23, 190, 207), (158, 218, 229)] + +def bboxes_draw_on_img(img, classes, scores, bboxes, thickness=2): + shape = img.shape + scale = 0.4 + text_thickness = 1 + line_type = 8 + for i in range(bboxes.shape[0]): + if classes[i] < 1: continue + bbox = bboxes[i] + color = colors_tableau[classes[i]] + # Draw bounding boxes + p1 = (int(bbox[0] * shape[0]), int(bbox[1] * shape[1])) + p2 = (int(bbox[2] * shape[0]), int(bbox[3] * shape[1])) + if (p2[0] - p1[0] < 1) or (p2[1] - p1[1] < 1): + continue + + cv2.rectangle(img, p1[::-1], p2[::-1], color, thickness) + # Draw text + s = '%s/%.1f%%' % (label2name_table[classes[i]], scores[i]*100) + # text_size is (width, height) + text_size, baseline = cv2.getTextSize(s, cv2.FONT_HERSHEY_SIMPLEX, scale, text_thickness) + p1 = (p1[0] - text_size[1], p1[1]) + + cv2.rectangle(img, (p1[1] - thickness//2, p1[0] - thickness - baseline), (p1[1] + text_size[0], p1[0] + text_size[1]), color, -1) + + cv2.putText(img, s, (p1[1], p1[0] + baseline), cv2.FONT_HERSHEY_SIMPLEX, scale, (255,255,255), text_thickness, line_type) + + return img + diff --git a/utils/external/SSD.TensorFlow/utility/scaffolds.py b/utils/external/SSD.TensorFlow/utility/scaffolds.py new file mode 100644 index 0000000..820dabb --- /dev/null +++ b/utils/external/SSD.TensorFlow/utility/scaffolds.py @@ -0,0 +1,86 @@ +# Copyright 2018 Changan Wang + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import sys + +import tensorflow as tf + +def get_init_fn_for_scaffold(model_dir, checkpoint_path, model_scope, checkpoint_model_scope, checkpoint_exclude_scopes, ignore_missing_vars, name_remap=None): + if tf.train.latest_checkpoint(model_dir): + tf.logging.info('Ignoring --checkpoint_path because a checkpoint already exists in %s.' % model_dir) + return None + exclusion_scopes = [] + if checkpoint_exclude_scopes: + exclusion_scopes = [scope.strip() for scope in checkpoint_exclude_scopes.split(',')] + + variables_to_restore = [] + for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES): + excluded = False + for exclusion in exclusion_scopes: + if exclusion in var.op.name:#.startswith(exclusion): + excluded = True + break + if not excluded: + variables_to_restore.append(var) + if checkpoint_model_scope is not None: + if checkpoint_model_scope.strip() == '': + variables_to_restore = {var.op.name.replace(model_scope + '/', ''): var for var in variables_to_restore} + else: + variables_to_restore = {var.op.name.replace(model_scope, checkpoint_model_scope.strip()): var for var in variables_to_restore} + if name_remap is not None: + renamed_variables_to_restore = dict() + for var_name, var in variables_to_restore.items(): + found = False + for k, v in name_remap.items(): + if k in var_name: + renamed_variables_to_restore[var_name.replace(k, v)] = var + found = True + break + if not found: + renamed_variables_to_restore[var_name] = var + variables_to_restore = renamed_variables_to_restore + + checkpoint_path = tf.train.latest_checkpoint(checkpoint_path) if tf.gfile.IsDirectory(checkpoint_path) else checkpoint_path + + tf.logging.info('Fine-tuning from %s. Ignoring missing vars: %s.' % (checkpoint_path, ignore_missing_vars)) + + if not variables_to_restore: + raise ValueError('variables_to_restore cannot be empty') + if ignore_missing_vars: + reader = tf.train.NewCheckpointReader(checkpoint_path) + if isinstance(variables_to_restore, dict): + var_dict = variables_to_restore + else: + var_dict = {var.op.name: var for var in variables_to_restore} + available_vars = {} + for var in var_dict: + if reader.has_tensor(var): + available_vars[var] = var_dict[var] + else: + tf.logging.warning('Variable %s missing in checkpoint %s.', var, checkpoint_path) + variables_to_restore = available_vars + if variables_to_restore: + saver = tf.train.Saver(variables_to_restore, reshape=False) + saver.build() + def callback(scaffold, session): + saver.restore(session, checkpoint_path) + return callback + else: + tf.logging.warning('No Variables to restore.') + return None diff --git a/utils/external/SSD.TensorFlow/voc_eval.py b/utils/external/SSD.TensorFlow/voc_eval.py new file mode 100644 index 0000000..ea9c76a --- /dev/null +++ b/utils/external/SSD.TensorFlow/voc_eval.py @@ -0,0 +1,269 @@ +# Copyright 2018 Changan Wang + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import sys +import os +import numpy as np +import pickle + +if sys.version_info[0] == 2: + import xml.etree.cElementTree as ET +else: + import xml.etree.ElementTree as ET + +from dataset import dataset_common + +''' +VOC2007TEST + Annotations + ... + ImageSets +''' +#dataset_path = '/media/rs/7A0EE8880EE83EAF/Detections/PASCAL/VOC/VOC2007TEST' +dataset_path = '/data1/jonathan/datasets/Pascal.VOC.2007.2012/VOC2007TEST' +# change above path according to your system settings +pred_path = './logs/predict' +pred_file = 'results_{}.txt' # from 1-num_classes +output_path = './logs/predict/eval_output' +cache_path = './logs/predict/eval_cache' +anno_files = 'Annotations/{}.xml' +all_images_file = 'ImageSets/Main/test.txt' + +def parse_rec(filename): + """ Parse a PASCAL VOC xml file """ + tree = ET.parse(filename) + objects = [] + for obj in tree.findall('object'): + obj_struct = {} + obj_struct['name'] = obj.find('name').text + obj_struct['pose'] = obj.find('pose').text + obj_struct['truncated'] = int(obj.find('truncated').text) + obj_struct['difficult'] = int(obj.find('difficult').text) + bbox = obj.find('bndbox') + obj_struct['bbox'] = [int(bbox.find('xmin').text) - 1, + int(bbox.find('ymin').text) - 1, + int(bbox.find('xmax').text) - 1, + int(bbox.find('ymax').text) - 1] + objects.append(obj_struct) + + return objects + +def do_python_eval(use_07=True): + aps = [] + # The PASCAL VOC metric changed in 2010 + use_07_metric = use_07 + print('VOC07 metric? ' + ('Yes' if use_07_metric else 'No')) + if not os.path.isdir(output_path): + os.mkdir(output_path) + for cls_name, cls_pair in dataset_common.VOC_LABELS.items(): + if 'none' in cls_name: + continue + cls_id = cls_pair[0] + filename = os.path.join(pred_path, pred_file.format(cls_id)) + rec, prec, ap = voc_eval(filename, os.path.join(dataset_path, anno_files), + os.path.join(dataset_path, all_images_file), cls_name, cache_path, + ovthresh=0.5, use_07_metric=use_07_metric) + aps += [ap] + print('AP for {} = {:.4f}'.format(cls_name, ap)) + with open(os.path.join(output_path, cls_name + '_pr.pkl'), 'wb') as f: + pickle.dump({'rec': rec, 'prec': prec, 'ap': ap}, f) + print('Mean AP = {:.4f}'.format(np.mean(aps))) + print('~~~~~~~~') + print('Results:') + for ap in aps: + print('{:.3f}'.format(ap)) + print('{:.3f}'.format(np.mean(aps))) + print('~~~~~~~~') + print('') + print('--------------------------------------------------------------') + print('Results computed with the **unofficial** Python eval code.') + print('Results should be very close to the official MATLAB eval code.') + print('--------------------------------------------------------------') + + +def voc_ap(rec, prec, use_07_metric=True): + """ ap = voc_ap(rec, prec, [use_07_metric]) + Compute VOC AP given precision and recall. + If use_07_metric is true, uses the + VOC 07 11 point method (default:False). + """ + if use_07_metric: + # 11 point metric + ap = 0. + for t in np.arange(0., 1.1, 0.1): + if np.sum(rec >= t) == 0: + p = 0 + else: + p = np.max(prec[rec >= t]) + ap = ap + p / 11. + else: + # correct AP calculation + # first append sentinel values at the end + mrec = np.concatenate(([0.], rec, [1.])) + mpre = np.concatenate(([0.], prec, [0.])) + + # compute the precision envelope + for i in range(mpre.size - 1, 0, -1): + mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i]) + + # to calculate area under PR curve, look for points + # where X axis (recall) changes value + i = np.where(mrec[1:] != mrec[:-1])[0] + + # and sum (\Delta recall) * prec + ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) + return ap + + +def voc_eval(detpath, + annopath, + imagesetfile, + classname, + cachedir, + ovthresh=0.5, + use_07_metric=True): + """rec, prec, ap = voc_eval(detpath, + annopath, + imagesetfile, + classname, + [ovthresh], + [use_07_metric]) + Top level function that does the PASCAL VOC evaluation. + detpath: Path to detections + detpath.format(classname) should produce the detection results file. + annopath: Path to annotations + annopath.format(imagename) should be the xml annotations file. + imagesetfile: Text file containing the list of images, one image per line. + classname: Category name (duh) + cachedir: Directory for caching the annotations + [ovthresh]: Overlap threshold (default = 0.5) + [use_07_metric]: Whether to use VOC07's 11 point AP computation + (default False) + """ + # assumes detections are in detpath.format(classname) + # assumes annotations are in annopath.format(imagename) + # assumes imagesetfile is a text file with each line an image name + # cachedir caches the annotations in a pickle file + # first load gt + if not os.path.isdir(cachedir): + os.mkdir(cachedir) + cachefile = os.path.join(cachedir, 'annots.pkl') + # read list of images + with open(imagesetfile, 'r') as f: + lines = f.readlines() + imagenames = [x.strip() for x in lines] + if not os.path.isfile(cachefile): + # load annots + recs = {} + for i, imagename in enumerate(imagenames): + recs[imagename] = parse_rec(annopath.format(imagename)) + if i % 100 == 0: + print('Reading annotation for {:d}/{:d}'.format( + i + 1, len(imagenames))) + # save + print('Saving cached annotations to {:s}'.format(cachefile)) + with open(cachefile, 'wb') as f: + pickle.dump(recs, f) + else: + # load + with open(cachefile, 'rb') as f: + recs = pickle.load(f) + + # extract gt objects for this class + class_recs = {} + npos = 0 + + for imagename in imagenames: + R = [obj for obj in recs[imagename] if obj['name'] == classname] + bbox = np.array([x['bbox'] for x in R]) + difficult = np.array([x['difficult'] for x in R]).astype(np.bool) + det = [False] * len(R) + npos = npos + sum(~difficult) + class_recs[imagename] = {'bbox': bbox, + 'difficult': difficult, + 'det': det} + # read dets + with open(detpath, 'r') as f: + lines = f.readlines() + + if any(lines) == 1: + + splitlines = [x.strip().split(' ') for x in lines] + image_ids = [x[0] for x in splitlines] + confidence = np.array([float(x[1]) for x in splitlines]) + BB = np.array([[float(z) for z in x[2:]] for x in splitlines]) + + # sort by confidence + sorted_ind = np.argsort(-confidence) + sorted_scores = np.sort(-confidence) + BB = BB[sorted_ind, :] + image_ids = [image_ids[x] for x in sorted_ind] + + # go down dets and mark TPs and FPs + nd = len(image_ids) + tp = np.zeros(nd) + fp = np.zeros(nd) + for d in range(nd): + R = class_recs[image_ids[d]] + bb = BB[d, :].astype(float) + ovmax = -np.inf + BBGT = R['bbox'].astype(float) + if BBGT.size > 0: + # compute overlaps + # intersection + ixmin = np.maximum(BBGT[:, 0], bb[0]) + iymin = np.maximum(BBGT[:, 1], bb[1]) + ixmax = np.minimum(BBGT[:, 2], bb[2]) + iymax = np.minimum(BBGT[:, 3], bb[3]) + iw = np.maximum(ixmax - ixmin, 0.) + ih = np.maximum(iymax - iymin, 0.) + inters = iw * ih + uni = ((bb[2] - bb[0]) * (bb[3] - bb[1]) + + (BBGT[:, 2] - BBGT[:, 0]) * + (BBGT[:, 3] - BBGT[:, 1]) - inters) + overlaps = inters / uni + ovmax = np.max(overlaps) + jmax = np.argmax(overlaps) + + if ovmax > ovthresh: + if not R['difficult'][jmax]: + if not R['det'][jmax]: + tp[d] = 1. + R['det'][jmax] = 1 + else: + fp[d] = 1. + else: + fp[d] = 1. + + # compute precision recall + fp = np.cumsum(fp) + tp = np.cumsum(tp) + rec = tp / float(npos) + # avoid divide by zero in case the first detection matches a difficult + # ground truth + prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps) + ap = voc_ap(rec, prec, use_07_metric) + else: + rec = -1. + prec = -1. + ap = -1. + + return rec, prec, ap + +if __name__ == '__main__': + do_python_eval() From 70b1c4730d8897159556a6ba3168cf63769e8e1f Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Sat, 29 Dec 2018 11:07:09 +0800 Subject: [PATCH 028/173] rename to --- .../{SSD.TensorFlow => ssd_tensorflow}/LICENSE | 0 .../{SSD.TensorFlow => ssd_tensorflow}/README.md | 0 .../dataset/convert_tfrecords.py | 0 .../dataset/dataset_common.py | 0 .../dataset/dataset_inspect.py | 0 .../demo/demo1.jpg | Bin .../demo/demo2.jpg | Bin .../demo/demo3.jpg | Bin .../{SSD.TensorFlow => ssd_tensorflow}/eval_ssd.py | 0 .../net/ssd_net.py | 0 .../preprocessing/preprocessing_unittest.py | 0 .../preprocessing/ssd_preprocessing.py | 0 .../simple_ssd_demo.py | 0 .../{SSD.TensorFlow => ssd_tensorflow}/train_ssd.py | 0 .../utility/anchor_manipulator.py | 0 .../utility/anchor_manipulator_unittest.py | 0 .../utility/checkpint_inspect.py | 0 .../utility/draw_toolbox.py | 0 .../utility/scaffolds.py | 0 .../{SSD.TensorFlow => ssd_tensorflow}/voc_eval.py | 0 20 files changed, 0 insertions(+), 0 deletions(-) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/LICENSE (100%) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/README.md (100%) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/dataset/convert_tfrecords.py (100%) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/dataset/dataset_common.py (100%) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/dataset/dataset_inspect.py (100%) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/demo/demo1.jpg (100%) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/demo/demo2.jpg (100%) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/demo/demo3.jpg (100%) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/eval_ssd.py (100%) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/net/ssd_net.py (100%) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/preprocessing/preprocessing_unittest.py (100%) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/preprocessing/ssd_preprocessing.py (100%) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/simple_ssd_demo.py (100%) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/train_ssd.py (100%) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/utility/anchor_manipulator.py (100%) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/utility/anchor_manipulator_unittest.py (100%) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/utility/checkpint_inspect.py (100%) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/utility/draw_toolbox.py (100%) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/utility/scaffolds.py (100%) rename utils/external/{SSD.TensorFlow => ssd_tensorflow}/voc_eval.py (100%) diff --git a/utils/external/SSD.TensorFlow/LICENSE b/utils/external/ssd_tensorflow/LICENSE similarity index 100% rename from utils/external/SSD.TensorFlow/LICENSE rename to utils/external/ssd_tensorflow/LICENSE diff --git a/utils/external/SSD.TensorFlow/README.md b/utils/external/ssd_tensorflow/README.md similarity index 100% rename from utils/external/SSD.TensorFlow/README.md rename to utils/external/ssd_tensorflow/README.md diff --git a/utils/external/SSD.TensorFlow/dataset/convert_tfrecords.py b/utils/external/ssd_tensorflow/dataset/convert_tfrecords.py similarity index 100% rename from utils/external/SSD.TensorFlow/dataset/convert_tfrecords.py rename to utils/external/ssd_tensorflow/dataset/convert_tfrecords.py diff --git a/utils/external/SSD.TensorFlow/dataset/dataset_common.py b/utils/external/ssd_tensorflow/dataset/dataset_common.py similarity index 100% rename from utils/external/SSD.TensorFlow/dataset/dataset_common.py rename to utils/external/ssd_tensorflow/dataset/dataset_common.py diff --git a/utils/external/SSD.TensorFlow/dataset/dataset_inspect.py b/utils/external/ssd_tensorflow/dataset/dataset_inspect.py similarity index 100% rename from utils/external/SSD.TensorFlow/dataset/dataset_inspect.py rename to utils/external/ssd_tensorflow/dataset/dataset_inspect.py diff --git a/utils/external/SSD.TensorFlow/demo/demo1.jpg b/utils/external/ssd_tensorflow/demo/demo1.jpg similarity index 100% rename from utils/external/SSD.TensorFlow/demo/demo1.jpg rename to utils/external/ssd_tensorflow/demo/demo1.jpg diff --git a/utils/external/SSD.TensorFlow/demo/demo2.jpg b/utils/external/ssd_tensorflow/demo/demo2.jpg similarity index 100% rename from utils/external/SSD.TensorFlow/demo/demo2.jpg rename to utils/external/ssd_tensorflow/demo/demo2.jpg diff --git a/utils/external/SSD.TensorFlow/demo/demo3.jpg b/utils/external/ssd_tensorflow/demo/demo3.jpg similarity index 100% rename from utils/external/SSD.TensorFlow/demo/demo3.jpg rename to utils/external/ssd_tensorflow/demo/demo3.jpg diff --git a/utils/external/SSD.TensorFlow/eval_ssd.py b/utils/external/ssd_tensorflow/eval_ssd.py similarity index 100% rename from utils/external/SSD.TensorFlow/eval_ssd.py rename to utils/external/ssd_tensorflow/eval_ssd.py diff --git a/utils/external/SSD.TensorFlow/net/ssd_net.py b/utils/external/ssd_tensorflow/net/ssd_net.py similarity index 100% rename from utils/external/SSD.TensorFlow/net/ssd_net.py rename to utils/external/ssd_tensorflow/net/ssd_net.py diff --git a/utils/external/SSD.TensorFlow/preprocessing/preprocessing_unittest.py b/utils/external/ssd_tensorflow/preprocessing/preprocessing_unittest.py similarity index 100% rename from utils/external/SSD.TensorFlow/preprocessing/preprocessing_unittest.py rename to utils/external/ssd_tensorflow/preprocessing/preprocessing_unittest.py diff --git a/utils/external/SSD.TensorFlow/preprocessing/ssd_preprocessing.py b/utils/external/ssd_tensorflow/preprocessing/ssd_preprocessing.py similarity index 100% rename from utils/external/SSD.TensorFlow/preprocessing/ssd_preprocessing.py rename to utils/external/ssd_tensorflow/preprocessing/ssd_preprocessing.py diff --git a/utils/external/SSD.TensorFlow/simple_ssd_demo.py b/utils/external/ssd_tensorflow/simple_ssd_demo.py similarity index 100% rename from utils/external/SSD.TensorFlow/simple_ssd_demo.py rename to utils/external/ssd_tensorflow/simple_ssd_demo.py diff --git a/utils/external/SSD.TensorFlow/train_ssd.py b/utils/external/ssd_tensorflow/train_ssd.py similarity index 100% rename from utils/external/SSD.TensorFlow/train_ssd.py rename to utils/external/ssd_tensorflow/train_ssd.py diff --git a/utils/external/SSD.TensorFlow/utility/anchor_manipulator.py b/utils/external/ssd_tensorflow/utility/anchor_manipulator.py similarity index 100% rename from utils/external/SSD.TensorFlow/utility/anchor_manipulator.py rename to utils/external/ssd_tensorflow/utility/anchor_manipulator.py diff --git a/utils/external/SSD.TensorFlow/utility/anchor_manipulator_unittest.py b/utils/external/ssd_tensorflow/utility/anchor_manipulator_unittest.py similarity index 100% rename from utils/external/SSD.TensorFlow/utility/anchor_manipulator_unittest.py rename to utils/external/ssd_tensorflow/utility/anchor_manipulator_unittest.py diff --git a/utils/external/SSD.TensorFlow/utility/checkpint_inspect.py b/utils/external/ssd_tensorflow/utility/checkpint_inspect.py similarity index 100% rename from utils/external/SSD.TensorFlow/utility/checkpint_inspect.py rename to utils/external/ssd_tensorflow/utility/checkpint_inspect.py diff --git a/utils/external/SSD.TensorFlow/utility/draw_toolbox.py b/utils/external/ssd_tensorflow/utility/draw_toolbox.py similarity index 100% rename from utils/external/SSD.TensorFlow/utility/draw_toolbox.py rename to utils/external/ssd_tensorflow/utility/draw_toolbox.py diff --git a/utils/external/SSD.TensorFlow/utility/scaffolds.py b/utils/external/ssd_tensorflow/utility/scaffolds.py similarity index 100% rename from utils/external/SSD.TensorFlow/utility/scaffolds.py rename to utils/external/ssd_tensorflow/utility/scaffolds.py diff --git a/utils/external/SSD.TensorFlow/voc_eval.py b/utils/external/ssd_tensorflow/voc_eval.py similarity index 100% rename from utils/external/SSD.TensorFlow/voc_eval.py rename to utils/external/ssd_tensorflow/voc_eval.py From 89ff26791d3770c15d9855a77f7d65d1782ff678 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Sat, 29 Dec 2018 11:44:43 +0800 Subject: [PATCH 029/173] add data input pipeline for Pascal VOC dataset --- datasets/pascalvoc_dataset.py | 189 ++++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 datasets/pascalvoc_dataset.py diff --git a/datasets/pascalvoc_dataset.py b/datasets/pascalvoc_dataset.py new file mode 100644 index 0000000..9e3342e --- /dev/null +++ b/datasets/pascalvoc_dataset.py @@ -0,0 +1,189 @@ +# Tencent is pleased to support the open source community by making PocketFlow available. +# +# Copyright (C) 2018 THL A29 Limited, a Tencent company. All rights reserved. +# +# Licensed under the BSD 3-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Pascal VOC dataset.""" + +import os +import tensorflow as tf + +from datasets.abstract_dataset import AbstractDataset +from utils.external.ssd_tensorflow.preprocessing.ssd_vgg_preprocessing import preprocess_image + +FLAGS = tf.app.flags.FLAGS + +tf.app.flags.DEFINE_integer('image_size', 300, 'output image size') +tf.app.flags.DEFINE_integer('image_size_eval', 300, 'output image size for evaluation') +tf.app.flags.DEFINE_integer('nb_bboxs_max', 100, 'maximal # of bounding boxes per image') +tf.app.flags.DEFINE_integer('nb_smpls_train', 22136, '# of samples for training') +tf.app.flags.DEFINE_integer('nb_smpls_val', 500, '# of samples for validation') +tf.app.flags.DEFINE_integer('nb_smpls_eval', 4952, '# of samples for evaluation') +tf.app.flags.DEFINE_integer('batch_size', 1, 'batch size per GPU for training') +tf.app.flags.DEFINE_integer('batch_size_eval', 1, 'batch size for evaluation') + +# Pascal VOC specifications +IMAGE_HEI = 224 +IMAGE_WID = 224 +IMAGE_CHN = 3 + +def parse_example_proto(example_serialized): + """Parse the unserialized feature data from the serialized data. + + Args: + * example_serialized: serialized example data + + Returns: + * features: unserialized feature data + """ + + # parse features from the serialized data + feature_map = { + 'image/encoded': tf.FixedLenFeature([], dtype=tf.string, default_value=''), + 'image/format': tf.FixedLenFeature([], dtype=tf.string, default_value='jpeg'), + 'image/height': tf.FixedLenFeature([1], dtype=tf.int64), + 'image/width': tf.FixedLenFeature([1], dtype=tf.int64), + 'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32), + 'image/object/bbox/label_text': tf.VarLenFeature(dtype=tf.string), + 'image/object/bbox/label': tf.VarLenFeature(dtype=tf.int64), + 'image/object/bbox/difficult': tf.VarLenFeature(dtype=tf.int64), + 'image/object/bbox/truncated': tf.VarLenFeature(dtype=tf.int64), + } + features = tf.parse_single_example(example_serialized, feature_map) + + return features + +def pack_annotations(bboxes, labels, difficults=None, truncateds=None): + """Pack all the annotations into one tensor. + + Args: + * bboxes: list of bounding box coordinates (N x 4) + * labels: list of category labels (N) + * difficults: list of difficulty flags (N) + * truncateds: list of truncation flags (N) + + Returns: + * objects: one tensor with all the annotations packed together (FLAGS.nb_bboxs_max x 8) + """ + + # pack & with a leading + labels = tf.expand_dims(labels, 1) + flags = tf.ones(tf.shape(labels)) + objects = tf.concat([flags, bboxes, labels], axis=1) + + # pack & if supplied + if difficults is not None and truncateds is not None: + difficults = tf.expand_dims(difficults, 1) + truncateds = tf.expand_dims(truncateds, 1) + objects = tf.concat([objects, difficults, truncateds], axis=1) + + # pad to fixed number of bounding boxes + pad_size = FLAGS.nb_bboxs_max - tf.shape(objects)[0] + objects = tf.pad(objects, [[0, pad_size], [0, 0]]) + + return objects + +def parse_fn(example_serialized, is_train): + """Parse image & objects from the serialized data. + + Args: + * example_serialized: serialized example data + * is_train: whether to construct the training subset + + Returns: + * image: image tensor + * objects: one tensor with all the annotations packed together + """ + + # unserialize the example proto + features = parse_example_proto(example_serialized) + + # obtain the image data + image_raw = tf.image.decode_jpeg(features['image/encoded'], channels=IMAGE_CHN) + + # obtain bounding boxes' coordinates + # Note that we impose an ordering of (y, x) just to make life difficult. + xmins = tf.expand_dims(features['image/object/bbox/xmin'].values, 1) + ymins = tf.expand_dims(features['image/object/bbox/ymin'].values, 1) + xmaxs = tf.expand_dims(features['image/object/bbox/xmax'].values, 1) + ymaxs = tf.expand_dims(features['image/object/bbox/ymax'].values, 1) + bboxes_raw = tf.concat([ymins, xmins, ymaxs, xmaxs], axis=1) # N x 4 + + # obtain other annotation data + labels_raw = tf.cast(tf.sparse_tensor_to_dense(features['image/object/bbox/label']), tf.float32) + difficults = tf.cast(features['image/object/bbox/difficult'].values, tf.float32) + truncateds = tf.cast(features['image/object/bbox/truncated'].values, tf.float32) + + # filter difficult training samples + if is_train: + # if all is difficult, then keep the first one + mask = tf.cond( + tf.count_nonzero(difficults, dtype=tf.int32) < tf.shape(difficults)[0], + lambda: difficults < tf.ones_like(difficults), + lambda: tf.one_hot(0, tf.shape(difficults)[0], on_value=True, off_value=False, dtype=tf.bool)) + labels_raw = tf.boolean_mask(labels_raw, mask) + bboxes_raw = tf.boolean_mask(bboxes_raw, mask) + + # pre-process image, labels, and bboxes + data_format = 'channels_last' # use the channel-last ordering by default + if is_train: + out_shape = (FLAGS.image_size, FLAGS.image_size) + image, labels, bboxes = preprocess_image( + image_raw, labels_raw, bboxes_raw, out_shape, is_training=True, data_format=data_format) + else: + out_shape = (FLAGS.image_size_eval, FLAGS.image_size_eval) + image = preprocess_image(image_raw, labels_raw, bboxes_raw, out_shape) + labels, bboxes = labels_raw, bboxes_raw + + # pack all the annotations into one tensor + objects = pack_annotations(bboxes, labels) + + return image, objects + +class PascalVocDataset(AbstractDataset): + """Pascal VOC dataset.""" + + def __init__(self, is_train): + """Constructor function. + + Args: + * is_train: whether to construct the training subset + """ + + # initialize the base class + super(PascalVocDataset, self).__init__(is_train) + + # choose local files or HDFS files w.r.t. FLAGS.data_disk + if FLAGS.data_disk == 'local': + assert FLAGS.data_dir_local is not None, ' must not be None' + data_dir = FLAGS.data_dir_local + elif FLAGS.data_disk == 'hdfs': + assert FLAGS.data_hdfs_host is not None and FLAGS.data_dir_hdfs is not None, \ + 'both and must not be None' + data_dir = FLAGS.data_hdfs_host + FLAGS.data_dir_hdfs + else: + raise ValueError('unrecognized data disk: ' + FLAGS.data_disk) + + # configure file patterns & function handlers + if is_train: + self.file_pattern = os.path.join(data_dir, '*train*') + self.batch_size = FLAGS.batch_size + else: + self.file_pattern = os.path.join(data_dir, '*val*') + self.batch_size = FLAGS.batch_size_eval + self.dataset_fn = tf.data.TFRecordDataset + self.parse_fn = lambda x: parse_fn(x, is_train=is_train) From 387d7112a41591b3981337b89b6fbbce8effdb2a Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Sat, 29 Dec 2018 17:19:00 +0800 Subject: [PATCH 030/173] add execution script for VGG @ Pascal-VOC --- nets/vgg_at_pascalvoc_run.py | 69 ++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 nets/vgg_at_pascalvoc_run.py diff --git a/nets/vgg_at_pascalvoc_run.py b/nets/vgg_at_pascalvoc_run.py new file mode 100644 index 0000000..2bcf9c2 --- /dev/null +++ b/nets/vgg_at_pascalvoc_run.py @@ -0,0 +1,69 @@ +# Tencent is pleased to support the open source community by making PocketFlow available. +# +# Copyright (C) 2018 THL A29 Limited, a Tencent company. All rights reserved. +# +# Licensed under the BSD 3-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Execution script for VGG models on the Pascal VOC dataset.""" + +import traceback +import tensorflow as tf + +from nets.vgg_at_pascalvoc import ModelHelper +from learners.learner_utils import create_learner + +FLAGS = tf.app.flags.FLAGS + +tf.app.flags.DEFINE_string('log_dir', './logs', 'logging directory') +tf.app.flags.DEFINE_boolean('enbl_multi_gpu', False, 'enable multi-GPU training') +tf.app.flags.DEFINE_string('learner', 'full-prec', 'learner\'s name') +tf.app.flags.DEFINE_string('exec_mode', 'train', 'execution mode: train / eval') +tf.app.flags.DEFINE_boolean('debug', False, 'debugging information') + +def main(unused_argv): + """Main entry.""" + + try: + # setup the TF logging routine + if FLAGS.debug: + tf.logging.set_verbosity(tf.logging.DEBUG) + else: + tf.logging.set_verbosity(tf.logging.INFO) + sm_writer = tf.summary.FileWriter(FLAGS.log_dir) + + # display FLAGS's values + tf.logging.info('FLAGS:') + for key, value in FLAGS.flag_values_dict().items(): + tf.logging.info('{}: {}'.format(key, value)) + + # build the model helper & learner + model_helper = ModelHelper() + learner = create_learner(sm_writer, model_helper) + + # execute the learner + if FLAGS.exec_mode == 'train': + learner.train() + elif FLAGS.exec_mode == 'eval': + learner.download_model() + learner.evaluate() + else: + raise ValueError('unrecognized execution mode: ' + FLAGS.exec_mode) + + # exit normally + return 0 + except ValueError: + traceback.print_exc() + return 1 # exit with errors + +if __name__ == '__main__': + tf.app.run() From bc7793a992c9c3c3d10ce82b5fe2c3f6e8ca68a4 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Wed, 2 Jan 2019 15:02:28 +0800 Subject: [PATCH 031/173] add ModelHelper for VGG @ Pascal-VOC --- nets/vgg_at_pascalvoc.py | 510 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 510 insertions(+) create mode 100644 nets/vgg_at_pascalvoc.py diff --git a/nets/vgg_at_pascalvoc.py b/nets/vgg_at_pascalvoc.py new file mode 100644 index 0000000..4a027f8 --- /dev/null +++ b/nets/vgg_at_pascalvoc.py @@ -0,0 +1,510 @@ +# Tencent is pleased to support the open source community by making PocketFlow available. +# +# Copyright (C) 2018 THL A29 Limited, a Tencent company. All rights reserved. +# +# Licensed under the BSD 3-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model helper for creating a VGG model for the Pascal VOC dataset.""" + +import tensorflow as tf +from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw + +from nets.abstract_model_helper import AbstractModelHelper +from datasets.pascalvoc_dataset import PascalVocDataset + +from utils.external.ssd_tensorflow.net import ssd_net +from utils.external.ssd_tensorflow.utility import anchor_manipulator +from utils.external.ssd_tensorflow.utility import scaffolds + +FLAGS = tf.app.flags.FLAGS + +### REQUIRED ### +#tf.app.flags.DEFINE_integer('resnet_size', 20, '# of layers in the ResNet model') +#tf.app.flags.DEFINE_float('nb_epochs_rat', 1.0, '# of training epochs\'s ratio') +#tf.app.flags.DEFINE_float('lrn_rate_init', 1e-1, 'initial learning rate') +#tf.app.flags.DEFINE_float('batch_size_norm', 128, 'normalization factor of batch size') +#tf.app.flags.DEFINE_float('momentum', 0.9, 'momentum coefficient') +#tf.app.flags.DEFINE_float('loss_w_dcy', 2e-4, 'weight decaying loss\'s coefficient') +### REQUIRED ### + +# hardware related configuration +tf.app.flags.DEFINE_integer('num_readers', 8, + 'The number of parallel readers that read data from the dataset.') +tf.app.flags.DEFINE_integer('num_preprocessing_threads', 24, + 'The number of threads used to create the batches.') +tf.app.flags.DEFINE_integer('num_cpu_threads', 0, 'The number of cpu cores used to train.') +tf.app.flags.DEFINE_float('gpu_memory_fraction', 1.0, 'GPU memory fraction to use.') + +# scaffold related configuration +tf.app.flags.DEFINE_string('data_dir', './tfrecords', + 'The directory where the dataset input data is stored.') +tf.app.flags.DEFINE_integer('num_classes', 21, 'Number of classes to use in the dataset.') +tf.app.flags.DEFINE_string('model_dir', './logs/', 'The directory where the model will be stored.') +tf.app.flags.DEFINE_integer('log_every_n_steps', 10, 'The frequency with which logs are printed.') +tf.app.flags.DEFINE_integer('save_summary_steps', 500, + 'The frequency with which summaries are saved, in seconds.') +tf.app.flags.DEFINE_integer('save_checkpoints_secs', 7200, + 'The frequency with which the model is saved, in seconds.') + +# model related configuration +tf.app.flags.DEFINE_integer('train_image_size', 300, + 'The size of the input image for the model to use.') +tf.app.flags.DEFINE_integer('train_epochs', None, 'The number of epochs to use for training.') +tf.app.flags.DEFINE_integer('max_number_of_steps', 120000, + 'The max number of steps to use for training.') +tf.app.flags.DEFINE_string('data_format', 'channels_last', # 'channels_first' or 'channels_last' + 'A flag to override the data format used in the model. channels_first ' + 'provides a performance boost on GPU but is not always compatible ' + 'with CPU. If left unspecified, the data format will be chosen ' + 'automatically based on whether TensorFlow was built for CPU or GPU.') +tf.app.flags.DEFINE_float('negative_ratio', 3.0, 'Negative ratio in the loss function.') +tf.app.flags.DEFINE_float('match_threshold', 0.5, 'Matching threshold in the loss function.') +tf.app.flags.DEFINE_float('neg_threshold', 0.5, + 'Matching threshold for the negtive examples in the loss function.') + +# optimizer related configuration +tf.app.flags.DEFINE_integer('tf_random_seed', 20190101, 'Random seed for TensorFlow initializers.') +tf.app.flags.DEFINE_float('weight_decay', 5e-4, 'The weight decay on the model weights.') +tf.app.flags.DEFINE_float('momentum', 0.9, + 'The momentum for the MomentumOptimizer and RMSPropOptimizer.') +tf.app.flags.DEFINE_float('learning_rate', 1e-3, 'Initial learning rate.') +tf.app.flags.DEFINE_float('end_learning_rate', 1e-6, + 'The minimal end learning rate used by a polynomial decay learning rate.') + +# for learning rate piecewise_constant decay +tf.app.flags.DEFINE_string('decay_boundaries', '500, 80000, 100000', + 'Learning rate decay boundaries by global_step (comma-separated list).') +tf.app.flags.DEFINE_string('lr_decay_factors', '0.1, 1, 0.1, 0.01', + 'The values of learning_rate decay factor for each segment between ' + 'boundaries (comma-separated list).') + +# checkpoint related configuration +tf.app.flags.DEFINE_string('checkpoint_path', './model/', + 'The path to a checkpoint from which to fine-tune.') +tf.app.flags.DEFINE_string('checkpoint_model_scope', 'vgg_16', + 'Model scope in the checkpoint. None if the same as the trained model.') +tf.app.flags.DEFINE_string('model_scope', 'ssd300', + 'Model scope name used to replace the name_scope in checkpoint.') +tf.app.flags.DEFINE_string('checkpoint_exclude_scopes', + 'ssd300/multibox_head, ssd300/additional_layers, ssd300/conv4_3_scale', + 'Comma-separated list of scopes of variables to exclude when restoring ' + 'from a checkpoint.') +tf.app.flags.DEFINE_boolean('ignore_missing_vars', True, + 'When restoring a checkpoint would ignore missing variables.') +tf.app.flags.DEFINE_boolean('multi_gpu', False, 'Whether there is GPU to use for training.') + +params = {} + +def parse_comma_list(args): + """Convert a comma-separated list to a list of floating-point numbers.""" + + return [float(s.strip()) for s in args.split(',')] + +def setup_params(): + """Setup hyper-parameters (from FLAGS to dict).""" + + global params + params = { + 'num_gpus': 1, + 'max_number_of_steps': FLAGS.max_number_of_steps, + 'train_image_size': FLAGS.train_image_size, + 'data_format': FLAGS.data_format, + 'batch_size': FLAGS.batch_size, + 'model_scope': FLAGS.model_scope, + 'num_classes': FLAGS.num_classes, + 'negative_ratio': FLAGS.negative_ratio, + 'match_threshold': FLAGS.match_threshold, + 'neg_threshold': FLAGS.neg_threshold, + 'weight_decay': FLAGS.weight_decay, + 'momentum': FLAGS.momentum, + 'learning_rate': FLAGS.learning_rate, + 'end_learning_rate': FLAGS.end_learning_rate, + 'decay_boundaries': parse_comma_list(FLAGS.decay_boundaries), + 'lr_decay_factors': parse_comma_list(FLAGS.lr_decay_factors) + } + +def __setup_anchor_info(): + """Setup the anchor bounding boxes' information.""" + + # get all anchor bounding boxes + out_shape = [params['train_image_size']] * 2 + anchor_creator = anchor_manipulator.AnchorCreator( + out_shape, + layers_shapes=[(38, 38), (19, 19), (10, 10), (5, 5), (3, 3), (1, 1)], + anchor_scales=[(0.1,), (0.2,), (0.375,), (0.55,), (0.725,), (0.9,)], + extra_anchor_scales=[(0.1414,), (0.2739,), (0.4541,), (0.6315,), (0.8078,), (0.9836,)], + anchor_ratios=[(1., 2., .5), (1., 2., 3., .5, 0.3333), (1., 2., 3., .5, 0.3333), + (1., 2., 3., .5, 0.3333), (1., 2., .5), (1., 2., .5)], + layer_steps=[8, 16, 32, 64, 100, 300]) + all_anchors, all_num_anchors_depth, all_num_anchors_spatial = anchor_creator.get_all_anchors() + + # construct the anchor bounding boxes' encoder & decoder + num_anchors_per_layer = [] + for ind in range(len(all_anchors)): + num_anchors_per_layer.append(all_num_anchors_depth[ind] * all_num_anchors_spatial[ind]) + anchor_encoder = anchor_manipulator.AnchorEncoder( + allowed_borders=[1.0] * 6, positive_threshold=params['match_threshold'], + ignore_threshold=params['neg_threshold'], prior_scaling=[0.1, 0.1, 0.2, 0.2]) + + # pack all the information into one dictionary + anchor_info = { + 'encode_fn': lambda glabels_, gbboxes_: anchor_encoder.encode_all_anchors( + glabels_, gbboxes_, all_anchors, all_num_anchors_depth, all_num_anchors_spatial), + 'decode_fn': lambda pred: anchor_encoder.decode_all_anchors(pred, num_anchors_per_layer), + 'num_anchors_per_layer': num_anchors_per_layer, + 'all_num_anchors_depth': all_num_anchors_depth, + } + + return anchor_info + +def modified_smooth_l1( + bbox_pred, bbox_targets, bbox_inside_weights=1., bbox_outside_weights=1., sigma=1.): + """Modified smooth L1-loss. + + Description: + * ResultLoss = outside_weights * SmoothL1(inside_weights * (bbox_pred - bbox_targets)) + * SmoothL1(x) = 0.5 * (sigma * x)^2, if |x| < 1 / sigma^2 + |x| - 0.5 / sigma^2, otherwise + """ + + with tf.name_scope('smooth_l1', [bbox_pred, bbox_targets]): + sigma2 = sigma * sigma + + inside_mul = tf.multiply(bbox_inside_weights, tf.subtract(bbox_pred, bbox_targets)) + + smooth_l1_sign = tf.cast(tf.less(tf.abs(inside_mul), 1.0 / sigma2), tf.float32) + smooth_l1_option1 = tf.multiply(tf.multiply(inside_mul, inside_mul), 0.5 * sigma2) + smooth_l1_option2 = tf.subtract(tf.abs(inside_mul), 0.5 / sigma2) + smooth_l1_result = tf.add(tf.multiply(smooth_l1_option1, smooth_l1_sign), + tf.multiply(smooth_l1_option2, tf.abs(smooth_l1_sign - 1.0))) + + outside_mul = tf.multiply(bbox_outside_weights, smooth_l1_result) + + return outside_mul + +def forward_fn(inputs, is_train, data_format, anchor_info): + """Forward pass function. + + Args: + * inputs: input tensor to the network's forward pass + * is_train: whether to use the forward pass with training operations inserted + * data_format: data format ('channels_last' OR 'channels_first') + * anchor_info: anchor bounding boxes' information + + Returns: + * outputs: a dictionary of output tensors + """ + + all_num_anchors_depth = anchor_info['all_num_anchors_depth'] + with tf.variable_scope(params['model_scope'], values=[inputs], reuse=tf.AUTO_REUSE): + # obtain predictions for localization & classification + backbone = ssd_net.VGG16Backbone(data_format) + feature_layers = backbone.forward(inputs, training=is_train) + loc_pred, cls_pred = ssd_net.multibox_head( + feature_layers, params['num_classes'], all_num_anchors_depth, data_format=data_format) + if data_format == 'channels_first': + cls_pred = [tf.transpose(pred, [0, 2, 3, 1]) for pred in cls_pred] + loc_pred = [tf.transpose(pred, [0, 2, 3, 1]) for pred in loc_pred] + + # flatten predictions + def reshape_fn(preds, nb_dims): + preds = [tf.reshape(pred, [tf.shape(inputs)[0], -1, nb_dims]) for pred in preds] + preds = tf.concat(preds, axis=1) + preds = tf.reshape(preds, [-1, nb_dims]) + return preds + cls_pred = reshape_fn(cls_pred, params['num_classes']) + loc_pred = reshape_fn(loc_pred, 4) + + # pack all the output tensors together + outputs = {'cls_pred': cls_pred, 'loc_pred': loc_pred} + + return outputs + +def calc_loss_fn(objects, outputs, trainable_vars, anchor_info): + """Calculate the loss function's value. + + Args: + * objects: one tensor with all the annotations packed together + * outputs: a dictionary of output tensors + * trainable_vars: list of trainable variables + * anchor_info: anchor bounding boxes' information + + Returns: + * loss: loss function's value + * metrics: dictionary of extra evaluation metrics + """ + + # extract output tensors + batch_size = params['batch_size'] + cls_pred = outputs['cls_pred'] + loc_pred = outputs['loc_pred'] + + # extract anchor bounding boxes' information + encode_fn = anchor_info['encode_fn'] + decode_fn = anchor_info['decode_fn'] + num_anchors_per_layer = anchor_info['num_anchors_per_layer'] + all_num_anchors_depth = anchor_info['all_num_anchors_depth'] + + # extract target localization & classification results from + # TODO use tf.map_fn + gt_locations_list = [] + gt_labels_list = [] + gt_scores_list = [] + b_flags, b_bboxes, b_labels = tf.split(objects, [1, 4, 1], -1) + b_flags = tf.squeeze(tf.cast(b_flags, dtype=tf.int64), axis=-1) + b_labels = tf.squeeze(tf.cast(b_labels, dtype=tf.int64), axis=-1) + for batch_index in range(batch_size): + index = tf.where(b_flags[batch_index] > 0) + labels = tf.gather_nd(b_labels[batch_index], index) + bboxes = tf.gather_nd(b_bboxes[batch_index], index) + gt_locations, gt_labels, gt_scores = encode_fn(labels, bboxes) + gt_locations_list += [tf.expand_dims(gt_locations, 0)] + gt_labels_list += [tf.expand_dims(gt_labels, 0)] + gt_scores_list += [tf.expand_dims(gt_scores, 0)] + loc_targets = tf.concat(gt_locations_list, axis=0) + cls_targets = tf.concat(gt_labels_list, axis=0) + match_scores = tf.concat(gt_scores_list, axis=0) + + # post-forward operations + with tf.control_dependencies([cls_pred, loc_pred]): + with tf.name_scope('post_forward'): + bboxes_pred = tf.map_fn(lambda _preds: decode_fn(_preds), + tf.reshape(loc_pred, [batch_size, -1, 4]), + dtype=[tf.float32] * len(num_anchors_per_layer), back_prop=False) + bboxes_pred = [tf.reshape(preds, [-1, 4]) for preds in bboxes_pred] + bboxes_pred = tf.concat(bboxes_pred, axis=0) + + flatten_loc_targets = tf.reshape(loc_targets, [-1, 4]) + flatten_cls_targets = tf.reshape(cls_targets, [-1]) + flatten_match_scores = tf.reshape(match_scores, [-1]) + + # each positive examples has one label + positive_mask = flatten_cls_targets > 0 + n_positives = tf.count_nonzero(positive_mask) + batch_n_positives = tf.count_nonzero(cls_targets, -1) + batch_negtive_mask = tf.equal(cls_targets, 0) + batch_n_negtives = tf.count_nonzero(batch_negtive_mask, -1) + batch_n_neg_select = tf.cast( + params['negative_ratio'] * tf.cast(batch_n_positives, tf.float32), tf.int32) + batch_n_neg_select = tf.minimum(batch_n_neg_select, tf.cast(batch_n_negtives, tf.int32)) + + # hard negative mining for classification + predictions_for_bg = tf.nn.softmax( + tf.reshape(cls_pred, [batch_size, -1, params['num_classes']]))[:, :, 0] + prob_for_negtives = tf.where(batch_negtive_mask, + 0. - predictions_for_bg, + 0. - tf.ones_like(predictions_for_bg)) + topk_prob_for_bg, _ = tf.nn.top_k(prob_for_negtives, k=tf.shape(prob_for_negtives)[1]) + score_at_k = tf.gather_nd(topk_prob_for_bg, + tf.stack([tf.range(batch_size), batch_n_neg_select - 1], axis=-1)) + selected_neg_mask = prob_for_negtives >= tf.expand_dims(score_at_k, axis=-1) + + # include both selected negtive and all positive examples + final_mask = tf.stop_gradient(tf.logical_or( + tf.reshape(tf.logical_and(batch_negtive_mask, selected_neg_mask), [-1]), positive_mask)) + total_examples = tf.count_nonzero(final_mask) + + cls_pred = tf.boolean_mask(cls_pred, final_mask) + loc_pred = tf.boolean_mask(loc_pred, tf.stop_gradient(positive_mask)) + flatten_cls_targets = tf.boolean_mask( + tf.clip_by_value(flatten_cls_targets, 0, params['num_classes']), final_mask) + flatten_loc_targets = tf.stop_gradient(tf.boolean_mask(flatten_loc_targets, positive_mask)) + + # final predictions & classification accuracy + predictions = { + 'classes': tf.argmax(cls_pred, axis=-1), + 'probabilities': tf.reduce_max(tf.nn.softmax(cls_pred, name='softmax_tensor'), axis=-1), + 'loc_predict': bboxes_pred, + } + cls_accuracy = tf.metrics.accuracy(flatten_cls_targets, predictions['classes']) + tf.identity(cls_accuracy[1], name='cls_accuracy') + tf.summary.scalar('cls_accuracy', cls_accuracy[1]) + metrics = {'accuracy': cls_accuracy[1]) + + # cross-entropy loss + ce_loss = (params['negative_ratio'] + 1.) * \ + tf.losses.sparse_softmax_cross_entropy(flatten_cls_targets, cls_pred) + tf.identity(ce_loss, name='cross_entropy_loss') + tf.summary.scalar('cross_entropy_loss', ce_loss) + + # localization loss + loc_loss = tf.reduce_mean( + tf.reduce_sum(modified_smooth_l1(loc_pred, flatten_loc_targets, sigma=1.), axis=-1)) + tf.identity(loc_loss, name='localization_loss') + tf.summary.scalar('localization_loss', loc_loss) + + # L2-regularization loss + l2_loss_list = [] + for var in trainable_vars: + if '_bn' not in var.name: + if 'conv4_3_scale' not in var.name: + l2_loss_list.append(tf.nn.l2_loss(var)) + else: + l2_loss_list.append(tf.nn.l2_loss(var) * 0.1) + l2_loss = tf.add_n(l2_loss_list) + tf.identity(loc_loss, name='localization_loss') + tf.summary.scalar('localization_loss', loc_loss) + + # overall loss + loss = ce_loss + loc_loss + params['weight_decay'] * tf.add_n(l2_loss) + + return loss, metrics + +class ModelHelper(AbstractModelHelper): + """Model helper for creating a VGG model for the VOC dataset.""" + + def __init__(self): + """Constructor function.""" + + # class-independent initialization + super(ModelHelper, self).__init__() + + # initialize training & evaluation subsets + self.dataset_train = PascalVocDataset(is_train=True) + self.dataset_eval = PascalVocDataset(is_train=False) + + # setup hyper-parameters & anchor information + setup_params() + self.anchor_info = setup_anchor_info() + + def build_dataset_train(self, enbl_trn_val_split=False): + """Build the data subset for training, usually with data augmentation.""" + + return self.dataset_train.build() + + def build_dataset_eval(self): + """Build the data subset for evaluation, usually without data augmentation.""" + + return self.dataset_eval.build() + + def forward_train(self, inputs, data_format='channels_last'): + """Forward computation at training.""" + + return forward_fn(inputs, True, data_format, self.anchor_info) + + def forward_eval(self, inputs, data_format='channels_last'): + """Forward computation at evaluation.""" + + return forward_fn(inputs, False, data_format, self.anchor_info) + + def calc_loss(self, objects, outputs, trainable_vars): + """Calculate loss (and some extra evaluation metrics).""" + + return calc_loss_fn(objects, outputs, trainable_vars, self.anchor_info) + + def get_init_fn(self, sess): + """Get the initialization function. + + We use a pre-trained ImageNet classification model to initialize the backbone part of the SSD + model for feature extraction. If the SSD model's checkpoint files already exist, then skip. + + Args: + * sess: TensorFlow session to restore model weights + """ + + # early return if checkpoint files already exist + if tf.train.latest_checkpoint(FLAGS.model_dir): + tf.logging.info('checkpoint files already exist in ' + FLAGS.model_dir) + return + + # obtain a list of scopes to be excluded from initialization + excluded_scopes = [] + if FLAGS.checkpoint_exclude_scopes: + excluded_scopes = [scope.strip() for scope in FLAGS.checkpoint_exclude_scopes.split(',')] + tf.logging.info('excluded scopes: {}'.format(excluded_scopes)) + + # obtain a list of variables to be initialized + vars_list = [] + for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES): + excluded = False + for scope in excluded_scopes: + if var.name.startswith(scope): + excluded = True + break + if not excluded: + vars_list.append(var) + + # rename variables to be initialized + if FLAGS.checkpoint_model_scope is not None: + # rename the variable scope + if FLAGS.checkpoint_model_scope.strip() == '': + vars_list = {var.op.name.replace(FLAGS.model_scope + '/', ''): var for var in vars_list} + else: + vars_list = {var.op.name.replace( + FLAGS.model_scope, FLAGS.checkpoint_model_scope.strip()): var for var in vars_list} + + # re-map the variable's name + name_remap = {'/kernel': '/weights', '/bias': '/biases'} + vars_list_remap = {} + for var_name, var in vars_list.items(): + for name_old, name_new in name_remap.items(): + if name_old in var_name: + var_name = var_name.replace(name_old, name_new) + break + vars_list_remap[var_name] = var + vars_list = vars_list_remap + + # display all the variables to be initialized + for var_name, var in vars_list.items(): + tf.logging.info('using %s to initialize %s' % (var_name, var.op.name)) + if not vars_list: + raise ValueError('variables to be restored cannot be empty') + + # obtain the checkpoint files' path + if tf.gfile.IsDirectory(FLAGS.checkpoint_path): + ckpt_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path) + else: + ckpt_path = FLAGS.checkpoint_path + tf.logging.info('restoring model weights from ' + ckpt_path) + + # remove missing variables from the list + if FLAGS.ignore_missing_vars: + reader = tf.train.NewCheckpointReader(ckpt_path) + vars_list_avail = {} + for var in var_list: + if reader.has_tensor(var): + vars_list_avail[var] = var_list[var] + else: + tf.logging.warning('variable %s not found in checkpoint files %s.' % (var, ckpt_path)) + vars_list = vars_list_avail + if not vars_list: + tf.logging.warning('no variables to restore.') + return + + # restore variables from checkpoint files + saver = tf.train.Saver(vars_list, reshape=False) + saver.build() + saver.restore(sess, ckpt_path) + + def setup_lrn_rate(self, global_step): + """Setup the learning rate (and number of training iterations).""" + + bnds = [int(x) for x in params['decay_boundaries']] + vals = [params['learning_rate'] * x for x in params['lr_decay_factors']] + lrn_rate = tf.train.piecewise_constant(global_step, bnds, vals) + lrn_rate = tf.maximum(lrn_rate, tf.constant(params['end_learning_rate'], dtype=lrn_rate.dtype)) + nb_iters = params['max_number_of_steps'] + + return lrn_rate, nb_iters + + @property + def model_name(self): + """Model's name.""" + + return 'ssd_vgg_300' + + @property + def dataset_name(self): + """Dataset's name.""" + + return 'pascalvoc' From 16fefaa155c17b8a5787b535eec4a885f3b7b990 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Wed, 2 Jan 2019 15:24:07 +0800 Subject: [PATCH 032/173] add warm_start() to --- nets/abstract_model_helper.py | 8 ++++++++ nets/vgg_at_pascalvoc.py | 32 +++++++++++++++----------------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/nets/abstract_model_helper.py b/nets/abstract_model_helper.py index f91a162..a589b02 100644 --- a/nets/abstract_model_helper.py +++ b/nets/abstract_model_helper.py @@ -115,6 +115,14 @@ def setup_lrn_rate(self, global_step): """ pass + def warm_start(self, sess): + """Initialize the model for warm-start. + + Args: + * sess: TensorFlow session + """ + pass + @property @abstractmethod def model_name(self): diff --git a/nets/vgg_at_pascalvoc.py b/nets/vgg_at_pascalvoc.py index 4a027f8..457ecfd 100644 --- a/nets/vgg_at_pascalvoc.py +++ b/nets/vgg_at_pascalvoc.py @@ -402,14 +402,23 @@ def calc_loss(self, objects, outputs, trainable_vars): return calc_loss_fn(objects, outputs, trainable_vars, self.anchor_info) - def get_init_fn(self, sess): - """Get the initialization function. + def setup_lrn_rate(self, global_step): + """Setup the learning rate (and number of training iterations).""" - We use a pre-trained ImageNet classification model to initialize the backbone part of the SSD - model for feature extraction. If the SSD model's checkpoint files already exist, then skip. + bnds = [int(x) for x in params['decay_boundaries']] + vals = [params['learning_rate'] * x for x in params['lr_decay_factors']] + lrn_rate = tf.train.piecewise_constant(global_step, bnds, vals) + lrn_rate = tf.maximum(lrn_rate, tf.constant(params['end_learning_rate'], dtype=lrn_rate.dtype)) + nb_iters = params['max_number_of_steps'] + + return lrn_rate, nb_iters + + def warm_start(self, sess): + """Initialize the model for warm-start. - Args: - * sess: TensorFlow session to restore model weights + Description: + * We use a pre-trained ImageNet classification model to initialize the backbone part of the SSD + model for feature extraction. If the SSD model's checkpoint files already exist, then skip. """ # early return if checkpoint files already exist @@ -486,17 +495,6 @@ def get_init_fn(self, sess): saver.build() saver.restore(sess, ckpt_path) - def setup_lrn_rate(self, global_step): - """Setup the learning rate (and number of training iterations).""" - - bnds = [int(x) for x in params['decay_boundaries']] - vals = [params['learning_rate'] * x for x in params['lr_decay_factors']] - lrn_rate = tf.train.piecewise_constant(global_step, bnds, vals) - lrn_rate = tf.maximum(lrn_rate, tf.constant(params['end_learning_rate'], dtype=lrn_rate.dtype)) - nb_iters = params['max_number_of_steps'] - - return lrn_rate, nb_iters - @property def model_name(self): """Model's name.""" From c88cd24513a2a0f73d7d0cdea8483370a8fa3f13 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Wed, 2 Jan 2019 15:33:25 +0800 Subject: [PATCH 033/173] call warm_start() in the learner --- learners/abstract_learner.py | 1 + learners/full_precision/learner.py | 1 + 2 files changed, 2 insertions(+) diff --git a/learners/abstract_learner.py b/learners/abstract_learner.py index a9dcda1..fe9f883 100644 --- a/learners/abstract_learner.py +++ b/learners/abstract_learner.py @@ -78,6 +78,7 @@ def __init__(self, sm_writer, model_helper): self.forward_eval = model_helper.forward_eval self.calc_loss = model_helper.calc_loss self.setup_lrn_rate = model_helper.setup_lrn_rate + self.warm_start = model_helper.warm_start self.model_name = model_helper.model_name self.dataset_name = model_helper.dataset_name diff --git a/learners/full_precision/learner.py b/learners/full_precision/learner.py index 473a2aa..bc709a5 100644 --- a/learners/full_precision/learner.py +++ b/learners/full_precision/learner.py @@ -59,6 +59,7 @@ def train(self): # initialization self.sess_train.run(self.init_op) + self.warm_start(self.sess_train) if FLAGS.enbl_multi_gpu: self.sess_train.run(self.bcast_op) From 55f6600afe9da4389625a5c82de9a980aac8e92e Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Wed, 2 Jan 2019 15:55:59 +0800 Subject: [PATCH 034/173] do not build the evaluation graph for SSD models --- learners/full_precision/learner.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/learners/full_precision/learner.py b/learners/full_precision/learner.py index bc709a5..ec52555 100644 --- a/learners/full_precision/learner.py +++ b/learners/full_precision/learner.py @@ -52,7 +52,7 @@ def __init__(self, sm_writer, model_helper, model_scope=None, enbl_dst=None): if self.enbl_dst: self.helper_dst = DistillationHelper(sm_writer, model_helper, self.mpi_comm) self.__build(is_train=True) - self.__build(is_train=False) + #self.__build(is_train=False) def train(self): """Train a model and periodically produce checkpoint files.""" @@ -79,14 +79,14 @@ def train(self): # save & evaluate the model at certain steps if self.is_primary_worker('global') and (idx_iter + 1) % FLAGS.save_step == 0: self.__save_model(is_train=True) - self.evaluate() + #self.evaluate() # save the final model if self.is_primary_worker('global'): self.__save_model(is_train=True) - self.__restore_model(is_train=False) - self.__save_model(is_train=False) - self.evaluate() + #self.__restore_model(is_train=False) + #self.__save_model(is_train=False) + #self.evaluate() def evaluate(self): """Restore a model from the latest checkpoint files and then evaluate it.""" From bef22802ff031d13927599be2ffb7daf38672b9f Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Wed, 2 Jan 2019 17:17:47 +0800 Subject: [PATCH 035/173] define as list; typo fixed --- datasets/pascalvoc_dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datasets/pascalvoc_dataset.py b/datasets/pascalvoc_dataset.py index 9e3342e..b684fe1 100644 --- a/datasets/pascalvoc_dataset.py +++ b/datasets/pascalvoc_dataset.py @@ -20,7 +20,7 @@ import tensorflow as tf from datasets.abstract_dataset import AbstractDataset -from utils.external.ssd_tensorflow.preprocessing.ssd_vgg_preprocessing import preprocess_image +from utils.external.ssd_tensorflow.preprocessing.ssd_preprocessing import preprocess_image FLAGS = tf.app.flags.FLAGS @@ -141,11 +141,11 @@ def parse_fn(example_serialized, is_train): # pre-process image, labels, and bboxes data_format = 'channels_last' # use the channel-last ordering by default if is_train: - out_shape = (FLAGS.image_size, FLAGS.image_size) + out_shape = [FLAGS.image_size, FLAGS.image_size] image, labels, bboxes = preprocess_image( image_raw, labels_raw, bboxes_raw, out_shape, is_training=True, data_format=data_format) else: - out_shape = (FLAGS.image_size_eval, FLAGS.image_size_eval) + out_shape = [FLAGS.image_size_eval, FLAGS.image_size_eval] image = preprocess_image(image_raw, labels_raw, bboxes_raw, out_shape) labels, bboxes = labels_raw, bboxes_raw From 0bd6d398dcd9e403ae6404a91a0a714cf764a889 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Wed, 2 Jan 2019 17:18:58 +0800 Subject: [PATCH 036/173] use empty model scope; pass list of vars to warm_start() --- learners/full_precision/learner.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/learners/full_precision/learner.py b/learners/full_precision/learner.py index ec52555..ea75476 100644 --- a/learners/full_precision/learner.py +++ b/learners/full_precision/learner.py @@ -48,6 +48,8 @@ def __init__(self, sm_writer, model_helper, model_scope=None, enbl_dst=None): self.model_scope = model_scope self.enbl_dst = enbl_dst if enbl_dst is not None else FLAGS.enbl_dst + self.model_scope = '' + # class-dependent initialization if self.enbl_dst: self.helper_dst = DistillationHelper(sm_writer, model_helper, self.mpi_comm) @@ -59,7 +61,7 @@ def train(self): # initialization self.sess_train.run(self.init_op) - self.warm_start(self.sess_train) + self.warm_start(self.sess_train, self.trainable_vars_cache) if FLAGS.enbl_multi_gpu: self.sess_train.run(self.bcast_op) @@ -157,6 +159,7 @@ def __build(self, is_train): # pylint: disable=too-many-locals if FLAGS.enbl_multi_gpu: self.bcast_op = mgw.broadcast_global_variables(0) self.saver_train = tf.train.Saver(self.vars) + self.trainable_vars_cache = self.trainable_vars else: self.sess_eval = sess self.eval_op = [loss] + list(metrics.values()) From 453b9ddd2897cab34ffae5a9a6ec0db20fced74a Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Wed, 2 Jan 2019 17:20:28 +0800 Subject: [PATCH 037/173] require list of variables when calling warm_start() --- nets/abstract_model_helper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nets/abstract_model_helper.py b/nets/abstract_model_helper.py index a589b02..ab9bda8 100644 --- a/nets/abstract_model_helper.py +++ b/nets/abstract_model_helper.py @@ -115,11 +115,12 @@ def setup_lrn_rate(self, global_step): """ pass - def warm_start(self, sess): + def warm_start(self, sess, vars_list): """Initialize the model for warm-start. Args: * sess: TensorFlow session + * vars_list: list of variables to be updated """ pass From c8e5c2c508ac050193ee5ad3a5f2f785ef376d0f Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Wed, 2 Jan 2019 17:23:03 +0800 Subject: [PATCH 038/173] minor bugs fixed; run passed at local & seven --- nets/vgg_at_pascalvoc.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/nets/vgg_at_pascalvoc.py b/nets/vgg_at_pascalvoc.py index 457ecfd..675f31b 100644 --- a/nets/vgg_at_pascalvoc.py +++ b/nets/vgg_at_pascalvoc.py @@ -133,7 +133,7 @@ def setup_params(): 'lr_decay_factors': parse_comma_list(FLAGS.lr_decay_factors) } -def __setup_anchor_info(): +def setup_anchor_info(): """Setup the anchor bounding boxes' information.""" # get all anchor bounding boxes @@ -326,10 +326,11 @@ def calc_loss_fn(objects, outputs, trainable_vars, anchor_info): 'probabilities': tf.reduce_max(tf.nn.softmax(cls_pred, name='softmax_tensor'), axis=-1), 'loc_predict': bboxes_pred, } - cls_accuracy = tf.metrics.accuracy(flatten_cls_targets, predictions['classes']) - tf.identity(cls_accuracy[1], name='cls_accuracy') - tf.summary.scalar('cls_accuracy', cls_accuracy[1]) - metrics = {'accuracy': cls_accuracy[1]) + #cls_accuracy = tf.metrics.accuracy(flatten_cls_targets, predictions['classes']) + #tf.identity(cls_accuracy[1], name='cls_accuracy') + #tf.summary.scalar('cls_accuracy', cls_accuracy[1]) + #metrics = {'accuracy': cls_accuracy[1]} + metrics = {} # cross-entropy loss ce_loss = (params['negative_ratio'] + 1.) * \ @@ -356,7 +357,7 @@ def calc_loss_fn(objects, outputs, trainable_vars, anchor_info): tf.summary.scalar('localization_loss', loc_loss) # overall loss - loss = ce_loss + loc_loss + params['weight_decay'] * tf.add_n(l2_loss) + loss = ce_loss + loc_loss + params['weight_decay'] * l2_loss return loss, metrics @@ -375,7 +376,7 @@ def __init__(self): # setup hyper-parameters & anchor information setup_params() - self.anchor_info = setup_anchor_info() + self.anchor_info = None def build_dataset_train(self, enbl_trn_val_split=False): """Build the data subset for training, usually with data augmentation.""" @@ -390,12 +391,15 @@ def build_dataset_eval(self): def forward_train(self, inputs, data_format='channels_last'): """Forward computation at training.""" + if self.anchor_info is None: + self.anchor_info = setup_anchor_info() + return forward_fn(inputs, True, data_format, self.anchor_info) def forward_eval(self, inputs, data_format='channels_last'): """Forward computation at evaluation.""" - return forward_fn(inputs, False, data_format, self.anchor_info) + return forward_fn(inputs, False, data_format, self.anchor_info) # FIXME cannot build def calc_loss(self, objects, outputs, trainable_vars): """Calculate loss (and some extra evaluation metrics).""" @@ -413,7 +417,7 @@ def setup_lrn_rate(self, global_step): return lrn_rate, nb_iters - def warm_start(self, sess): + def warm_start(self, sess, vars_list): """Initialize the model for warm-start. Description: @@ -433,15 +437,16 @@ def warm_start(self, sess): tf.logging.info('excluded scopes: {}'.format(excluded_scopes)) # obtain a list of variables to be initialized - vars_list = [] - for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES): + vars_list_scope = [] + for var in vars_list: excluded = False for scope in excluded_scopes: if var.name.startswith(scope): excluded = True break if not excluded: - vars_list.append(var) + vars_list_scope.append(var) + vars_list = vars_list_scope # rename variables to be initialized if FLAGS.checkpoint_model_scope is not None: @@ -480,9 +485,9 @@ def warm_start(self, sess): if FLAGS.ignore_missing_vars: reader = tf.train.NewCheckpointReader(ckpt_path) vars_list_avail = {} - for var in var_list: + for var in vars_list: if reader.has_tensor(var): - vars_list_avail[var] = var_list[var] + vars_list_avail[var] = vars_list[var] else: tf.logging.warning('variable %s not found in checkpoint files %s.' % (var, ckpt_path)) vars_list = vars_list_avail From c4d723b58dfd15af45774b55b5fa2cae68c7a269 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Thu, 3 Jan 2019 11:14:49 +0800 Subject: [PATCH 039/173] remove unused variables --- datasets/pascalvoc_dataset.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/datasets/pascalvoc_dataset.py b/datasets/pascalvoc_dataset.py index b684fe1..5dfa7e2 100644 --- a/datasets/pascalvoc_dataset.py +++ b/datasets/pascalvoc_dataset.py @@ -34,8 +34,6 @@ tf.app.flags.DEFINE_integer('batch_size_eval', 1, 'batch size for evaluation') # Pascal VOC specifications -IMAGE_HEI = 224 -IMAGE_WID = 224 IMAGE_CHN = 3 def parse_example_proto(example_serialized): From 11a2839888a6129bc8e8b5e758085bda68cc54c4 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Thu, 3 Jan 2019 15:13:38 +0800 Subject: [PATCH 040/173] bugfix: unable to use TensorBoard at seven --- main.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/main.sh b/main.sh index b29d9fc..d476ea6 100755 --- a/main.sh +++ b/main.sh @@ -24,7 +24,6 @@ export PYTHONPATH=${PYTHONPATH}:`pwd` LOG_DIR=/opt/ml/log mkdir -p ${LOG_DIR} nohup tensorboard \ - --path_prefix=/seven-forward-port/${SEVEN_HTTP_FORWARD_PORT}/ \ --port=${SEVEN_HTTP_FORWARD_PORT} \ --host=127.0.0.1 \ --logdir=${LOG_DIR} \ From befcc30ee82dacea82e50919ee308f8fcd22b439 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Thu, 3 Jan 2019 15:14:14 +0800 Subject: [PATCH 041/173] rename tf.summary.scalar variables --- nets/vgg_at_pascalvoc.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nets/vgg_at_pascalvoc.py b/nets/vgg_at_pascalvoc.py index 675f31b..da82f13 100644 --- a/nets/vgg_at_pascalvoc.py +++ b/nets/vgg_at_pascalvoc.py @@ -335,14 +335,14 @@ def calc_loss_fn(objects, outputs, trainable_vars, anchor_info): # cross-entropy loss ce_loss = (params['negative_ratio'] + 1.) * \ tf.losses.sparse_softmax_cross_entropy(flatten_cls_targets, cls_pred) - tf.identity(ce_loss, name='cross_entropy_loss') - tf.summary.scalar('cross_entropy_loss', ce_loss) + tf.identity(ce_loss, name='ce_loss') + tf.summary.scalar('ce_loss', ce_loss) # localization loss loc_loss = tf.reduce_mean( tf.reduce_sum(modified_smooth_l1(loc_pred, flatten_loc_targets, sigma=1.), axis=-1)) - tf.identity(loc_loss, name='localization_loss') - tf.summary.scalar('localization_loss', loc_loss) + tf.identity(loc_loss, name='loc_loss') + tf.summary.scalar('loc_loss', loc_loss) # L2-regularization loss l2_loss_list = [] @@ -353,8 +353,8 @@ def calc_loss_fn(objects, outputs, trainable_vars, anchor_info): else: l2_loss_list.append(tf.nn.l2_loss(var) * 0.1) l2_loss = tf.add_n(l2_loss_list) - tf.identity(loc_loss, name='localization_loss') - tf.summary.scalar('localization_loss', loc_loss) + tf.identity(l2_loss, name='l2_loss') + tf.summary.scalar('l2_loss', l2_loss) # overall loss loss = ce_loss + loc_loss + params['weight_decay'] * l2_loss From 52a8b11cb469a1cb741d61e3008d22de31029dac Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Thu, 3 Jan 2019 16:26:03 +0800 Subject: [PATCH 042/173] set to False in preprocess_image() --- datasets/pascalvoc_dataset.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/datasets/pascalvoc_dataset.py b/datasets/pascalvoc_dataset.py index 5dfa7e2..69f9662 100644 --- a/datasets/pascalvoc_dataset.py +++ b/datasets/pascalvoc_dataset.py @@ -50,13 +50,15 @@ def parse_example_proto(example_serialized): feature_map = { 'image/encoded': tf.FixedLenFeature([], dtype=tf.string, default_value=''), 'image/format': tf.FixedLenFeature([], dtype=tf.string, default_value='jpeg'), + 'image/filename': tf.FixedLenFeature((), dtype=tf.string, default_value=''), 'image/height': tf.FixedLenFeature([1], dtype=tf.int64), 'image/width': tf.FixedLenFeature([1], dtype=tf.int64), + 'image/channels': tf.FixedLenFeature([1], dtype=tf.int64), + 'image/shape': tf.FixedLenFeature([3], dtype=tf.int64), 'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32), - 'image/object/bbox/label_text': tf.VarLenFeature(dtype=tf.string), 'image/object/bbox/label': tf.VarLenFeature(dtype=tf.int64), 'image/object/bbox/difficult': tf.VarLenFeature(dtype=tf.int64), 'image/object/bbox/truncated': tf.VarLenFeature(dtype=tf.int64), @@ -122,13 +124,13 @@ def parse_fn(example_serialized, is_train): bboxes_raw = tf.concat([ymins, xmins, ymaxs, xmaxs], axis=1) # N x 4 # obtain other annotation data - labels_raw = tf.cast(tf.sparse_tensor_to_dense(features['image/object/bbox/label']), tf.float32) - difficults = tf.cast(features['image/object/bbox/difficult'].values, tf.float32) - truncateds = tf.cast(features['image/object/bbox/truncated'].values, tf.float32) + labels_raw = tf.cast(features['image/object/bbox/label'].values, tf.int64) + difficults = tf.cast(features['image/object/bbox/difficult'].values, tf.int64) + truncateds = tf.cast(features['image/object/bbox/truncated'].values, tf.int64) - # filter difficult training samples + # filter out difficult objects if is_train: - # if all is difficult, then keep the first one + # if all is difficult, then keep the first one; otherwise, use all the non-difficult objects mask = tf.cond( tf.count_nonzero(difficults, dtype=tf.int32) < tf.shape(difficults)[0], lambda: difficults < tf.ones_like(difficults), @@ -141,10 +143,13 @@ def parse_fn(example_serialized, is_train): if is_train: out_shape = [FLAGS.image_size, FLAGS.image_size] image, labels, bboxes = preprocess_image( - image_raw, labels_raw, bboxes_raw, out_shape, is_training=True, data_format=data_format) + image_raw, labels_raw, bboxes_raw, out_shape, + is_training=True, data_format=data_format, output_rgb=False) else: out_shape = [FLAGS.image_size_eval, FLAGS.image_size_eval] - image = preprocess_image(image_raw, labels_raw, bboxes_raw, out_shape) + image = preprocess_image( + image_raw, labels_raw, bboxes_raw, out_shape, + is_training=False, data_format=data_format, output_rgb=False) labels, bboxes = labels_raw, bboxes_raw # pack all the annotations into one tensor From e2787f067414f2c62c450b759b33580afc787cf5 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Thu, 3 Jan 2019 16:52:04 +0800 Subject: [PATCH 043/173] bugfix: incompatible data type in tf.concat() --- datasets/pascalvoc_dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datasets/pascalvoc_dataset.py b/datasets/pascalvoc_dataset.py index 69f9662..130cd2c 100644 --- a/datasets/pascalvoc_dataset.py +++ b/datasets/pascalvoc_dataset.py @@ -81,14 +81,14 @@ def pack_annotations(bboxes, labels, difficults=None, truncateds=None): """ # pack & with a leading - labels = tf.expand_dims(labels, 1) + labels = tf.cast(tf.expand_dims(labels, 1), tf.float32) flags = tf.ones(tf.shape(labels)) objects = tf.concat([flags, bboxes, labels], axis=1) # pack & if supplied if difficults is not None and truncateds is not None: - difficults = tf.expand_dims(difficults, 1) - truncateds = tf.expand_dims(truncateds, 1) + difficults = tf.cast(tf.expand_dims(difficults, 1), tf.float32) + truncateds = tf.cast(tf.expand_dims(truncateds, 1), tf.float32) objects = tf.concat([objects, difficults, truncateds], axis=1) # pad to fixed number of bounding boxes From 97685819d6a11f09ac4bed4dffb70ee446464ba9 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Thu, 3 Jan 2019 17:02:14 +0800 Subject: [PATCH 044/173] remove unused code; calc cls. accuracy in --- nets/vgg_at_pascalvoc.py | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/nets/vgg_at_pascalvoc.py b/nets/vgg_at_pascalvoc.py index da82f13..8f146fe 100644 --- a/nets/vgg_at_pascalvoc.py +++ b/nets/vgg_at_pascalvoc.py @@ -28,15 +28,6 @@ FLAGS = tf.app.flags.FLAGS -### REQUIRED ### -#tf.app.flags.DEFINE_integer('resnet_size', 20, '# of layers in the ResNet model') -#tf.app.flags.DEFINE_float('nb_epochs_rat', 1.0, '# of training epochs\'s ratio') -#tf.app.flags.DEFINE_float('lrn_rate_init', 1e-1, 'initial learning rate') -#tf.app.flags.DEFINE_float('batch_size_norm', 128, 'normalization factor of batch size') -#tf.app.flags.DEFINE_float('momentum', 0.9, 'momentum coefficient') -#tf.app.flags.DEFINE_float('loss_w_dcy', 2e-4, 'weight decaying loss\'s coefficient') -### REQUIRED ### - # hardware related configuration tf.app.flags.DEFINE_integer('num_readers', 8, 'The number of parallel readers that read data from the dataset.') @@ -179,15 +170,12 @@ def modified_smooth_l1( with tf.name_scope('smooth_l1', [bbox_pred, bbox_targets]): sigma2 = sigma * sigma - inside_mul = tf.multiply(bbox_inside_weights, tf.subtract(bbox_pred, bbox_targets)) - smooth_l1_sign = tf.cast(tf.less(tf.abs(inside_mul), 1.0 / sigma2), tf.float32) smooth_l1_option1 = tf.multiply(tf.multiply(inside_mul, inside_mul), 0.5 * sigma2) smooth_l1_option2 = tf.subtract(tf.abs(inside_mul), 0.5 / sigma2) smooth_l1_result = tf.add(tf.multiply(smooth_l1_option1, smooth_l1_sign), tf.multiply(smooth_l1_option2, tf.abs(smooth_l1_sign - 1.0))) - outside_mul = tf.multiply(bbox_outside_weights, smooth_l1_result) return outside_mul @@ -326,11 +314,9 @@ def calc_loss_fn(objects, outputs, trainable_vars, anchor_info): 'probabilities': tf.reduce_max(tf.nn.softmax(cls_pred, name='softmax_tensor'), axis=-1), 'loc_predict': bboxes_pred, } - #cls_accuracy = tf.metrics.accuracy(flatten_cls_targets, predictions['classes']) - #tf.identity(cls_accuracy[1], name='cls_accuracy') - #tf.summary.scalar('cls_accuracy', cls_accuracy[1]) - #metrics = {'accuracy': cls_accuracy[1]} - metrics = {} + accuracy = tf.reduce_mean( + tf.cast(tf.equal(flatten_cls_targets, predictions['classes']), tf.float32)) + metrics = {'accuracy': accuracy} # cross-entropy loss ce_loss = (params['negative_ratio'] + 1.) * \ From 5fae99bbfd92eb81decc916397dc14583e43ba18 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Fri, 4 Jan 2019 16:54:40 +0800 Subject: [PATCH 045/173] SSD: support specifying the model scope in learners --- learners/full_precision/learner.py | 2 -- nets/vgg_at_pascalvoc.py | 18 ++++++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/learners/full_precision/learner.py b/learners/full_precision/learner.py index ea75476..c629fd7 100644 --- a/learners/full_precision/learner.py +++ b/learners/full_precision/learner.py @@ -48,8 +48,6 @@ def __init__(self, sm_writer, model_helper, model_scope=None, enbl_dst=None): self.model_scope = model_scope self.enbl_dst = enbl_dst if enbl_dst is not None else FLAGS.enbl_dst - self.model_scope = '' - # class-dependent initialization if self.enbl_dst: self.helper_dst = DistillationHelper(sm_writer, model_helper, self.mpi_comm) diff --git a/nets/vgg_at_pascalvoc.py b/nets/vgg_at_pascalvoc.py index 8f146fe..fdc1220 100644 --- a/nets/vgg_at_pascalvoc.py +++ b/nets/vgg_at_pascalvoc.py @@ -195,6 +195,9 @@ def forward_fn(inputs, is_train, data_format, anchor_info): all_num_anchors_depth = anchor_info['all_num_anchors_depth'] with tf.variable_scope(params['model_scope'], values=[inputs], reuse=tf.AUTO_REUSE): + # obtain the current model scope + model_scope = tf.get_default_graph().get_name_scope() + # obtain predictions for localization & classification backbone = ssd_net.VGG16Backbone(data_format) feature_layers = backbone.forward(inputs, training=is_train) @@ -216,7 +219,7 @@ def reshape_fn(preds, nb_dims): # pack all the output tensors together outputs = {'cls_pred': cls_pred, 'loc_pred': loc_pred} - return outputs + return outputs, model_scope def calc_loss_fn(objects, outputs, trainable_vars, anchor_info): """Calculate the loss function's value. @@ -362,6 +365,7 @@ def __init__(self): # setup hyper-parameters & anchor information setup_params() + self.model_scope = None self.anchor_info = None def build_dataset_train(self, enbl_trn_val_split=False): @@ -379,8 +383,9 @@ def forward_train(self, inputs, data_format='channels_last'): if self.anchor_info is None: self.anchor_info = setup_anchor_info() + outputs, self.model_scope = forward_fn(inputs, True, data_format, self.anchor_info) - return forward_fn(inputs, True, data_format, self.anchor_info) + return outputs def forward_eval(self, inputs, data_format='channels_last'): """Forward computation at evaluation.""" @@ -427,7 +432,7 @@ def warm_start(self, sess, vars_list): for var in vars_list: excluded = False for scope in excluded_scopes: - if var.name.startswith(scope): + if scope in var.name: excluded = True break if not excluded: @@ -437,11 +442,12 @@ def warm_start(self, sess, vars_list): # rename variables to be initialized if FLAGS.checkpoint_model_scope is not None: # rename the variable scope - if FLAGS.checkpoint_model_scope.strip() == '': - vars_list = {var.op.name.replace(FLAGS.model_scope + '/', ''): var for var in vars_list} + ckpt_model_scope = FLAGS.checkpoint_model_scope.strip() + if ckpt_model_scope == '': + vars_list = {var.op.name.replace(self.model_scope + '/', ''): var for var in vars_list} else: vars_list = {var.op.name.replace( - FLAGS.model_scope, FLAGS.checkpoint_model_scope.strip()): var for var in vars_list} + self.model_scope, ckpt_model_scope): var for var in vars_list} # re-map the variable's name name_remap = {'/kernel': '/weights', '/bias': '/biases'} From b4af0b9442e6d264a6a04809352a3224cd4e9518 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Fri, 4 Jan 2019 17:12:24 +0800 Subject: [PATCH 046/173] add FLAGS variables for evaluation --- nets/vgg_at_pascalvoc.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nets/vgg_at_pascalvoc.py b/nets/vgg_at_pascalvoc.py index fdc1220..006b7d4 100644 --- a/nets/vgg_at_pascalvoc.py +++ b/nets/vgg_at_pascalvoc.py @@ -62,6 +62,13 @@ tf.app.flags.DEFINE_float('match_threshold', 0.5, 'Matching threshold in the loss function.') tf.app.flags.DEFINE_float('neg_threshold', 0.5, 'Matching threshold for the negtive examples in the loss function.') +tf.app.flags.DEFINE_float('select_threshold', 0.01, + 'Class-specific confidence score threshold for selecting a box.') +tf.app.flags.DEFINE_float('min_size', 0.03, 'The min size of bboxes to keep.') +tf.app.flags.DEFINE_float('nms_threshold', 0.45, 'Matching threshold in NMS algorithm.') +tf.app.flags.DEFINE_integer('nms_topk', 200, 'Number of total object to keep after NMS.') +tf.app.flags.DEFINE_integer('keep_topk', 400, + 'Number of total object to keep for each image before nms.') # optimizer related configuration tf.app.flags.DEFINE_integer('tf_random_seed', 20190101, 'Random seed for TensorFlow initializers.') From c794f963beddf110fdadac66ff7615e1c65ce864 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Sun, 6 Jan 2019 16:43:15 +0800 Subject: [PATCH 047/173] use tf.map_fn to obtain target values & loc. predictions --- nets/vgg_at_pascalvoc.py | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/nets/vgg_at_pascalvoc.py b/nets/vgg_at_pascalvoc.py index 006b7d4..251e404 100644 --- a/nets/vgg_at_pascalvoc.py +++ b/nets/vgg_at_pascalvoc.py @@ -253,32 +253,26 @@ def calc_loss_fn(objects, outputs, trainable_vars, anchor_info): num_anchors_per_layer = anchor_info['num_anchors_per_layer'] all_num_anchors_depth = anchor_info['all_num_anchors_depth'] - # extract target localization & classification results from - # TODO use tf.map_fn - gt_locations_list = [] - gt_labels_list = [] - gt_scores_list = [] - b_flags, b_bboxes, b_labels = tf.split(objects, [1, 4, 1], -1) - b_flags = tf.squeeze(tf.cast(b_flags, dtype=tf.int64), axis=-1) - b_labels = tf.squeeze(tf.cast(b_labels, dtype=tf.int64), axis=-1) - for batch_index in range(batch_size): - index = tf.where(b_flags[batch_index] > 0) - labels = tf.gather_nd(b_labels[batch_index], index) - bboxes = tf.gather_nd(b_bboxes[batch_index], index) - gt_locations, gt_labels, gt_scores = encode_fn(labels, bboxes) - gt_locations_list += [tf.expand_dims(gt_locations, 0)] - gt_labels_list += [tf.expand_dims(gt_labels, 0)] - gt_scores_list += [tf.expand_dims(gt_scores, 0)] - loc_targets = tf.concat(gt_locations_list, axis=0) - cls_targets = tf.concat(gt_labels_list, axis=0) - match_scores = tf.concat(gt_scores_list, axis=0) + # extract target values & predicted localization results + def encode_objects_n_decode_loc_pred(objects_n_loc_pred): + objects = objects_n_loc_pred[0] + loc_pred = objects_n_loc_pred[1] + flags, bboxes, labels = tf.split(objects, [1, 4, 1], axis=-1) + flags = tf.squeeze(tf.cast(flags, dtype=tf.int64), axis=-1) + labels = tf.squeeze(tf.cast(labels, dtype=tf.int64), axis=-1) + index = tf.where(flags > 0) + loc, cls, scr = encode_fn(tf.gather_nd(labels, index), tf.gather_nd(bboxes, index)) + bbox = decode_fn(loc_pred) + return loc, cls, scr, bbox # post-forward operations with tf.control_dependencies([cls_pred, loc_pred]): with tf.name_scope('post_forward'): - bboxes_pred = tf.map_fn(lambda _preds: decode_fn(_preds), - tf.reshape(loc_pred, [batch_size, -1, 4]), - dtype=[tf.float32] * len(num_anchors_per_layer), back_prop=False) + loc_targets, cls_targets, match_scores, bboxes_pred = tf.map_fn( + encode_objects_n_decode_loc_pred, + (tf.reshape(objects, [batch_size, -1, 6]), tf.reshape(loc_pred, [batch_size, -1, 4])), + dtype=(tf.float32, tf.int64, tf.float32, [tf.float32] * len(num_anchors_per_layer)), + back_prop=False) bboxes_pred = [tf.reshape(preds, [-1, 4]) for preds in bboxes_pred] bboxes_pred = tf.concat(bboxes_pred, axis=0) From e2358435299eb22564547d82b7e0cfeb681cabbd Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 7 Jan 2019 11:13:38 +0800 Subject: [PATCH 048/173] use a separate function to initialize all anchors --- .../utility/anchor_manipulator.py | 98 +++++++++---------- 1 file changed, 48 insertions(+), 50 deletions(-) diff --git a/utils/external/ssd_tensorflow/utility/anchor_manipulator.py b/utils/external/ssd_tensorflow/utility/anchor_manipulator.py index a5d14c7..2e51fb0 100644 --- a/utils/external/ssd_tensorflow/utility/anchor_manipulator.py +++ b/utils/external/ssd_tensorflow/utility/anchor_manipulator.py @@ -124,103 +124,101 @@ def point2center(self, ymin, xmin, ymax, xmax): height, width = (ymax - ymin), (xmax - xmin) return ymin + height / 2., xmin + width / 2., height, width - def encode_all_anchors(self, labels, bboxes, all_anchors, all_num_anchors_depth, all_num_anchors_spatial, debug=False): - # y, x, h, w are all in range [0, 1] relative to the original image size - # shape info: - # y_on_image, x_on_image: layers_shapes[0] * layers_shapes[1] - # h_on_image, w_on_image: num_anchors - assert (len(all_num_anchors_depth)==len(all_num_anchors_spatial)) and (len(all_num_anchors_depth)==len(all_anchors)), 'inconsist num layers for anchors.' - with tf.name_scope('encode_all_anchors'): - num_layers = len(all_num_anchors_depth) + def init_all_anchors(self, all_anchors, all_num_anchors_depth, all_num_anchors_spatial): + assert len(all_num_anchors_depth) == len(all_num_anchors_spatial) \ + and len(all_num_anchors_depth) == len(all_anchors), 'inconsist num layers for anchors.' + + with tf.name_scope('init_all_anchors'): list_anchors_ymin = [] list_anchors_xmin = [] list_anchors_ymax = [] list_anchors_xmax = [] - tiled_allowed_borders = [] for ind, anchor in enumerate(all_anchors): - anchors_ymin_, anchors_xmin_, anchors_ymax_, anchors_xmax_ = self.center2point(anchor[0], anchor[1], anchor[2], anchor[3]) - + anchors_ymin_, anchors_xmin_, anchors_ymax_, anchors_xmax_ = \ + self.center2point(anchor[0], anchor[1], anchor[2], anchor[3]) list_anchors_ymin.append(tf.reshape(anchors_ymin_, [-1])) list_anchors_xmin.append(tf.reshape(anchors_xmin_, [-1])) list_anchors_ymax.append(tf.reshape(anchors_ymax_, [-1])) list_anchors_xmax.append(tf.reshape(anchors_xmax_, [-1])) - tiled_allowed_borders.extend([self._allowed_borders[ind]] * all_num_anchors_depth[ind] * all_num_anchors_spatial[ind]) - anchors_ymin = tf.concat(list_anchors_ymin, 0, name='concat_ymin') anchors_xmin = tf.concat(list_anchors_xmin, 0, name='concat_xmin') anchors_ymax = tf.concat(list_anchors_ymax, 0, name='concat_ymax') anchors_xmax = tf.concat(list_anchors_xmax, 0, name='concat_xmax') - if self._clip: anchors_ymin = tf.clip_by_value(anchors_ymin, 0., 1.) anchors_xmin = tf.clip_by_value(anchors_xmin, 0., 1.) anchors_ymax = tf.clip_by_value(anchors_ymax, 0., 1.) anchors_xmax = tf.clip_by_value(anchors_xmax, 0., 1.) - anchor_allowed_borders = tf.stack(tiled_allowed_borders, 0, name='concat_allowed_borders') - - inside_mask = tf.logical_and(tf.logical_and(anchors_ymin > -anchor_allowed_borders * 1., - anchors_xmin > -anchor_allowed_borders * 1.), - tf.logical_and(anchors_ymax < (1. + anchor_allowed_borders * 1.), - anchors_xmax < (1. + anchor_allowed_borders * 1.))) + anchor_cy, anchor_cx, anchor_h, anchor_w = \ + self.point2center(anchors_ymin, anchors_xmin, anchors_ymax, anchors_xmax) + self._all_anchors = (anchor_cy, anchor_cx, anchor_h, anchor_w) - anchors_point = tf.stack([anchors_ymin, anchors_xmin, anchors_ymax, anchors_xmax], axis=-1) + def encode_all_anchors(self, labels, bboxes, all_anchors, all_num_anchors_depth, all_num_anchors_spatial, debug=False): + assert self._all_anchors is not None, 'no anchors to encode.' - # save_anchors_op = tf.py_func(save_anchors, - # [bboxes, - # labels, - # anchors_point], - # tf.int64, stateful=True) + with tf.name_scope('encode_all_anchors'): + anchor_cy = self._all_anchors[0] + anchor_cx = self._all_anchors[1] + anchor_h = self._all_anchors[2] + anchor_w = self._all_anchors[3] + anchors_ymin, anchors_xmin, anchors_ymax, anchors_xmax = \ + self.center2point(anchor_cy, anchor_cx, anchor_h, anchor_w) + anchors_point = tf.stack( + [anchors_ymin, anchors_xmin, anchors_ymax, anchors_xmax], axis=-1) - # with tf.control_dependencies([save_anchors_op]): - overlap_matrix = iou_matrix(bboxes, anchors_point) * tf.cast(tf.expand_dims(inside_mask, 0), tf.float32) - matched_gt, gt_scores = do_dual_max_match(overlap_matrix, self._ignore_threshold, self._positive_threshold) - # get all positive matching positions + tiled_allowed_borders = [] + for ind, anchor in enumerate(all_anchors): + tiled_allowed_borders.extend([self._allowed_borders[ind]] + * all_num_anchors_depth[ind] * all_num_anchors_spatial[ind]) + anchor_allowed_borders = tf.stack( + tiled_allowed_borders, 0, name='concat_allowed_borders') + + inside_mask = tf.logical_and( + tf.logical_and(anchors_ymin > -anchor_allowed_borders * 1., + anchors_xmin > -anchor_allowed_borders * 1.), + tf.logical_and(anchors_ymax < (1. + anchor_allowed_borders * 1.), + anchors_xmax < (1. + anchor_allowed_borders * 1.))) + + overlap_matrix = iou_matrix(bboxes, anchors_point) \ + * tf.cast(tf.expand_dims(inside_mask, 0), tf.float32) + matched_gt, gt_scores = do_dual_max_match( + overlap_matrix, self._ignore_threshold, self._positive_threshold) matched_gt_mask = matched_gt > -1 matched_indices = tf.clip_by_value(matched_gt, 0, tf.int64.max) - # the labels here maybe chaos at those non-positive positions + gt_labels = tf.gather(labels, matched_indices) - # filter the invalid labels gt_labels = gt_labels * tf.cast(matched_gt_mask, tf.int64) - # set those ignored positions to -1 gt_labels = gt_labels + (-1 * tf.cast(matched_gt < -1, tf.int64)) - - gt_ymin, gt_xmin, gt_ymax, gt_xmax = tf.unstack(tf.gather(bboxes, matched_indices), 4, axis=-1) - - # transform to center / size. + gt_ymin, gt_xmin, gt_ymax, gt_xmax = \ + tf.unstack(tf.gather(bboxes, matched_indices), 4, axis=-1) gt_cy, gt_cx, gt_h, gt_w = self.point2center(gt_ymin, gt_xmin, gt_ymax, gt_xmax) - anchor_cy, anchor_cx, anchor_h, anchor_w = self.point2center(anchors_ymin, anchors_xmin, anchors_ymax, anchors_xmax) - # encode features. - # the prior_scaling (in fact is 5 and 10) is use for balance the regression loss of center and with(or height) gt_cy = (gt_cy - anchor_cy) / anchor_h / self._prior_scaling[0] gt_cx = (gt_cx - anchor_cx) / anchor_w / self._prior_scaling[1] gt_h = tf.log(gt_h / anchor_h) / self._prior_scaling[2] gt_w = tf.log(gt_w / anchor_w) / self._prior_scaling[3] - # now gt_localizations is our regression object, but also maybe chaos at those non-positive positions if debug: - gt_targets = tf.stack([anchors_ymin, anchors_xmin, anchors_ymax, anchors_xmax], axis=-1) + gt_targets = tf.stack( + [anchors_ymin, anchors_xmin, anchors_ymax, anchors_xmax], axis=-1) else: gt_targets = tf.stack([gt_cy, gt_cx, gt_h, gt_w], axis=-1) - # set all targets of non-positive positions to 0 gt_targets = tf.expand_dims(tf.cast(matched_gt_mask, tf.float32), -1) * gt_targets - self._all_anchors = (anchor_cy, anchor_cx, anchor_h, anchor_w) - return gt_targets, gt_labels, gt_scores - # return a list, of which each is: - # shape: [feature_h, feature_w, num_anchors, 4] - # order: ymin, xmin, ymax, xmax + return gt_targets, gt_labels, gt_scores + def decode_all_anchors(self, pred_location, num_anchors_per_layer): assert self._all_anchors is not None, 'no anchors to decode.' + with tf.name_scope('decode_all_anchors', [pred_location]): anchor_cy, anchor_cx, anchor_h, anchor_w = self._all_anchors - pred_h = tf.exp(pred_location[:, -2] * self._prior_scaling[2]) * anchor_h pred_w = tf.exp(pred_location[:, -1] * self._prior_scaling[3]) * anchor_w pred_cy = pred_location[:, 0] * self._prior_scaling[0] * anchor_h + anchor_cy pred_cx = pred_location[:, 1] * self._prior_scaling[1] * anchor_w + anchor_cx - return tf.split(tf.stack(self.center2point(pred_cy, pred_cx, pred_h, pred_w), axis=-1), num_anchors_per_layer, axis=0) + return tf.split(tf.stack(self.center2point( + pred_cy, pred_cx, pred_h, pred_w), axis=-1), num_anchors_per_layer, axis=0) def ext_decode_all_anchors(self, pred_location, all_anchors, all_num_anchors_depth, all_num_anchors_spatial): assert (len(all_num_anchors_depth)==len(all_num_anchors_spatial)) and (len(all_num_anchors_depth)==len(all_anchors)), 'inconsist num layers for anchors.' From 4e1e49770ddc80d5b3937c0f9fb1204b34ab7dcd Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 7 Jan 2019 11:14:58 +0800 Subject: [PATCH 049/173] enable building the evaluation graph --- learners/full_precision/learner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learners/full_precision/learner.py b/learners/full_precision/learner.py index c629fd7..debf90d 100644 --- a/learners/full_precision/learner.py +++ b/learners/full_precision/learner.py @@ -52,7 +52,7 @@ def __init__(self, sm_writer, model_helper, model_scope=None, enbl_dst=None): if self.enbl_dst: self.helper_dst = DistillationHelper(sm_writer, model_helper, self.mpi_comm) self.__build(is_train=True) - #self.__build(is_train=False) + self.__build(is_train=False) def train(self): """Train a model and periodically produce checkpoint files.""" From c138b17597a7e6299635bd4392dc50ef232f3fe6 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 7 Jan 2019 11:15:37 +0800 Subject: [PATCH 050/173] parse filename & shape from TFRecord files --- datasets/pascalvoc_dataset.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/datasets/pascalvoc_dataset.py b/datasets/pascalvoc_dataset.py index 130cd2c..c52f049 100644 --- a/datasets/pascalvoc_dataset.py +++ b/datasets/pascalvoc_dataset.py @@ -114,6 +114,8 @@ def parse_fn(example_serialized, is_train): # obtain the image data image_raw = tf.image.decode_jpeg(features['image/encoded'], channels=IMAGE_CHN) + filename = features['image/filename'] + shape = features['image/shape'] # obtain bounding boxes' coordinates # Note that we impose an ordering of (y, x) just to make life difficult. @@ -153,9 +155,10 @@ def parse_fn(example_serialized, is_train): labels, bboxes = labels_raw, bboxes_raw # pack all the annotations into one tensor + image_info = {'image': image, 'filename': filename, 'shape': shape} objects = pack_annotations(bboxes, labels) - return image, objects + return image_info, objects class PascalVocDataset(AbstractDataset): """Pascal VOC dataset.""" From 53ce63dfb38398c365aa1b7199106a25ddd4906a Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 7 Jan 2019 11:17:47 +0800 Subject: [PATCH 051/173] evaluation graph built passed --- nets/vgg_at_pascalvoc.py | 134 +++++++++++++++++++++++++++++++++++---- 1 file changed, 122 insertions(+), 12 deletions(-) diff --git a/nets/vgg_at_pascalvoc.py b/nets/vgg_at_pascalvoc.py index 251e404..a843b02 100644 --- a/nets/vgg_at_pascalvoc.py +++ b/nets/vgg_at_pascalvoc.py @@ -123,6 +123,11 @@ def setup_params(): 'negative_ratio': FLAGS.negative_ratio, 'match_threshold': FLAGS.match_threshold, 'neg_threshold': FLAGS.neg_threshold, + 'select_threshold': FLAGS.select_threshold, + 'min_size': FLAGS.min_size, + 'nms_threshold': FLAGS.nms_threshold, + 'nms_topk': FLAGS.nms_topk, + 'keep_topk': FLAGS.keep_topk, 'weight_decay': FLAGS.weight_decay, 'momentum': FLAGS.momentum, 'learning_rate': FLAGS.learning_rate, @@ -156,6 +161,8 @@ def setup_anchor_info(): # pack all the information into one dictionary anchor_info = { + 'init_fn': lambda: anchor_encoder.init_all_anchors( + all_anchors, all_num_anchors_depth, all_num_anchors_spatial), 'encode_fn': lambda glabels_, gbboxes_: anchor_encoder.encode_all_anchors( glabels_, gbboxes_, all_anchors, all_num_anchors_depth, all_num_anchors_spatial), 'decode_fn': lambda pred: anchor_encoder.decode_all_anchors(pred, num_anchors_per_layer), @@ -187,6 +194,76 @@ def modified_smooth_l1( return outside_mul +def select_bboxes(scores_pred, bboxes_pred, num_classes, select_threshold): + selected_bboxes = {} + selected_scores = {} + with tf.name_scope('select_bboxes', [scores_pred, bboxes_pred]): + for class_ind in range(1, num_classes): + class_scores = scores_pred[:, class_ind] + select_mask = class_scores > select_threshold + select_mask = tf.cast(select_mask, tf.float32) + selected_bboxes[class_ind] = tf.multiply(bboxes_pred, tf.expand_dims(select_mask, axis=-1)) + selected_scores[class_ind] = tf.multiply(class_scores, select_mask) + + return selected_bboxes, selected_scores + +def clip_bboxes(ymin, xmin, ymax, xmax, name): + with tf.name_scope(name, 'clip_bboxes', [ymin, xmin, ymax, xmax]): + ymin = tf.maximum(ymin, 0.) + xmin = tf.maximum(xmin, 0.) + ymax = tf.minimum(ymax, 1.) + xmax = tf.minimum(xmax, 1.) + ymin = tf.minimum(ymin, ymax) + xmin = tf.minimum(xmin, xmax) + + return ymin, xmin, ymax, xmax + +def filter_bboxes(scores_pred, ymin, xmin, ymax, xmax, min_size, name): + with tf.name_scope(name, 'filter_bboxes', [scores_pred, ymin, xmin, ymax, xmax]): + width = xmax - xmin + height = ymax - ymin + filter_mask = tf.logical_and(width > min_size, height > min_size) + filter_mask = tf.cast(filter_mask, tf.float32) + + return tf.multiply(ymin, filter_mask), tf.multiply(xmin, filter_mask), \ + tf.multiply(ymax, filter_mask), tf.multiply(xmax, filter_mask), \ + tf.multiply(scores_pred, filter_mask) + +def sort_bboxes(scores_pred, ymin, xmin, ymax, xmax, keep_topk, name): + with tf.name_scope(name, 'sort_bboxes', [scores_pred, ymin, xmin, ymax, xmax]): + cur_bboxes = tf.shape(scores_pred)[0] + scores, idxes = tf.nn.top_k(scores_pred, k=tf.minimum(keep_topk, cur_bboxes), sorted=True) + ymin, xmin, ymax, xmax = tf.gather(ymin, idxes), tf.gather(xmin, idxes), tf.gather(ymax, idxes), tf.gather(xmax, idxes) + paddings_scores = tf.expand_dims(tf.stack([0, tf.maximum(keep_topk-cur_bboxes, 0)], axis=0), axis=0) + + return tf.pad(ymin, paddings_scores, "CONSTANT"), tf.pad(xmin, paddings_scores, "CONSTANT"),\ + tf.pad(ymax, paddings_scores, "CONSTANT"), tf.pad(xmax, paddings_scores, "CONSTANT"),\ + tf.pad(scores, paddings_scores, "CONSTANT") + +def nms_bboxes(scores_pred, bboxes_pred, nms_topk, nms_threshold, name): + with tf.name_scope(name, 'nms_bboxes', [scores_pred, bboxes_pred]): + idxes = tf.image.non_max_suppression(bboxes_pred, scores_pred, nms_topk, nms_threshold) + + return tf.gather(scores_pred, idxes), tf.gather(bboxes_pred, idxes) + +def parse_by_class(cls_pred, bboxes_pred, num_classes, select_threshold, min_size, keep_topk, nms_topk, nms_threshold): + with tf.name_scope('select_bboxes', [cls_pred, bboxes_pred]): + scores_pred = tf.nn.softmax(cls_pred) + selected_bboxes, selected_scores = select_bboxes(scores_pred, bboxes_pred, num_classes, select_threshold) + for class_ind in range(1, num_classes): + ymin, xmin, ymax, xmax = tf.unstack(selected_bboxes[class_ind], 4, axis=-1) + #ymin, xmin, ymax, xmax = tf.split(selected_bboxes[class_ind], 4, axis=-1) + #ymin, xmin, ymax, xmax = tf.squeeze(ymin), tf.squeeze(xmin), tf.squeeze(ymax), tf.squeeze(xmax) + ymin, xmin, ymax, xmax = clip_bboxes(ymin, xmin, ymax, xmax, 'clip_bboxes_{}'.format(class_ind)) + ymin, xmin, ymax, xmax, selected_scores[class_ind] = filter_bboxes(selected_scores[class_ind], + ymin, xmin, ymax, xmax, min_size, 'filter_bboxes_{}'.format(class_ind)) + ymin, xmin, ymax, xmax, selected_scores[class_ind] = sort_bboxes(selected_scores[class_ind], + ymin, xmin, ymax, xmax, keep_topk, 'sort_bboxes_{}'.format(class_ind)) + selected_bboxes[class_ind] = tf.stack([ymin, xmin, ymax, xmax], axis=-1) + selected_scores[class_ind], selected_bboxes[class_ind] = nms_bboxes(selected_scores[class_ind], selected_bboxes[class_ind], nms_topk, nms_threshold, 'nms_bboxes_{}'.format(class_ind)) + + return selected_bboxes, selected_scores + def forward_fn(inputs, is_train, data_format, anchor_info): """Forward pass function. @@ -200,14 +277,26 @@ def forward_fn(inputs, is_train, data_format, anchor_info): * outputs: a dictionary of output tensors """ + tf.logging.info('building forward with is_train = {}'.format(is_train)) + + # extract anchor boundiing boxes' information + images = inputs['image'] + filenames = inputs['filename'] + shapes = inputs['shape'] + decode_fn = anchor_info['decode_fn'] all_num_anchors_depth = anchor_info['all_num_anchors_depth'] - with tf.variable_scope(params['model_scope'], values=[inputs], reuse=tf.AUTO_REUSE): + + # initialize anchor bounding boxes + anchor_info['init_fn']() + + # compute output tensors + with tf.variable_scope(params['model_scope'], values=[images], reuse=tf.AUTO_REUSE): # obtain the current model scope model_scope = tf.get_default_graph().get_name_scope() # obtain predictions for localization & classification backbone = ssd_net.VGG16Backbone(data_format) - feature_layers = backbone.forward(inputs, training=is_train) + feature_layers = backbone.forward(images, training=is_train) loc_pred, cls_pred = ssd_net.multibox_head( feature_layers, params['num_classes'], all_num_anchors_depth, data_format=data_format) if data_format == 'channels_first': @@ -216,15 +305,29 @@ def forward_fn(inputs, is_train, data_format, anchor_info): # flatten predictions def reshape_fn(preds, nb_dims): - preds = [tf.reshape(pred, [tf.shape(inputs)[0], -1, nb_dims]) for pred in preds] + preds = [tf.reshape(pred, [tf.shape(images)[0], -1, nb_dims]) for pred in preds] preds = tf.concat(preds, axis=1) preds = tf.reshape(preds, [-1, nb_dims]) return preds cls_pred = reshape_fn(cls_pred, params['num_classes']) loc_pred = reshape_fn(loc_pred, 4) + # obtain per-class predictions on bounding boxes and scores + if is_train: + predictions = None#tf.no_op() + else: + bboxes_pred = decode_fn(loc_pred) # evaluation batch size is 1 + bboxes_pred = tf.concat(bboxes_pred, axis=0) + selected_bboxes, selected_scores = parse_by_class( + cls_pred, bboxes_pred, params['num_classes'], params['select_threshold'], + params['min_size'], params['keep_topk'], params['nms_topk'], params['nms_threshold']) + predictions = {'filename': filenames, 'shape': shapes} + for idx_cls in range(1, params['num_classes']): + predictions['scores_{}'.format(idx_cls)] = tf.expand_dims(selected_scores[idx_cls], axis=0) + predictions['bboxes_{}'.format(idx_cls)] = tf.expand_dims(selected_bboxes[idx_cls], axis=0) + # pack all the output tensors together - outputs = {'cls_pred': cls_pred, 'loc_pred': loc_pred} + outputs = {'cls_pred': cls_pred, 'loc_pred': loc_pred, 'predictions': predictions} return outputs, model_scope @@ -268,17 +371,17 @@ def encode_objects_n_decode_loc_pred(objects_n_loc_pred): # post-forward operations with tf.control_dependencies([cls_pred, loc_pred]): with tf.name_scope('post_forward'): + # obtain target values & localization predictions loc_targets, cls_targets, match_scores, bboxes_pred = tf.map_fn( encode_objects_n_decode_loc_pred, (tf.reshape(objects, [batch_size, -1, 6]), tf.reshape(loc_pred, [batch_size, -1, 4])), dtype=(tf.float32, tf.int64, tf.float32, [tf.float32] * len(num_anchors_per_layer)), back_prop=False) - bboxes_pred = [tf.reshape(preds, [-1, 4]) for preds in bboxes_pred] - bboxes_pred = tf.concat(bboxes_pred, axis=0) - flatten_loc_targets = tf.reshape(loc_targets, [-1, 4]) flatten_cls_targets = tf.reshape(cls_targets, [-1]) flatten_match_scores = tf.reshape(match_scores, [-1]) + bboxes_pred = [tf.reshape(preds, [-1, 4]) for preds in bboxes_pred] + bboxes_pred = tf.concat(bboxes_pred, axis=0) # each positive examples has one label positive_mask = flatten_cls_targets > 0 @@ -366,8 +469,8 @@ def __init__(self): # setup hyper-parameters & anchor information setup_params() + self.anchor_info = None # track the most recently-used one self.model_scope = None - self.anchor_info = None def build_dataset_train(self, enbl_trn_val_split=False): """Build the data subset for training, usually with data augmentation.""" @@ -382,16 +485,20 @@ def build_dataset_eval(self): def forward_train(self, inputs, data_format='channels_last'): """Forward computation at training.""" - if self.anchor_info is None: - self.anchor_info = setup_anchor_info() - outputs, self.model_scope = forward_fn(inputs, True, data_format, self.anchor_info) + anchor_info = setup_anchor_info() + outputs, self.model_scope = forward_fn(inputs, True, data_format, anchor_info) + self.anchor_info = anchor_info return outputs def forward_eval(self, inputs, data_format='channels_last'): """Forward computation at evaluation.""" - return forward_fn(inputs, False, data_format, self.anchor_info) # FIXME cannot build + anchor_info = setup_anchor_info() + outputs, __ = forward_fn(inputs, False, data_format, anchor_info) + self.anchor_info = anchor_info + + return outputs def calc_loss(self, objects, outputs, trainable_vars): """Calculate loss (and some extra evaluation metrics).""" @@ -493,6 +600,9 @@ def warm_start(self, sess, vars_list): saver.build() saver.restore(sess, ckpt_path) + def dump_outputs(self, outputs): + pass + @property def model_name(self): """Model's name.""" From 88b75619850513a7a1a023ffaef68e0c8c1b6881 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 7 Jan 2019 11:25:54 +0800 Subject: [PATCH 052/173] remove ; use FLAGS instead --- nets/vgg_at_pascalvoc.py | 69 +++++++++++----------------------------- 1 file changed, 19 insertions(+), 50 deletions(-) diff --git a/nets/vgg_at_pascalvoc.py b/nets/vgg_at_pascalvoc.py index a843b02..64af7a5 100644 --- a/nets/vgg_at_pascalvoc.py +++ b/nets/vgg_at_pascalvoc.py @@ -101,46 +101,16 @@ 'When restoring a checkpoint would ignore missing variables.') tf.app.flags.DEFINE_boolean('multi_gpu', False, 'Whether there is GPU to use for training.') -params = {} - def parse_comma_list(args): """Convert a comma-separated list to a list of floating-point numbers.""" return [float(s.strip()) for s in args.split(',')] -def setup_params(): - """Setup hyper-parameters (from FLAGS to dict).""" - - global params - params = { - 'num_gpus': 1, - 'max_number_of_steps': FLAGS.max_number_of_steps, - 'train_image_size': FLAGS.train_image_size, - 'data_format': FLAGS.data_format, - 'batch_size': FLAGS.batch_size, - 'model_scope': FLAGS.model_scope, - 'num_classes': FLAGS.num_classes, - 'negative_ratio': FLAGS.negative_ratio, - 'match_threshold': FLAGS.match_threshold, - 'neg_threshold': FLAGS.neg_threshold, - 'select_threshold': FLAGS.select_threshold, - 'min_size': FLAGS.min_size, - 'nms_threshold': FLAGS.nms_threshold, - 'nms_topk': FLAGS.nms_topk, - 'keep_topk': FLAGS.keep_topk, - 'weight_decay': FLAGS.weight_decay, - 'momentum': FLAGS.momentum, - 'learning_rate': FLAGS.learning_rate, - 'end_learning_rate': FLAGS.end_learning_rate, - 'decay_boundaries': parse_comma_list(FLAGS.decay_boundaries), - 'lr_decay_factors': parse_comma_list(FLAGS.lr_decay_factors) - } - def setup_anchor_info(): """Setup the anchor bounding boxes' information.""" # get all anchor bounding boxes - out_shape = [params['train_image_size']] * 2 + out_shape = [FLAGS.train_image_size] * 2 anchor_creator = anchor_manipulator.AnchorCreator( out_shape, layers_shapes=[(38, 38), (19, 19), (10, 10), (5, 5), (3, 3), (1, 1)], @@ -156,8 +126,8 @@ def setup_anchor_info(): for ind in range(len(all_anchors)): num_anchors_per_layer.append(all_num_anchors_depth[ind] * all_num_anchors_spatial[ind]) anchor_encoder = anchor_manipulator.AnchorEncoder( - allowed_borders=[1.0] * 6, positive_threshold=params['match_threshold'], - ignore_threshold=params['neg_threshold'], prior_scaling=[0.1, 0.1, 0.2, 0.2]) + allowed_borders=[1.0] * 6, positive_threshold=FLAGS.match_threshold, + ignore_threshold=FLAGS.neg_threshold, prior_scaling=[0.1, 0.1, 0.2, 0.2]) # pack all the information into one dictionary anchor_info = { @@ -290,7 +260,7 @@ def forward_fn(inputs, is_train, data_format, anchor_info): anchor_info['init_fn']() # compute output tensors - with tf.variable_scope(params['model_scope'], values=[images], reuse=tf.AUTO_REUSE): + with tf.variable_scope(FLAGS.model_scope, values=[images], reuse=tf.AUTO_REUSE): # obtain the current model scope model_scope = tf.get_default_graph().get_name_scope() @@ -298,7 +268,7 @@ def forward_fn(inputs, is_train, data_format, anchor_info): backbone = ssd_net.VGG16Backbone(data_format) feature_layers = backbone.forward(images, training=is_train) loc_pred, cls_pred = ssd_net.multibox_head( - feature_layers, params['num_classes'], all_num_anchors_depth, data_format=data_format) + feature_layers, FLAGS.num_classes, all_num_anchors_depth, data_format=data_format) if data_format == 'channels_first': cls_pred = [tf.transpose(pred, [0, 2, 3, 1]) for pred in cls_pred] loc_pred = [tf.transpose(pred, [0, 2, 3, 1]) for pred in loc_pred] @@ -309,7 +279,7 @@ def reshape_fn(preds, nb_dims): preds = tf.concat(preds, axis=1) preds = tf.reshape(preds, [-1, nb_dims]) return preds - cls_pred = reshape_fn(cls_pred, params['num_classes']) + cls_pred = reshape_fn(cls_pred, FLAGS.num_classes) loc_pred = reshape_fn(loc_pred, 4) # obtain per-class predictions on bounding boxes and scores @@ -319,10 +289,10 @@ def reshape_fn(preds, nb_dims): bboxes_pred = decode_fn(loc_pred) # evaluation batch size is 1 bboxes_pred = tf.concat(bboxes_pred, axis=0) selected_bboxes, selected_scores = parse_by_class( - cls_pred, bboxes_pred, params['num_classes'], params['select_threshold'], - params['min_size'], params['keep_topk'], params['nms_topk'], params['nms_threshold']) + cls_pred, bboxes_pred, FLAGS.num_classes, FLAGS.select_threshold, + FLAGS.min_size, FLAGS.keep_topk, FLAGS.nms_topk, FLAGS.nms_threshold) predictions = {'filename': filenames, 'shape': shapes} - for idx_cls in range(1, params['num_classes']): + for idx_cls in range(1, FLAGS.num_classes): predictions['scores_{}'.format(idx_cls)] = tf.expand_dims(selected_scores[idx_cls], axis=0) predictions['bboxes_{}'.format(idx_cls)] = tf.expand_dims(selected_bboxes[idx_cls], axis=0) @@ -346,7 +316,7 @@ def calc_loss_fn(objects, outputs, trainable_vars, anchor_info): """ # extract output tensors - batch_size = params['batch_size'] + batch_size = FLAGS.batch_size cls_pred = outputs['cls_pred'] loc_pred = outputs['loc_pred'] @@ -390,12 +360,12 @@ def encode_objects_n_decode_loc_pred(objects_n_loc_pred): batch_negtive_mask = tf.equal(cls_targets, 0) batch_n_negtives = tf.count_nonzero(batch_negtive_mask, -1) batch_n_neg_select = tf.cast( - params['negative_ratio'] * tf.cast(batch_n_positives, tf.float32), tf.int32) + FLAGS.negative_ratio * tf.cast(batch_n_positives, tf.float32), tf.int32) batch_n_neg_select = tf.minimum(batch_n_neg_select, tf.cast(batch_n_negtives, tf.int32)) # hard negative mining for classification predictions_for_bg = tf.nn.softmax( - tf.reshape(cls_pred, [batch_size, -1, params['num_classes']]))[:, :, 0] + tf.reshape(cls_pred, [batch_size, -1, FLAGS.num_classes]))[:, :, 0] prob_for_negtives = tf.where(batch_negtive_mask, 0. - predictions_for_bg, 0. - tf.ones_like(predictions_for_bg)) @@ -412,7 +382,7 @@ def encode_objects_n_decode_loc_pred(objects_n_loc_pred): cls_pred = tf.boolean_mask(cls_pred, final_mask) loc_pred = tf.boolean_mask(loc_pred, tf.stop_gradient(positive_mask)) flatten_cls_targets = tf.boolean_mask( - tf.clip_by_value(flatten_cls_targets, 0, params['num_classes']), final_mask) + tf.clip_by_value(flatten_cls_targets, 0, FLAGS.num_classes), final_mask) flatten_loc_targets = tf.stop_gradient(tf.boolean_mask(flatten_loc_targets, positive_mask)) # final predictions & classification accuracy @@ -426,7 +396,7 @@ def encode_objects_n_decode_loc_pred(objects_n_loc_pred): metrics = {'accuracy': accuracy} # cross-entropy loss - ce_loss = (params['negative_ratio'] + 1.) * \ + ce_loss = (FLAGS.negative_ratio + 1.) * \ tf.losses.sparse_softmax_cross_entropy(flatten_cls_targets, cls_pred) tf.identity(ce_loss, name='ce_loss') tf.summary.scalar('ce_loss', ce_loss) @@ -450,7 +420,7 @@ def encode_objects_n_decode_loc_pred(objects_n_loc_pred): tf.summary.scalar('l2_loss', l2_loss) # overall loss - loss = ce_loss + loc_loss + params['weight_decay'] * l2_loss + loss = ce_loss + loc_loss + FLAGS.weight_decay * l2_loss return loss, metrics @@ -468,7 +438,6 @@ def __init__(self): self.dataset_eval = PascalVocDataset(is_train=False) # setup hyper-parameters & anchor information - setup_params() self.anchor_info = None # track the most recently-used one self.model_scope = None @@ -508,11 +477,11 @@ def calc_loss(self, objects, outputs, trainable_vars): def setup_lrn_rate(self, global_step): """Setup the learning rate (and number of training iterations).""" - bnds = [int(x) for x in params['decay_boundaries']] - vals = [params['learning_rate'] * x for x in params['lr_decay_factors']] + bnds = [int(x) for x in parse_comma_list(FLAGS.decay_boundaries)] + vals = [FLAGS.learning_rate * x for x in parse_comma_list(FLAGS.lr_decay_factors)] lrn_rate = tf.train.piecewise_constant(global_step, bnds, vals) - lrn_rate = tf.maximum(lrn_rate, tf.constant(params['end_learning_rate'], dtype=lrn_rate.dtype)) - nb_iters = params['max_number_of_steps'] + lrn_rate = tf.maximum(lrn_rate, tf.constant(FLAGS.end_learning_rate, dtype=lrn_rate.dtype)) + nb_iters = FLAGS.max_number_of_steps return lrn_rate, nb_iters From bd57b5861a16bd82571548865f21ae9b5ad88986 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 7 Jan 2019 11:33:19 +0800 Subject: [PATCH 053/173] code style revised --- nets/vgg_at_pascalvoc.py | 43 +++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/nets/vgg_at_pascalvoc.py b/nets/vgg_at_pascalvoc.py index 64af7a5..727a3f4 100644 --- a/nets/vgg_at_pascalvoc.py +++ b/nets/vgg_at_pascalvoc.py @@ -113,12 +113,12 @@ def setup_anchor_info(): out_shape = [FLAGS.train_image_size] * 2 anchor_creator = anchor_manipulator.AnchorCreator( out_shape, - layers_shapes=[(38, 38), (19, 19), (10, 10), (5, 5), (3, 3), (1, 1)], - anchor_scales=[(0.1,), (0.2,), (0.375,), (0.55,), (0.725,), (0.9,)], - extra_anchor_scales=[(0.1414,), (0.2739,), (0.4541,), (0.6315,), (0.8078,), (0.9836,)], - anchor_ratios=[(1., 2., .5), (1., 2., 3., .5, 0.3333), (1., 2., 3., .5, 0.3333), - (1., 2., 3., .5, 0.3333), (1., 2., .5), (1., 2., .5)], - layer_steps=[8, 16, 32, 64, 100, 300]) + layers_shapes = [(38, 38), (19, 19), (10, 10), (5, 5), (3, 3), (1, 1)], + anchor_scales = [(0.1,), (0.2,), (0.375,), (0.55,), (0.725,), (0.9,)], + extra_anchor_scales = [(0.1414,), (0.2739,), (0.4541,), (0.6315,), (0.8078,), (0.9836,)], + anchor_ratios = [(1., 2., .5), (1., 2., 3., .5, 0.3333), (1., 2., 3., .5, 0.3333), + (1., 2., 3., .5, 0.3333), (1., 2., .5), (1., 2., .5)], + layer_steps = [8, 16, 32, 64, 100, 300]) all_anchors, all_num_anchors_depth, all_num_anchors_spatial = anchor_creator.get_all_anchors() # construct the anchor bounding boxes' encoder & decoder @@ -203,8 +203,10 @@ def sort_bboxes(scores_pred, ymin, xmin, ymax, xmax, keep_topk, name): with tf.name_scope(name, 'sort_bboxes', [scores_pred, ymin, xmin, ymax, xmax]): cur_bboxes = tf.shape(scores_pred)[0] scores, idxes = tf.nn.top_k(scores_pred, k=tf.minimum(keep_topk, cur_bboxes), sorted=True) - ymin, xmin, ymax, xmax = tf.gather(ymin, idxes), tf.gather(xmin, idxes), tf.gather(ymax, idxes), tf.gather(xmax, idxes) - paddings_scores = tf.expand_dims(tf.stack([0, tf.maximum(keep_topk-cur_bboxes, 0)], axis=0), axis=0) + ymin, xmin, ymax, xmax = \ + tf.gather(ymin, idxes), tf.gather(xmin, idxes), tf.gather(ymax, idxes), tf.gather(xmax, idxes) + paddings_scores = \ + tf.expand_dims(tf.stack([0, tf.maximum(keep_topk-cur_bboxes, 0)], axis=0), axis=0) return tf.pad(ymin, paddings_scores, "CONSTANT"), tf.pad(xmin, paddings_scores, "CONSTANT"),\ tf.pad(ymax, paddings_scores, "CONSTANT"), tf.pad(xmax, paddings_scores, "CONSTANT"),\ @@ -216,21 +218,26 @@ def nms_bboxes(scores_pred, bboxes_pred, nms_topk, nms_threshold, name): return tf.gather(scores_pred, idxes), tf.gather(bboxes_pred, idxes) -def parse_by_class(cls_pred, bboxes_pred, num_classes, select_threshold, min_size, keep_topk, nms_topk, nms_threshold): +def parse_by_class(cls_pred, bboxes_pred, num_classes, + select_threshold, min_size, keep_topk, nms_topk, nms_threshold): with tf.name_scope('select_bboxes', [cls_pred, bboxes_pred]): scores_pred = tf.nn.softmax(cls_pred) - selected_bboxes, selected_scores = select_bboxes(scores_pred, bboxes_pred, num_classes, select_threshold) + selected_bboxes, selected_scores = \ + select_bboxes(scores_pred, bboxes_pred, num_classes, select_threshold) for class_ind in range(1, num_classes): ymin, xmin, ymax, xmax = tf.unstack(selected_bboxes[class_ind], 4, axis=-1) - #ymin, xmin, ymax, xmax = tf.split(selected_bboxes[class_ind], 4, axis=-1) - #ymin, xmin, ymax, xmax = tf.squeeze(ymin), tf.squeeze(xmin), tf.squeeze(ymax), tf.squeeze(xmax) - ymin, xmin, ymax, xmax = clip_bboxes(ymin, xmin, ymax, xmax, 'clip_bboxes_{}'.format(class_ind)) - ymin, xmin, ymax, xmax, selected_scores[class_ind] = filter_bboxes(selected_scores[class_ind], - ymin, xmin, ymax, xmax, min_size, 'filter_bboxes_{}'.format(class_ind)) - ymin, xmin, ymax, xmax, selected_scores[class_ind] = sort_bboxes(selected_scores[class_ind], - ymin, xmin, ymax, xmax, keep_topk, 'sort_bboxes_{}'.format(class_ind)) + ymin, xmin, ymax, xmax = \ + clip_bboxes(ymin, xmin, ymax, xmax, 'clip_bboxes_{}'.format(class_ind)) + ymin, xmin, ymax, xmax, selected_scores[class_ind] = filter_bboxes( + selected_scores[class_ind], ymin, xmin, ymax, xmax, + min_size, 'filter_bboxes_{}'.format(class_ind)) + ymin, xmin, ymax, xmax, selected_scores[class_ind] = sort_bboxes( + selected_scores[class_ind], ymin, xmin, ymax, xmax, + keep_topk, 'sort_bboxes_{}'.format(class_ind)) selected_bboxes[class_ind] = tf.stack([ymin, xmin, ymax, xmax], axis=-1) - selected_scores[class_ind], selected_bboxes[class_ind] = nms_bboxes(selected_scores[class_ind], selected_bboxes[class_ind], nms_topk, nms_threshold, 'nms_bboxes_{}'.format(class_ind)) + selected_scores[class_ind], selected_bboxes[class_ind] = nms_bboxes( + selected_scores[class_ind], selected_bboxes[class_ind], + nms_topk, nms_threshold, 'nms_bboxes_{}'.format(class_ind)) return selected_bboxes, selected_scores From 58a656b4f4a29363a8a672638d7fe10c7f9d591b Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 7 Jan 2019 11:42:11 +0800 Subject: [PATCH 054/173] change the default batch size for training --- datasets/pascalvoc_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasets/pascalvoc_dataset.py b/datasets/pascalvoc_dataset.py index c52f049..05a6d3b 100644 --- a/datasets/pascalvoc_dataset.py +++ b/datasets/pascalvoc_dataset.py @@ -30,7 +30,7 @@ tf.app.flags.DEFINE_integer('nb_smpls_train', 22136, '# of samples for training') tf.app.flags.DEFINE_integer('nb_smpls_val', 500, '# of samples for validation') tf.app.flags.DEFINE_integer('nb_smpls_eval', 4952, '# of samples for evaluation') -tf.app.flags.DEFINE_integer('batch_size', 1, 'batch size per GPU for training') +tf.app.flags.DEFINE_integer('batch_size', 32, 'batch size per GPU for training') tf.app.flags.DEFINE_integer('batch_size_eval', 1, 'batch size for evaluation') # Pascal VOC specifications From a7d6921a4a5b62f0df161ea12e90d1878c25d7ed Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 7 Jan 2019 16:11:27 +0800 Subject: [PATCH 055/173] modify do_python_eval() so that ModelHelper can call it --- utils/external/ssd_tensorflow/voc_eval.py | 27 +++++++---------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/utils/external/ssd_tensorflow/voc_eval.py b/utils/external/ssd_tensorflow/voc_eval.py index ea9c76a..c439a43 100644 --- a/utils/external/ssd_tensorflow/voc_eval.py +++ b/utils/external/ssd_tensorflow/voc_eval.py @@ -26,23 +26,7 @@ else: import xml.etree.ElementTree as ET -from dataset import dataset_common - -''' -VOC2007TEST - Annotations - ... - ImageSets -''' -#dataset_path = '/media/rs/7A0EE8880EE83EAF/Detections/PASCAL/VOC/VOC2007TEST' -dataset_path = '/data1/jonathan/datasets/Pascal.VOC.2007.2012/VOC2007TEST' -# change above path according to your system settings -pred_path = './logs/predict' -pred_file = 'results_{}.txt' # from 1-num_classes -output_path = './logs/predict/eval_output' -cache_path = './logs/predict/eval_cache' -anno_files = 'Annotations/{}.xml' -all_images_file = 'ImageSets/Main/test.txt' +from utils.external.ssd_tensorflow.dataset import dataset_common def parse_rec(filename): """ Parse a PASCAL VOC xml file """ @@ -63,7 +47,12 @@ def parse_rec(filename): return objects -def do_python_eval(use_07=True): +def do_python_eval(dataset_path, pred_path, use_07=True): + output_path = os.path.join(pred_path, 'eval_output') + cache_path = os.path.join(pred_path, 'eval_cache') + anno_files = os.path.join(dataset_path, 'Annotations/{}.xml') + all_images_file = os.path.join(dataset_path, 'ImageSets/Main/test.txt') + aps = [] # The PASCAL VOC metric changed in 2010 use_07_metric = use_07 @@ -74,7 +63,7 @@ def do_python_eval(use_07=True): if 'none' in cls_name: continue cls_id = cls_pair[0] - filename = os.path.join(pred_path, pred_file.format(cls_id)) + filename = os.path.join(pred_path, 'results_%d.txt' % cls_id) rec, prec, ap = voc_eval(filename, os.path.join(dataset_path, anno_files), os.path.join(dataset_path, all_images_file), cls_name, cache_path, ovthresh=0.5, use_07_metric=use_07_metric) From d95bc58129eca783a1004c012589c74f2e187431 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 7 Jan 2019 16:26:02 +0800 Subject: [PATCH 056/173] ignore SSD model's outputs dumping directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4829b8b..c26a1f4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ start_multi.sh ratio.list path.conf nvidia-smi-dump +ssd_outputs From fee0a0427754573108be7b54f1e92ab6329dbd9f Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 7 Jan 2019 16:35:12 +0800 Subject: [PATCH 057/173] define as an attribute of PascalVocDataset --- datasets/pascalvoc_dataset.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datasets/pascalvoc_dataset.py b/datasets/pascalvoc_dataset.py index 05a6d3b..9e6adca 100644 --- a/datasets/pascalvoc_dataset.py +++ b/datasets/pascalvoc_dataset.py @@ -176,20 +176,20 @@ def __init__(self, is_train): # choose local files or HDFS files w.r.t. FLAGS.data_disk if FLAGS.data_disk == 'local': assert FLAGS.data_dir_local is not None, ' must not be None' - data_dir = FLAGS.data_dir_local + self.data_dir = FLAGS.data_dir_local elif FLAGS.data_disk == 'hdfs': assert FLAGS.data_hdfs_host is not None and FLAGS.data_dir_hdfs is not None, \ 'both and must not be None' - data_dir = FLAGS.data_hdfs_host + FLAGS.data_dir_hdfs + self.data_dir = FLAGS.data_hdfs_host + FLAGS.data_dir_hdfs else: raise ValueError('unrecognized data disk: ' + FLAGS.data_disk) # configure file patterns & function handlers if is_train: - self.file_pattern = os.path.join(data_dir, '*train*') + self.file_pattern = os.path.join(self.data_dir, '*train*') self.batch_size = FLAGS.batch_size else: - self.file_pattern = os.path.join(data_dir, '*val*') + self.file_pattern = os.path.join(self.data_dir, '*val*') self.batch_size = FLAGS.batch_size_eval self.dataset_fn = tf.data.TFRecordDataset self.parse_fn = lambda x: parse_fn(x, is_train=is_train) From 553103f252b4d26988d779bc0692668ea886e27f Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 7 Jan 2019 16:37:12 +0800 Subject: [PATCH 058/173] use for non-TF evaluation operations --- learners/abstract_learner.py | 1 + nets/abstract_model_helper.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/learners/abstract_learner.py b/learners/abstract_learner.py index fe9f883..86ebaff 100644 --- a/learners/abstract_learner.py +++ b/learners/abstract_learner.py @@ -79,6 +79,7 @@ def __init__(self, sm_writer, model_helper): self.calc_loss = model_helper.calc_loss self.setup_lrn_rate = model_helper.setup_lrn_rate self.warm_start = model_helper.warm_start + self.dump_n_eval = model_helper.dump_n_eval self.model_name = model_helper.model_name self.dataset_name = model_helper.dataset_name diff --git a/nets/abstract_model_helper.py b/nets/abstract_model_helper.py index ab9bda8..e22cb34 100644 --- a/nets/abstract_model_helper.py +++ b/nets/abstract_model_helper.py @@ -124,6 +124,15 @@ def warm_start(self, sess, vars_list): """ pass + def dump_n_eval(self, outputs, action): + """Dump the model's outputs to files and evaluate. + + Args: + * outputs: outputs from the network's forward pass + * action: 'init' | 'dump' | 'eval' + """ + pass + @property @abstractmethod def model_name(self): From 40d198a7bc7be896727e528a421e25d5226abb3e Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 7 Jan 2019 16:41:59 +0800 Subject: [PATCH 059/173] enable evaluation within FullPrecLearner --- learners/full_precision/learner.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/learners/full_precision/learner.py b/learners/full_precision/learner.py index debf90d..b698317 100644 --- a/learners/full_precision/learner.py +++ b/learners/full_precision/learner.py @@ -79,23 +79,26 @@ def train(self): # save & evaluate the model at certain steps if self.is_primary_worker('global') and (idx_iter + 1) % FLAGS.save_step == 0: self.__save_model(is_train=True) - #self.evaluate() + self.evaluate() # save the final model if self.is_primary_worker('global'): self.__save_model(is_train=True) - #self.__restore_model(is_train=False) - #self.__save_model(is_train=False) - #self.evaluate() + self.__restore_model(is_train=False) + self.__save_model(is_train=False) + self.evaluate() def evaluate(self): """Restore a model from the latest checkpoint files and then evaluate it.""" self.__restore_model(is_train=False) - nb_iters = int(np.ceil(float(FLAGS.nb_smpls_eval) / FLAGS.batch_size)) + nb_iters = int(np.ceil(float(FLAGS.nb_smpls_eval) / FLAGS.batch_size_eval)) eval_rslts = np.zeros((nb_iters, len(self.eval_op))) + self.dump_n_eval(outputs=None, action='init') for idx_iter in range(nb_iters): - eval_rslts[idx_iter] = self.sess_eval.run(self.eval_op) + eval_rslts[idx_iter], outputs = self.sess_eval.run([self.eval_op, self.outputs_eval]) + self.dump_n_eval(outputs=outputs, action='dump') + self.dump_n_eval(outputs=None, action='eval') for idx, name in enumerate(self.eval_op_names): tf.logging.info('%s = %.4e' % (name, np.mean(eval_rslts[:, idx]))) @@ -162,6 +165,7 @@ def __build(self, is_train): # pylint: disable=too-many-locals self.sess_eval = sess self.eval_op = [loss] + list(metrics.values()) self.eval_op_names = ['loss'] + list(metrics.keys()) + self.outputs_eval = logits self.saver_eval = tf.train.Saver(self.vars) def __save_model(self, is_train): From 60ef68cd1b66983a6f710a00766f528cb2be1cf6 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 7 Jan 2019 16:42:32 +0800 Subject: [PATCH 060/173] evaluate mAP via dump_n_eval() --- nets/vgg_at_pascalvoc.py | 50 ++++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/nets/vgg_at_pascalvoc.py b/nets/vgg_at_pascalvoc.py index 727a3f4..07e967c 100644 --- a/nets/vgg_at_pascalvoc.py +++ b/nets/vgg_at_pascalvoc.py @@ -16,6 +16,9 @@ # ============================================================================== """Model helper for creating a VGG model for the Pascal VOC dataset.""" +import os +import shutil +import numpy as np import tensorflow as tf from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw @@ -25,6 +28,7 @@ from utils.external.ssd_tensorflow.net import ssd_net from utils.external.ssd_tensorflow.utility import anchor_manipulator from utils.external.ssd_tensorflow.utility import scaffolds +from utils.external.ssd_tensorflow.voc_eval import do_python_eval FLAGS = tf.app.flags.FLAGS @@ -101,6 +105,9 @@ 'When restoring a checkpoint would ignore missing variables.') tf.app.flags.DEFINE_boolean('multi_gpu', False, 'Whether there is GPU to use for training.') +# evaluation related configuration +tf.app.flags.DEFINE_string('outputs_dump_dir', './ssd_outputs/', 'outputs\'s dumping directory') + def parse_comma_list(args): """Convert a comma-separated list to a list of floating-point numbers.""" @@ -300,15 +307,15 @@ def reshape_fn(preds, nb_dims): FLAGS.min_size, FLAGS.keep_topk, FLAGS.nms_topk, FLAGS.nms_threshold) predictions = {'filename': filenames, 'shape': shapes} for idx_cls in range(1, FLAGS.num_classes): - predictions['scores_{}'.format(idx_cls)] = tf.expand_dims(selected_scores[idx_cls], axis=0) - predictions['bboxes_{}'.format(idx_cls)] = tf.expand_dims(selected_bboxes[idx_cls], axis=0) + predictions['scores_%d' % idx_cls] = tf.expand_dims(selected_scores[idx_cls], axis=0) + predictions['bboxes_%d' % idx_cls] = tf.expand_dims(selected_bboxes[idx_cls], axis=0) # pack all the output tensors together outputs = {'cls_pred': cls_pred, 'loc_pred': loc_pred, 'predictions': predictions} return outputs, model_scope -def calc_loss_fn(objects, outputs, trainable_vars, anchor_info): +def calc_loss_fn(objects, outputs, trainable_vars, anchor_info, batch_size): """Calculate the loss function's value. Args: @@ -316,6 +323,7 @@ def calc_loss_fn(objects, outputs, trainable_vars, anchor_info): * outputs: a dictionary of output tensors * trainable_vars: list of trainable variables * anchor_info: anchor bounding boxes' information + * batch_size: batch size Returns: * loss: loss function's value @@ -323,7 +331,7 @@ def calc_loss_fn(objects, outputs, trainable_vars, anchor_info): """ # extract output tensors - batch_size = FLAGS.batch_size + #batch_size = FLAGS.batch_size cls_pred = outputs['cls_pred'] loc_pred = outputs['loc_pred'] @@ -446,6 +454,7 @@ def __init__(self): # setup hyper-parameters & anchor information self.anchor_info = None # track the most recently-used one + self.batch_size = None # track the most recently-used one self.model_scope = None def build_dataset_train(self, enbl_trn_val_split=False): @@ -464,6 +473,7 @@ def forward_train(self, inputs, data_format='channels_last'): anchor_info = setup_anchor_info() outputs, self.model_scope = forward_fn(inputs, True, data_format, anchor_info) self.anchor_info = anchor_info + self.batch_size = tf.shape(inputs['image'])[0] return outputs @@ -473,13 +483,14 @@ def forward_eval(self, inputs, data_format='channels_last'): anchor_info = setup_anchor_info() outputs, __ = forward_fn(inputs, False, data_format, anchor_info) self.anchor_info = anchor_info + self.batch_size = tf.shape(inputs['image'])[0] return outputs def calc_loss(self, objects, outputs, trainable_vars): """Calculate loss (and some extra evaluation metrics).""" - return calc_loss_fn(objects, outputs, trainable_vars, self.anchor_info) + return calc_loss_fn(objects, outputs, trainable_vars, self.anchor_info, self.batch_size) def setup_lrn_rate(self, global_step): """Setup the learning rate (and number of training iterations).""" @@ -576,8 +587,33 @@ def warm_start(self, sess, vars_list): saver.build() saver.restore(sess, ckpt_path) - def dump_outputs(self, outputs): - pass + def dump_n_eval(self, outputs, action): + """Dump the model's outputs to files and evaluate.""" + + if action == 'init': + if os.path.exists(FLAGS.outputs_dump_dir): + shutil.rmtree(FLAGS.outputs_dump_dir) + os.mkdir(FLAGS.outputs_dump_dir) + elif action == 'dump': + filename = outputs['predictions']['filename'][0].decode('utf8')[:-4] + shape = outputs['predictions']['shape'][0] + for idx_cls in range(1, FLAGS.num_classes): + with open(os.path.join(FLAGS.outputs_dump_dir, 'results_%d.txt' % idx_cls), 'a') as o_file: + scores = outputs['predictions']['scores_%d' % idx_cls][0] + bboxes = outputs['predictions']['bboxes_%d' % idx_cls][0] + bboxes[:, 0] = (bboxes[:, 0] * shape[0]).astype(np.int32, copy=False) + 1 + bboxes[:, 1] = (bboxes[:, 1] * shape[1]).astype(np.int32, copy=False) + 1 + bboxes[:, 2] = (bboxes[:, 2] * shape[0]).astype(np.int32, copy=False) + 1 + bboxes[:, 3] = (bboxes[:, 3] * shape[1]).astype(np.int32, copy=False) + 1 + for idx_bbox in range(bboxes.shape[0]): + bbox = bboxes[idx_bbox][:] + if bbox[2] > bbox[0] and bbox[3] > bbox[1]: + o_file.write('%s %.3f %.1f %.1f %.1f %.1f\n' + % (filename, scores[idx_bbox], bbox[1], bbox[0], bbox[3], bbox[2])) + elif action == 'eval': + do_python_eval(os.path.join(self.dataset_eval.data_dir, 'test'), FLAGS.outputs_dump_dir) + else: + raise ValueError('unrecognized action in dump_n_eval(): ' + action) @property def model_name(self): From 3f2b6894d03cd32d6cd2c750a67c2ae3d5943ee7 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 7 Jan 2019 18:16:05 +0800 Subject: [PATCH 061/173] only perform non-TF evaluation on the primary worker --- nets/vgg_at_pascalvoc.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nets/vgg_at_pascalvoc.py b/nets/vgg_at_pascalvoc.py index 07e967c..3f36634 100644 --- a/nets/vgg_at_pascalvoc.py +++ b/nets/vgg_at_pascalvoc.py @@ -24,6 +24,7 @@ from nets.abstract_model_helper import AbstractModelHelper from datasets.pascalvoc_dataset import PascalVocDataset +from utils.misc_utils import is_primary_worker from utils.external.ssd_tensorflow.net import ssd_net from utils.external.ssd_tensorflow.utility import anchor_manipulator @@ -590,6 +591,9 @@ def warm_start(self, sess, vars_list): def dump_n_eval(self, outputs, action): """Dump the model's outputs to files and evaluate.""" + if not is_primary_worker('global'): + return + if action == 'init': if os.path.exists(FLAGS.outputs_dump_dir): shutil.rmtree(FLAGS.outputs_dump_dir) From 5aaffaaa7262ecfd32e8a8b2a2583594ba81ed7c Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 7 Jan 2019 20:14:52 +0800 Subject: [PATCH 062/173] stop passing list of variables to warm_start() --- learners/full_precision/learner.py | 3 +-- nets/abstract_model_helper.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/learners/full_precision/learner.py b/learners/full_precision/learner.py index b698317..4184a62 100644 --- a/learners/full_precision/learner.py +++ b/learners/full_precision/learner.py @@ -59,7 +59,7 @@ def train(self): # initialization self.sess_train.run(self.init_op) - self.warm_start(self.sess_train, self.trainable_vars_cache) + self.warm_start(self.sess_train) if FLAGS.enbl_multi_gpu: self.sess_train.run(self.bcast_op) @@ -160,7 +160,6 @@ def __build(self, is_train): # pylint: disable=too-many-locals if FLAGS.enbl_multi_gpu: self.bcast_op = mgw.broadcast_global_variables(0) self.saver_train = tf.train.Saver(self.vars) - self.trainable_vars_cache = self.trainable_vars else: self.sess_eval = sess self.eval_op = [loss] + list(metrics.values()) diff --git a/nets/abstract_model_helper.py b/nets/abstract_model_helper.py index e22cb34..b66d245 100644 --- a/nets/abstract_model_helper.py +++ b/nets/abstract_model_helper.py @@ -115,12 +115,11 @@ def setup_lrn_rate(self, global_step): """ pass - def warm_start(self, sess, vars_list): + def warm_start(self, sess): """Initialize the model for warm-start. Args: * sess: TensorFlow session - * vars_list: list of variables to be updated """ pass From 440beeb37869e51b06cea9d8fb3508390fe28e64 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 7 Jan 2019 20:21:55 +0800 Subject: [PATCH 063/173] obtain list of trainable vars in SSD's ModelHelper --- nets/vgg_at_pascalvoc.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/nets/vgg_at_pascalvoc.py b/nets/vgg_at_pascalvoc.py index 3f36634..bfc4880 100644 --- a/nets/vgg_at_pascalvoc.py +++ b/nets/vgg_at_pascalvoc.py @@ -475,6 +475,8 @@ def forward_train(self, inputs, data_format='channels_last'): outputs, self.model_scope = forward_fn(inputs, True, data_format, anchor_info) self.anchor_info = anchor_info self.batch_size = tf.shape(inputs['image'])[0] + self.trainable_vars = tf.get_collection( + tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.model_scope) return outputs @@ -504,7 +506,7 @@ def setup_lrn_rate(self, global_step): return lrn_rate, nb_iters - def warm_start(self, sess, vars_list): + def warm_start(self, sess): """Initialize the model for warm-start. Description: @@ -524,16 +526,15 @@ def warm_start(self, sess, vars_list): tf.logging.info('excluded scopes: {}'.format(excluded_scopes)) # obtain a list of variables to be initialized - vars_list_scope = [] - for var in vars_list: + vars_list = [] + for var in self.trainable_vars: excluded = False for scope in excluded_scopes: if scope in var.name: excluded = True break if not excluded: - vars_list_scope.append(var) - vars_list = vars_list_scope + vars_list.append(var) # rename variables to be initialized if FLAGS.checkpoint_model_scope is not None: From 06204e91df01278dc0cee5318a5556d8a85c34c5 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Tue, 8 Jan 2019 14:07:09 +0800 Subject: [PATCH 064/173] remove unused FLAGS variables --- datasets/pascalvoc_dataset.py | 1 + nets/vgg_at_pascalvoc.py | 76 ++++++++++------------------------- 2 files changed, 23 insertions(+), 54 deletions(-) diff --git a/datasets/pascalvoc_dataset.py b/datasets/pascalvoc_dataset.py index 9e6adca..e1b59c3 100644 --- a/datasets/pascalvoc_dataset.py +++ b/datasets/pascalvoc_dataset.py @@ -27,6 +27,7 @@ tf.app.flags.DEFINE_integer('image_size', 300, 'output image size') tf.app.flags.DEFINE_integer('image_size_eval', 300, 'output image size for evaluation') tf.app.flags.DEFINE_integer('nb_bboxs_max', 100, 'maximal # of bounding boxes per image') +tf.app.flags.DEFINE_integer('nb_classes', 21, '# of classes') tf.app.flags.DEFINE_integer('nb_smpls_train', 22136, '# of samples for training') tf.app.flags.DEFINE_integer('nb_smpls_val', 500, '# of samples for validation') tf.app.flags.DEFINE_integer('nb_smpls_eval', 4952, '# of samples for evaluation') diff --git a/nets/vgg_at_pascalvoc.py b/nets/vgg_at_pascalvoc.py index bfc4880..d191468 100644 --- a/nets/vgg_at_pascalvoc.py +++ b/nets/vgg_at_pascalvoc.py @@ -33,36 +33,11 @@ FLAGS = tf.app.flags.FLAGS -# hardware related configuration -tf.app.flags.DEFINE_integer('num_readers', 8, - 'The number of parallel readers that read data from the dataset.') -tf.app.flags.DEFINE_integer('num_preprocessing_threads', 24, - 'The number of threads used to create the batches.') -tf.app.flags.DEFINE_integer('num_cpu_threads', 0, 'The number of cpu cores used to train.') -tf.app.flags.DEFINE_float('gpu_memory_fraction', 1.0, 'GPU memory fraction to use.') - # scaffold related configuration -tf.app.flags.DEFINE_string('data_dir', './tfrecords', - 'The directory where the dataset input data is stored.') -tf.app.flags.DEFINE_integer('num_classes', 21, 'Number of classes to use in the dataset.') tf.app.flags.DEFINE_string('model_dir', './logs/', 'The directory where the model will be stored.') -tf.app.flags.DEFINE_integer('log_every_n_steps', 10, 'The frequency with which logs are printed.') -tf.app.flags.DEFINE_integer('save_summary_steps', 500, - 'The frequency with which summaries are saved, in seconds.') -tf.app.flags.DEFINE_integer('save_checkpoints_secs', 7200, - 'The frequency with which the model is saved, in seconds.') # model related configuration -tf.app.flags.DEFINE_integer('train_image_size', 300, - 'The size of the input image for the model to use.') -tf.app.flags.DEFINE_integer('train_epochs', None, 'The number of epochs to use for training.') -tf.app.flags.DEFINE_integer('max_number_of_steps', 120000, - 'The max number of steps to use for training.') -tf.app.flags.DEFINE_string('data_format', 'channels_last', # 'channels_first' or 'channels_last' - 'A flag to override the data format used in the model. channels_first ' - 'provides a performance boost on GPU but is not always compatible ' - 'with CPU. If left unspecified, the data format will be chosen ' - 'automatically based on whether TensorFlow was built for CPU or GPU.') +tf.app.flags.DEFINE_integer('nb_iters_train', 120000, 'The number of training iterations.') tf.app.flags.DEFINE_float('negative_ratio', 3.0, 'Negative ratio in the loss function.') tf.app.flags.DEFINE_float('match_threshold', 0.5, 'Matching threshold in the loss function.') tf.app.flags.DEFINE_float('neg_threshold', 0.5, @@ -76,20 +51,14 @@ 'Number of total object to keep for each image before nms.') # optimizer related configuration -tf.app.flags.DEFINE_integer('tf_random_seed', 20190101, 'Random seed for TensorFlow initializers.') -tf.app.flags.DEFINE_float('weight_decay', 5e-4, 'The weight decay on the model weights.') -tf.app.flags.DEFINE_float('momentum', 0.9, - 'The momentum for the MomentumOptimizer and RMSPropOptimizer.') -tf.app.flags.DEFINE_float('learning_rate', 1e-3, 'Initial learning rate.') -tf.app.flags.DEFINE_float('end_learning_rate', 1e-6, - 'The minimal end learning rate used by a polynomial decay learning rate.') - -# for learning rate piecewise_constant decay -tf.app.flags.DEFINE_string('decay_boundaries', '500, 80000, 100000', - 'Learning rate decay boundaries by global_step (comma-separated list).') -tf.app.flags.DEFINE_string('lr_decay_factors', '0.1, 1, 0.1, 0.01', - 'The values of learning_rate decay factor for each segment between ' - 'boundaries (comma-separated list).') +tf.app.flags.DEFINE_float('lrn_rate_init', 1e-3, 'The initial learning rate.') +tf.app.flags.DEFINE_float('lrn_rate_min', 1e-6, 'The minimal learning rate') +tf.app.flags.DEFINE_string('lrn_rate_dcy_bnds', '500, 80000, 100000', + 'Learning rate decay boundaries.') +tf.app.flags.DEFINE_string('lrn_rate_dcy_rates', '0.1, 1, 0.1, 0.01', + 'Learning rate decay rates for each segment between boundaries') +tf.app.flags.DEFINE_float('momentum', 0.9, 'momentum coefficient') +tf.app.flags.DEFINE_float('loss_w_dcy', 5e-4, 'weight decaying loss\'s coefficient') # checkpoint related configuration tf.app.flags.DEFINE_string('checkpoint_path', './model/', @@ -104,7 +73,6 @@ 'from a checkpoint.') tf.app.flags.DEFINE_boolean('ignore_missing_vars', True, 'When restoring a checkpoint would ignore missing variables.') -tf.app.flags.DEFINE_boolean('multi_gpu', False, 'Whether there is GPU to use for training.') # evaluation related configuration tf.app.flags.DEFINE_string('outputs_dump_dir', './ssd_outputs/', 'outputs\'s dumping directory') @@ -118,7 +86,7 @@ def setup_anchor_info(): """Setup the anchor bounding boxes' information.""" # get all anchor bounding boxes - out_shape = [FLAGS.train_image_size] * 2 + out_shape = [FLAGS.image_size] * 2 anchor_creator = anchor_manipulator.AnchorCreator( out_shape, layers_shapes = [(38, 38), (19, 19), (10, 10), (5, 5), (3, 3), (1, 1)], @@ -283,7 +251,7 @@ def forward_fn(inputs, is_train, data_format, anchor_info): backbone = ssd_net.VGG16Backbone(data_format) feature_layers = backbone.forward(images, training=is_train) loc_pred, cls_pred = ssd_net.multibox_head( - feature_layers, FLAGS.num_classes, all_num_anchors_depth, data_format=data_format) + feature_layers, FLAGS.nb_classes, all_num_anchors_depth, data_format=data_format) if data_format == 'channels_first': cls_pred = [tf.transpose(pred, [0, 2, 3, 1]) for pred in cls_pred] loc_pred = [tf.transpose(pred, [0, 2, 3, 1]) for pred in loc_pred] @@ -294,7 +262,7 @@ def reshape_fn(preds, nb_dims): preds = tf.concat(preds, axis=1) preds = tf.reshape(preds, [-1, nb_dims]) return preds - cls_pred = reshape_fn(cls_pred, FLAGS.num_classes) + cls_pred = reshape_fn(cls_pred, FLAGS.nb_classes) loc_pred = reshape_fn(loc_pred, 4) # obtain per-class predictions on bounding boxes and scores @@ -304,10 +272,10 @@ def reshape_fn(preds, nb_dims): bboxes_pred = decode_fn(loc_pred) # evaluation batch size is 1 bboxes_pred = tf.concat(bboxes_pred, axis=0) selected_bboxes, selected_scores = parse_by_class( - cls_pred, bboxes_pred, FLAGS.num_classes, FLAGS.select_threshold, + cls_pred, bboxes_pred, FLAGS.nb_classes, FLAGS.select_threshold, FLAGS.min_size, FLAGS.keep_topk, FLAGS.nms_topk, FLAGS.nms_threshold) predictions = {'filename': filenames, 'shape': shapes} - for idx_cls in range(1, FLAGS.num_classes): + for idx_cls in range(1, FLAGS.nb_classes): predictions['scores_%d' % idx_cls] = tf.expand_dims(selected_scores[idx_cls], axis=0) predictions['bboxes_%d' % idx_cls] = tf.expand_dims(selected_bboxes[idx_cls], axis=0) @@ -381,7 +349,7 @@ def encode_objects_n_decode_loc_pred(objects_n_loc_pred): # hard negative mining for classification predictions_for_bg = tf.nn.softmax( - tf.reshape(cls_pred, [batch_size, -1, FLAGS.num_classes]))[:, :, 0] + tf.reshape(cls_pred, [batch_size, -1, FLAGS.nb_classes]))[:, :, 0] prob_for_negtives = tf.where(batch_negtive_mask, 0. - predictions_for_bg, 0. - tf.ones_like(predictions_for_bg)) @@ -398,7 +366,7 @@ def encode_objects_n_decode_loc_pred(objects_n_loc_pred): cls_pred = tf.boolean_mask(cls_pred, final_mask) loc_pred = tf.boolean_mask(loc_pred, tf.stop_gradient(positive_mask)) flatten_cls_targets = tf.boolean_mask( - tf.clip_by_value(flatten_cls_targets, 0, FLAGS.num_classes), final_mask) + tf.clip_by_value(flatten_cls_targets, 0, FLAGS.nb_classes), final_mask) flatten_loc_targets = tf.stop_gradient(tf.boolean_mask(flatten_loc_targets, positive_mask)) # final predictions & classification accuracy @@ -436,7 +404,7 @@ def encode_objects_n_decode_loc_pred(objects_n_loc_pred): tf.summary.scalar('l2_loss', l2_loss) # overall loss - loss = ce_loss + loc_loss + FLAGS.weight_decay * l2_loss + loss = ce_loss + loc_loss + FLAGS.loss_w_dcy * l2_loss return loss, metrics @@ -498,11 +466,11 @@ def calc_loss(self, objects, outputs, trainable_vars): def setup_lrn_rate(self, global_step): """Setup the learning rate (and number of training iterations).""" - bnds = [int(x) for x in parse_comma_list(FLAGS.decay_boundaries)] - vals = [FLAGS.learning_rate * x for x in parse_comma_list(FLAGS.lr_decay_factors)] + bnds = [int(x) for x in parse_comma_list(FLAGS.lrn_rate_dcy_bnds)] + vals = [FLAGS.lrn_rate_init * x for x in parse_comma_list(FLAGS.lrn_rate_dcy_rates)] lrn_rate = tf.train.piecewise_constant(global_step, bnds, vals) - lrn_rate = tf.maximum(lrn_rate, tf.constant(FLAGS.end_learning_rate, dtype=lrn_rate.dtype)) - nb_iters = FLAGS.max_number_of_steps + lrn_rate = tf.maximum(lrn_rate, tf.constant(FLAGS.lrn_rate_min, dtype=lrn_rate.dtype)) + nb_iters = FLAGS.nb_iters_train return lrn_rate, nb_iters @@ -602,7 +570,7 @@ def dump_n_eval(self, outputs, action): elif action == 'dump': filename = outputs['predictions']['filename'][0].decode('utf8')[:-4] shape = outputs['predictions']['shape'][0] - for idx_cls in range(1, FLAGS.num_classes): + for idx_cls in range(1, FLAGS.nb_classes): with open(os.path.join(FLAGS.outputs_dump_dir, 'results_%d.txt' % idx_cls), 'a') as o_file: scores = outputs['predictions']['scores_%d' % idx_cls][0] bboxes = outputs['predictions']['bboxes_%d' % idx_cls][0] From 743e17af374c2420ddccaf96702323fed0906225 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Tue, 8 Jan 2019 14:14:50 +0800 Subject: [PATCH 065/173] let learner restore weights if *.ckpt files exist --- nets/vgg_at_pascalvoc.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/nets/vgg_at_pascalvoc.py b/nets/vgg_at_pascalvoc.py index d191468..1881785 100644 --- a/nets/vgg_at_pascalvoc.py +++ b/nets/vgg_at_pascalvoc.py @@ -33,9 +33,6 @@ FLAGS = tf.app.flags.FLAGS -# scaffold related configuration -tf.app.flags.DEFINE_string('model_dir', './logs/', 'The directory where the model will be stored.') - # model related configuration tf.app.flags.DEFINE_integer('nb_iters_train', 120000, 'The number of training iterations.') tf.app.flags.DEFINE_float('negative_ratio', 3.0, 'Negative ratio in the loss function.') @@ -479,14 +476,10 @@ def warm_start(self, sess): Description: * We use a pre-trained ImageNet classification model to initialize the backbone part of the SSD - model for feature extraction. If the SSD model's checkpoint files already exist, then skip. + model for feature extraction. If the SSD model's checkpoint files already exist, then the + learner should restore model weights by itself. """ - # early return if checkpoint files already exist - if tf.train.latest_checkpoint(FLAGS.model_dir): - tf.logging.info('checkpoint files already exist in ' + FLAGS.model_dir) - return - # obtain a list of scopes to be excluded from initialization excluded_scopes = [] if FLAGS.checkpoint_exclude_scopes: From 0ce8554e1a34390c656b70dcd7d731add22ebfac Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Tue, 8 Jan 2019 14:27:06 +0800 Subject: [PATCH 066/173] set shape for images (as TF-tensor or as dict) --- learners/uniform_quantization_tf/learner.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/learners/uniform_quantization_tf/learner.py b/learners/uniform_quantization_tf/learner.py index 2a608d3..2b874b8 100644 --- a/learners/uniform_quantization_tf/learner.py +++ b/learners/uniform_quantization_tf/learner.py @@ -148,7 +148,11 @@ def __build_train(self): # pylint: disable=too-many-locals,too-many-statements with tf.variable_scope(self.data_scope): iterator = self.build_dataset_train() images, labels = iterator.get_next() - images.set_shape((FLAGS.batch_size, images.shape[1], images.shape[2], images.shape[3])) + if not isinstance(images, dict): + images.set_shape((FLAGS.batch_size, images.shape[1], images.shape[2], images.shape[3])) + else: + shape = images['image'].shape + images['image'].set_shape((FLAGS.batch_size, shape[1], shape[2], shape[3])) # model definition - uniform quantized model - part 1 with tf.variable_scope(self.model_scope_quan): From 4054fff16ce91e6fb392ffe163729d7968f44a9b Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Tue, 8 Jan 2019 15:47:07 +0800 Subject: [PATCH 067/173] revise warm-start-related FLAGS variables --- nets/vgg_at_pascalvoc.py | 59 ++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/nets/vgg_at_pascalvoc.py b/nets/vgg_at_pascalvoc.py index 1881785..9a106ae 100644 --- a/nets/vgg_at_pascalvoc.py +++ b/nets/vgg_at_pascalvoc.py @@ -58,16 +58,15 @@ tf.app.flags.DEFINE_float('loss_w_dcy', 5e-4, 'weight decaying loss\'s coefficient') # checkpoint related configuration -tf.app.flags.DEFINE_string('checkpoint_path', './model/', - 'The path to a checkpoint from which to fine-tune.') -tf.app.flags.DEFINE_string('checkpoint_model_scope', 'vgg_16', +tf.app.flags.DEFINE_string('backbone_ckpt_dir', './backbone_models/', + 'The backbone model\'s (e.g. VGG-16) checkpoint directory') +tf.app.flags.DEFINE_string('backbone_model_scope', 'vgg_16', 'Model scope in the checkpoint. None if the same as the trained model.') tf.app.flags.DEFINE_string('model_scope', 'ssd300', 'Model scope name used to replace the name_scope in checkpoint.') -tf.app.flags.DEFINE_string('checkpoint_exclude_scopes', +tf.app.flags.DEFINE_string('warm_start_excl_scopes', 'ssd300/multibox_head, ssd300/additional_layers, ssd300/conv4_3_scale', - 'Comma-separated list of scopes of variables to exclude when restoring ' - 'from a checkpoint.') + 'List of scopes to be excluded when restoring from a backbone model') tf.app.flags.DEFINE_boolean('ignore_missing_vars', True, 'When restoring a checkpoint would ignore missing variables.') @@ -481,42 +480,41 @@ def warm_start(self, sess): """ # obtain a list of scopes to be excluded from initialization - excluded_scopes = [] - if FLAGS.checkpoint_exclude_scopes: - excluded_scopes = [scope.strip() for scope in FLAGS.checkpoint_exclude_scopes.split(',')] - tf.logging.info('excluded scopes: {}'.format(excluded_scopes)) + excl_scopes = [] + if FLAGS.warm_start_excl_scopes: + excl_scopes = [scope.strip() for scope in FLAGS.warm_start_excl_scopes.split(',')] + tf.logging.info('excluded scopes: {}'.format(excl_scopes)) # obtain a list of variables to be initialized vars_list = [] for var in self.trainable_vars: excluded = False - for scope in excluded_scopes: + for scope in excl_scopes: if scope in var.name: excluded = True break if not excluded: vars_list.append(var) - # rename variables to be initialized - if FLAGS.checkpoint_model_scope is not None: - # rename the variable scope - ckpt_model_scope = FLAGS.checkpoint_model_scope.strip() - if ckpt_model_scope == '': + # rename the variables' scope + if FLAGS.backbone_model_scope is not None: + backbone_model_scope = FLAGS.backbone_model_scope.strip() + if backbone_model_scope == '': vars_list = {var.op.name.replace(self.model_scope + '/', ''): var for var in vars_list} else: vars_list = {var.op.name.replace( - self.model_scope, ckpt_model_scope): var for var in vars_list} - - # re-map the variable's name - name_remap = {'/kernel': '/weights', '/bias': '/biases'} - vars_list_remap = {} - for var_name, var in vars_list.items(): - for name_old, name_new in name_remap.items(): - if name_old in var_name: - var_name = var_name.replace(name_old, name_new) - break - vars_list_remap[var_name] = var - vars_list = vars_list_remap + self.model_scope, backbone_model_scope): var for var in vars_list} + + # re-map the variables' names + name_remap = {'/kernel': '/weights', '/bias': '/biases'} + vars_list_remap = {} + for var_name, var in vars_list.items(): + for name_old, name_new in name_remap.items(): + if name_old in var_name: + var_name = var_name.replace(name_old, name_new) + break + vars_list_remap[var_name] = var + vars_list = vars_list_remap # display all the variables to be initialized for var_name, var in vars_list.items(): @@ -525,10 +523,7 @@ def warm_start(self, sess): raise ValueError('variables to be restored cannot be empty') # obtain the checkpoint files' path - if tf.gfile.IsDirectory(FLAGS.checkpoint_path): - ckpt_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path) - else: - ckpt_path = FLAGS.checkpoint_path + ckpt_path = tf.train.latest_checkpoint(FLAGS.backbone_ckpt_dir) tf.logging.info('restoring model weights from ' + ckpt_path) # remove missing variables from the list From 8dbb7105f8f43573e6beabbc53a503b8e5d042d6 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Tue, 8 Jan 2019 19:47:46 +0800 Subject: [PATCH 068/173] reset the global step before quant-aware fine-tuning --- learners/uniform_quantization_tf/learner.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/learners/uniform_quantization_tf/learner.py b/learners/uniform_quantization_tf/learner.py index 2b874b8..6f18e58 100644 --- a/learners/uniform_quantization_tf/learner.py +++ b/learners/uniform_quantization_tf/learner.py @@ -96,10 +96,14 @@ def train(self): # initialization self.sess_train.run([self.init_op, self.init_opt_op]) + self.sess_train.run(self.global_step.initializer) # reset the global step if FLAGS.enbl_multi_gpu: self.sess_train.run(self.bcast_op) # train the model through iterations and periodically save & evaluate the model + for __ in range(10): + summary, log_rslt = self.sess_train.run([self.summary_op, self.log_op]) + self.__monitor_progress(summary, log_rslt, -1, 1.0) time_prev = timer() for idx_iter in range(self.nb_iters_train): # train the model From 997726026cd62bc6b9477a1c8bdf478998f59d54 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Tue, 8 Jan 2019 19:48:33 +0800 Subject: [PATCH 069/173] gradually increase the cls. loss's coefficient --- nets/vgg_at_pascalvoc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nets/vgg_at_pascalvoc.py b/nets/vgg_at_pascalvoc.py index 9a106ae..fc39937 100644 --- a/nets/vgg_at_pascalvoc.py +++ b/nets/vgg_at_pascalvoc.py @@ -400,7 +400,9 @@ def encode_objects_n_decode_loc_pred(objects_n_loc_pred): tf.summary.scalar('l2_loss', l2_loss) # overall loss - loss = ce_loss + loc_loss + FLAGS.loss_w_dcy * l2_loss + global_step = tf.train.get_or_create_global_step() + loss_w_cls = tf.clip_by_value(tf.cast(global_step, tf.float32) / tf.constant(1000.0), 0.0, 1.0) + loss = loss_w_cls * ce_loss + loc_loss + FLAGS.loss_w_dcy * l2_loss return loss, metrics From cda966ea57758ea3d43df351cef8f8d8fc7ab368 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Tue, 8 Jan 2019 20:10:25 +0800 Subject: [PATCH 070/173] print 10 iterations w/o training before fine-tuning --- learners/uniform_quantization_tf/learner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/learners/uniform_quantization_tf/learner.py b/learners/uniform_quantization_tf/learner.py index 6f18e58..74d86e1 100644 --- a/learners/uniform_quantization_tf/learner.py +++ b/learners/uniform_quantization_tf/learner.py @@ -101,9 +101,9 @@ def train(self): self.sess_train.run(self.bcast_op) # train the model through iterations and periodically save & evaluate the model - for __ in range(10): + for idx_iter in range(-10, 0): summary, log_rslt = self.sess_train.run([self.summary_op, self.log_op]) - self.__monitor_progress(summary, log_rslt, -1, 1.0) + self.__monitor_progress(summary, log_rslt, idx_iter, 1.0) time_prev = timer() for idx_iter in range(self.nb_iters_train): # train the model From 7f67fe3bd33b5dda64cf90526333e1bf2e171242 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Tue, 8 Jan 2019 20:35:55 +0800 Subject: [PATCH 071/173] use FLAGS.nb_iters_cls_wmup to control cls. loss's warm-up stage --- nets/vgg_at_pascalvoc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nets/vgg_at_pascalvoc.py b/nets/vgg_at_pascalvoc.py index fc39937..7194c98 100644 --- a/nets/vgg_at_pascalvoc.py +++ b/nets/vgg_at_pascalvoc.py @@ -55,6 +55,8 @@ tf.app.flags.DEFINE_string('lrn_rate_dcy_rates', '0.1, 1, 0.1, 0.01', 'Learning rate decay rates for each segment between boundaries') tf.app.flags.DEFINE_float('momentum', 0.9, 'momentum coefficient') +tf.app.flags.DEFINE_integer('nb_iters_cls_wmup', 10000, + 'The number of iterations for warming-up the classification loss') tf.app.flags.DEFINE_float('loss_w_dcy', 5e-4, 'weight decaying loss\'s coefficient') # checkpoint related configuration @@ -401,7 +403,8 @@ def encode_objects_n_decode_loc_pred(objects_n_loc_pred): # overall loss global_step = tf.train.get_or_create_global_step() - loss_w_cls = tf.clip_by_value(tf.cast(global_step, tf.float32) / tf.constant(1000.0), 0.0, 1.0) + loss_w_cls = tf.minimum( + tf.cast(global_step, tf.float32) / tf.constant(FLAGS.nb_iters_cls_wmup, dtype=tf.float32), 1.0) loss = loss_w_cls * ce_loss + loc_loss + FLAGS.loss_w_dcy * l2_loss return loss, metrics From 8bdfd066505bc2b0f51a0ad3b8c7a1f0b859bd06 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Wed, 9 Jan 2019 08:26:19 +0800 Subject: [PATCH 072/173] create in both train & eval graphs --- learners/uniform_quantization_tf/learner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/learners/uniform_quantization_tf/learner.py b/learners/uniform_quantization_tf/learner.py index 74d86e1..b39d2a0 100644 --- a/learners/uniform_quantization_tf/learner.py +++ b/learners/uniform_quantization_tf/learner.py @@ -167,6 +167,7 @@ def __build_train(self): # pylint: disable=too-many-locals,too-many-statements quant_delay=FLAGS.uqtf_quant_delay, freeze_bn_delay=FLAGS.uqtf_freeze_bn_delay, scope=self.model_scope_quan) + self.global_step = tf.train.get_or_create_global_step() self.vars_quan = get_vars_by_scope(self.model_scope_quan) self.saver_quan_train = tf.train.Saver(self.vars_quan['all']) @@ -195,7 +196,6 @@ def __build_train(self): # pylint: disable=too-many-locals,too-many-statements tf.summary.scalar(key, value) # learning rate schedule - self.global_step = tf.train.get_or_create_global_step() lrn_rate, self.nb_iters_train = self.setup_lrn_rate(self.global_step) lrn_rate *= FLAGS.uqtf_lrn_rate_dcy @@ -262,6 +262,7 @@ def __build_eval(self): weight_bits=FLAGS.uqtf_weight_bits, activation_bits=FLAGS.uqtf_activation_bits, scope=self.model_scope_quan) + global_step_eval = tf.train.get_or_create_global_step() vars_quan = get_vars_by_scope(self.model_scope_quan) # model definition - distilled model From c6a7a593c26794639c7f501c25e8e529fae1aff9 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Thu, 10 Jan 2019 11:43:03 +0800 Subject: [PATCH 073/173] add a tool to add tensors to certain collections --- tools/conversion/add_to_collection.py | 100 ++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 tools/conversion/add_to_collection.py diff --git a/tools/conversion/add_to_collection.py b/tools/conversion/add_to_collection.py new file mode 100644 index 0000000..90bc2d8 --- /dev/null +++ b/tools/conversion/add_to_collection.py @@ -0,0 +1,100 @@ +# Tencent is pleased to support the open source community by making PocketFlow available. +# +# Copyright (C) 2018 THL A29 Limited, a Tencent company. All rights reserved. +# +# Licensed under the BSD 3-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Add a list of tensors to specified collections (useful when exporting *.pb & *.tflite models).""" + +import os +import re +import traceback +import tensorflow as tf + +FLAGS = tf.app.flags.FLAGS + +tf.app.flags.DEFINE_string('model_dir_in', './models_in', 'input model directory') +tf.app.flags.DEFINE_string('model_dir_out', './models_out', 'output model directory') +tf.app.flags.DEFINE_string('tensor_names', None, 'list of tensors names (comma-separated)') +tf.app.flags.DEFINE_string('coll_names', None, 'list of collection names (comma-separated)') + +''' +Example: SSD (VGG-16) @ Pascal VOC + +Input: +* data/IteratorGetNext:1 / (?, 300, 300, 3) / images +Output: +* quant_model/ssd300/multibox_head/cls_5/Conv2D:0 / (?, 1, 1, 84) / cls_preds +* quant_model/ssd300/multibox_head/loc_5/Conv2D:0 / (?, 1, 1, 16) / loc_preds +''' + +def main(unused_argv): + """Main entry. + + Args: + * unused_argv: unused arguments (after FLAGS is parsed) + """ + + try: + # setup the TF logging routine + tf.logging.set_verbosity(tf.logging.INFO) + + # add a list of tensors to specified collections + with tf.Graph().as_default() as graph: + # create a TensorFlow session + config = tf.ConfigProto() + config.gpu_options.allow_growth = True + sess = tf.Session(config=config) + + # restore a model from *.ckpt files + ckpt_path = tf.train.latest_checkpoint(FLAGS.model_dir_in) + meta_path = ckpt_path + '.meta' + saver = tf.train.import_meta_graph(meta_path) + saver.restore(sess, ckpt_path) + + # parse tensor & collection names + tensor_names = [sub_str.strip() for sub_str in FLAGS.tensor_names.split(',')] + coll_names = [sub_str.strip() for sub_str in FLAGS.coll_names.split(',')] + assert len(tensor_names) == len(coll_names), \ + '# of tensors and collections does not match: %d (tensor) vs. %d (collection)' \ + % (len(tensor_names), len(coll_names)) + + # obtain the full list of tensors in the graph + tensors = set() + for op in graph.get_operations(): + tensors |= set(op.inputs) | set(op.outputs) + tensors = list(tensors) + tensors.sort(key=lambda x: x.name) + + # find tensors and add them to corresponding collections + for tensor in tensors: + if tensor.name in tensor_names: + tf.logging.info('tensor: {} / {}'.format(tensor.name, tensor.shape)) + coll_name = coll_names[tensor_names.index(tensor.name)] + tf.add_to_collection(coll_name, tensor) + tf.logging.info('added tensor <{}> to collection <{}>'.format(tensor.name, coll_name)) + + # save the modified model + vars_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) + saver_new = tf.train.Saver(vars_list) + save_path = saver_new.save(sess, os.path.join(FLAGS.model_dir_out, 'model.ckpt')) + tf.logging.info('model saved to ' + save_path) + + # exit normally + return 0 + except ValueError: + traceback.print_exc() + return 1 # exit with errors + +if __name__ == '__main__': + tf.app.run() From fd7e33d984232fdb412eb384676227abbe790413 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Thu, 10 Jan 2019 14:32:54 +0800 Subject: [PATCH 074/173] add full evaluation code for uniform-quantized SSD --- learners/uniform_quantization_tf/learner.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/learners/uniform_quantization_tf/learner.py b/learners/uniform_quantization_tf/learner.py index b39d2a0..eaa723b 100644 --- a/learners/uniform_quantization_tf/learner.py +++ b/learners/uniform_quantization_tf/learner.py @@ -96,14 +96,17 @@ def train(self): # initialization self.sess_train.run([self.init_op, self.init_opt_op]) - self.sess_train.run(self.global_step.initializer) # reset the global step if FLAGS.enbl_multi_gpu: self.sess_train.run(self.bcast_op) + if self.is_primary_worker('global'): + for idx_iter in range(-10, 0): + log_rslt = self.sess_train.run(self.log_op) + log_str = ' | '.join(['%s = %.4e' % (name, value) + for name, value in zip(self.log_op_names, log_rslt)]) + tf.logging.info('iter #%d: %s' % (idx_iter + 1, log_str)) + # train the model through iterations and periodically save & evaluate the model - for idx_iter in range(-10, 0): - summary, log_rslt = self.sess_train.run([self.summary_op, self.log_op]) - self.__monitor_progress(summary, log_rslt, idx_iter, 1.0) time_prev = timer() for idx_iter in range(self.nb_iters_train): # train the model @@ -132,10 +135,14 @@ def evaluate(self): """Restore a model from the latest checkpoint files and then evaluate it.""" self.__restore_model(is_train=False) + tf.logging.info('global_step_eval = %d' % self.sess_eval.run(self.global_step_eval)) nb_iters = int(np.ceil(float(FLAGS.nb_smpls_eval) / FLAGS.batch_size_eval)) eval_rslts = np.zeros((nb_iters, len(self.eval_op))) + self.dump_n_eval(outputs=None, action='init') for idx_iter in range(nb_iters): - eval_rslts[idx_iter] = self.sess_eval.run(self.eval_op) + eval_rslts[idx_iter], outputs = self.sess_eval.run([self.eval_op, self.outputs_eval]) + self.dump_n_eval(outputs=outputs, action='dump') + self.dump_n_eval(outputs=None, action='eval') for idx, name in enumerate(self.eval_op_names): tf.logging.info('%s = %.4e' % (name, np.mean(eval_rslts[:, idx]))) @@ -262,7 +269,7 @@ def __build_eval(self): weight_bits=FLAGS.uqtf_weight_bits, activation_bits=FLAGS.uqtf_activation_bits, scope=self.model_scope_quan) - global_step_eval = tf.train.get_or_create_global_step() + self.global_step_eval = tf.train.get_or_create_global_step() vars_quan = get_vars_by_scope(self.model_scope_quan) # model definition - distilled model @@ -279,6 +286,7 @@ def __build_eval(self): # TF operations for evaluation self.eval_op = [loss] + list(metrics.values()) self.eval_op_names = ['loss'] + list(metrics.keys()) + self.outputs_eval = logits self.saver_quan_eval = tf.train.Saver(vars_quan['all']) # add input & output tensors to certain collections From 985799b5da3f0ef3d2b91f1914abeec05a9ebe28 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Thu, 10 Jan 2019 15:58:00 +0800 Subject: [PATCH 075/173] update the docker image for seven --- seven.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seven.yaml b/seven.yaml index cf06306..f8eca5f 100644 --- a/seven.yaml +++ b/seven.yaml @@ -3,7 +3,7 @@ kind: standalone jobname: pocket-flow container: image: - docker.oa.com/g_tfplus/tfplus:tensorflow1.8-python3.6-cuda9.0-cudnn7.0.4.31-ubuntu16.04-tfplus-v2 + docker.oa.com/g_tfplus/tfplus:tensorflow1.8-python3.6-cuda9.0-cudnn7.0.4.31-ubuntu16.04-tfplus-v3 #docker.oa.com/g_tfplus/horovod:python3.5 resources: nvidia.com/gpu: 1 From 3576af668c5dcd1edee6e1561a82a3a915fc520c Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Fri, 11 Jan 2019 15:55:06 +0800 Subject: [PATCH 076/173] periodically print messages during evaluation --- learners/uniform_quantization_tf/learner.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/learners/uniform_quantization_tf/learner.py b/learners/uniform_quantization_tf/learner.py index eaa723b..20f1aa3 100644 --- a/learners/uniform_quantization_tf/learner.py +++ b/learners/uniform_quantization_tf/learner.py @@ -123,6 +123,7 @@ def train(self): if self.is_primary_worker('global') and (idx_iter + 1) % FLAGS.save_step == 0: self.__save_model(is_train=True) self.evaluate() + self.auto_barrier() # save the final model if self.is_primary_worker('global'): @@ -140,6 +141,8 @@ def evaluate(self): eval_rslts = np.zeros((nb_iters, len(self.eval_op))) self.dump_n_eval(outputs=None, action='init') for idx_iter in range(nb_iters): + if (idx_iter + 1) % 100 == 0: + tf.logging.info('process the %d-th mini-batch for evaluation' % (idx_iter + 1)) eval_rslts[idx_iter], outputs = self.sess_eval.run([self.eval_op, self.outputs_eval]) self.dump_n_eval(outputs=outputs, action='dump') self.dump_n_eval(outputs=None, action='eval') From 20f0ab833513c1295c3756cfbf691a24d0848fd5 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Fri, 11 Jan 2019 16:19:30 +0800 Subject: [PATCH 077/173] move add_to_collection.py to tools/graph_utils --- tools/{conversion => graph_tools}/add_to_collection.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tools/{conversion => graph_tools}/add_to_collection.py (100%) diff --git a/tools/conversion/add_to_collection.py b/tools/graph_tools/add_to_collection.py similarity index 100% rename from tools/conversion/add_to_collection.py rename to tools/graph_tools/add_to_collection.py From 0f2efd1ba2ff96c3e78e114e3601eb33d6bc0681 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 14 Jan 2019 14:20:28 +0800 Subject: [PATCH 078/173] ignore dump files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c26a1f4..4b01c45 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ ratio.list path.conf nvidia-smi-dump ssd_outputs +dump From 2f58a486bef3fda691804e5c2044cc05b5ab7b7c Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 14 Jan 2019 14:23:36 +0800 Subject: [PATCH 079/173] insert quantization operations for unquantized activation nodes --- learners/uniform_quantization_tf/learner.py | 28 ++++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/learners/uniform_quantization_tf/learner.py b/learners/uniform_quantization_tf/learner.py index 20f1aa3..0a33065 100644 --- a/learners/uniform_quantization_tf/learner.py +++ b/learners/uniform_quantization_tf/learner.py @@ -23,6 +23,8 @@ from learners.abstract_learner import AbstractLearner from learners.distillation_helper import DistillationHelper +from learners.uniform_quantization_tf.utils import find_unquant_act_nodes +from learners.uniform_quantization_tf.utils import insert_quant_op from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw FLAGS = tf.app.flags.FLAGS @@ -81,6 +83,11 @@ def __init__(self, sm_writer, model_helper): self.auto_barrier() tf.logging.info('model files: ' + ', '.join(os.listdir('./models'))) + # detect unquantized activations nodes + self.unquant_node_names = find_unquant_act_nodes( + model_helper, self.data_scope, self.model_scope_quan) + tf.logging.info('unquantized activation nodes: {}'.format(self.unquant_node_names)) + # class-dependent initialization if FLAGS.enbl_dst: self.helper_dst = DistillationHelper(sm_writer, model_helper, self.mpi_comm) @@ -152,10 +159,11 @@ def evaluate(self): def __build_train(self): # pylint: disable=too-many-locals,too-many-statements """Build the training graph.""" - with tf.Graph().as_default(): + with tf.Graph().as_default() as graph: # create a TF session for the current graph config = tf.ConfigProto() config.gpu_options.visible_device_list = str(mgw.local_rank() if FLAGS.enbl_multi_gpu else 0) # pylint: disable=no-member + config.gpu_options.allow_growth = True # pylint: disable=no-member sess = tf.Session(config=config) # data input pipeline @@ -177,6 +185,8 @@ def __build_train(self): # pylint: disable=too-many-locals,too-many-statements quant_delay=FLAGS.uqtf_quant_delay, freeze_bn_delay=FLAGS.uqtf_freeze_bn_delay, scope=self.model_scope_quan) + for node_name in self.unquant_node_names: + insert_quant_op(graph, node_name, is_train=True) self.global_step = tf.train.get_or_create_global_step() self.vars_quan = get_vars_by_scope(self.model_scope_quan) self.saver_quan_train = tf.train.Saver(self.vars_quan['all']) @@ -254,10 +264,11 @@ def __build_train(self): # pylint: disable=too-many-locals,too-many-statements def __build_eval(self): """Build the evaluation graph.""" - with tf.Graph().as_default(): + with tf.Graph().as_default() as graph: # create a TF session for the current graph config = tf.ConfigProto() config.gpu_options.visible_device_list = str(mgw.local_rank() if FLAGS.enbl_multi_gpu else 0) # pylint: disable=no-member + config.gpu_options.allow_growth = True # pylint: disable=no-member self.sess_eval = tf.Session(config=config) # data input pipeline @@ -272,6 +283,8 @@ def __build_eval(self): weight_bits=FLAGS.uqtf_weight_bits, activation_bits=FLAGS.uqtf_activation_bits, scope=self.model_scope_quan) + for node_name in self.unquant_node_names: + insert_quant_op(graph, node_name, is_train=False) self.global_step_eval = tf.train.get_or_create_global_step() vars_quan = get_vars_by_scope(self.model_scope_quan) @@ -293,8 +306,15 @@ def __build_eval(self): self.saver_quan_eval = tf.train.Saver(vars_quan['all']) # add input & output tensors to certain collections - tf.add_to_collection('images_final', images) - tf.add_to_collection('logits_final', logits) + if not isinstance(images, dict): + tf.add_to_collection('images_final', images) + else: + tf.add_to_collection('images_final', images['image']) + if not isinstance(logits, dict): + tf.add_to_collection('logits_final', labels) + else: + for value in labels.values(): + tf.add_to_collection('logits_final', value) def __save_model(self, is_train): """Save the current model for training or evaluation. From ceefcffb52dc38dccf175f8ac53b2eb0783dbce8 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 14 Jan 2019 14:24:15 +0800 Subject: [PATCH 080/173] add utilities to detect unquantized activation nodes --- learners/uniform_quantization_tf/utils.py | 223 ++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 learners/uniform_quantization_tf/utils.py diff --git a/learners/uniform_quantization_tf/utils.py b/learners/uniform_quantization_tf/utils.py new file mode 100644 index 0000000..78b9183 --- /dev/null +++ b/learners/uniform_quantization_tf/utils.py @@ -0,0 +1,223 @@ +# Tencent is pleased to support the open source community by making PocketFlow available. +# +# Copyright (C) 2018 THL A29 Limited, a Tencent company. All rights reserved. +# +# Licensed under the BSD 3-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utility functions.""" + +import os +import subprocess +import tensorflow as tf +from tensorflow.contrib.quantize.python import common +from tensorflow.contrib.quantize.python import input_to_ops +from tensorflow.contrib.quantize.python import quant_ops +from tensorflow.contrib.lite.python import lite_constants as constants + +FLAGS = tf.app.flags.FLAGS + +tf.app.flags.DEFINE_string('uqtf_save_path_probe', './models_uqtf_probe/model.ckpt', + 'UQ-TF: probe model\'s save path') + +def insert_quant_op(graph, node_name, is_train): + """Insert quantization operations to the specified activation node. + + Args: + * graph: TensorFlow graph + * node_name: activation node's name + * is_train: insert training-related operations or not + """ + + # locate the node & activation operation + for op in graph.get_operations(): + if node_name in [node.name for node in op.outputs]: + tf.logging.info('op: {} / inputs: {} / outputs: {}'.format( + op.name, [node.name for node in op.inputs], [node.name for node in op.outputs])) + node = op.outputs[0] + activation_op = op + break + + # re-route the graph to insert quantization operations + input_to_ops_map = input_to_ops.InputToOps(graph) + consumer_ops = input_to_ops_map.ConsumerOperations(activation_op) + node_quant = quant_ops.MovingAvgQuantize( + node, is_training=is_train, num_bits=FLAGS.uqtf_activation_bits) + nb_update_inputs = common.RerouteTensor(node_quant, node, consumer_ops) + tf.logging.info('nb_update_inputs = %d' % nb_update_inputs) + +def export_tflite_model(input_coll, output_coll, images_shape, images_name): + """Export a *.tflite model from checkpoint files. + + Args: + * input_coll: input collection's name + * output_coll: output collection's name + + Returns: + * unquant_node_name: unquantized activation node name (None if not found) + """ + + # remove previously generated *.pb & *.tflite models + model_dir = os.path.dirname(FLAGS.uqtf_save_path_probe) + pb_path = os.path.join(model_dir, 'model.pb') + tflite_path = os.path.join(model_dir, 'model.tflite') + if os.path.exists(pb_path): + os.remove(pb_path) + if os.path.exists(tflite_path): + os.remove(tflite_path) + + # convert checkpoint files to a *.pb model + images_name_ph = 'images' + with tf.Graph().as_default() as graph: + # create a TensorFlow session + config = tf.ConfigProto() + config.gpu_options.allow_growth = True + sess = tf.Session(config=config) + + # restore the graph with inputs replaced + ckpt_path = tf.train.latest_checkpoint(model_dir) + meta_path = ckpt_path + '.meta' + images = tf.placeholder(tf.float32, shape=images_shape, name=images_name_ph) + saver = tf.train.import_meta_graph(meta_path, input_map={images_name: images}) + saver.restore(sess, ckpt_path) + + # obtain input & output nodes + net_inputs = tf.get_collection(input_coll) + net_outputs = tf.get_collection(output_coll) + for node in net_inputs: + tf.logging.info('inputs: {} / {}'.format(node.name, node.shape)) + for node in net_outputs: + tf.logging.info('outputs: {} / {}'.format(node.name, node.shape)) + + # write the original grpah to *.pb file + graph_def = tf.graph_util.convert_variables_to_constants( + sess, graph.as_graph_def(), [node.name.replace(':0', '') for node in net_outputs]) + tf.train.write_graph(graph_def, model_dir, os.path.basename(pb_path), as_text=False) + assert os.path.exists(pb_path), 'failed to generate a *.pb model' + + # convert the *.pb model to a *.tflite model + tf.logging.info(pb_path + ' -> ' + tflite_path) + arg_list = [ + '--graph_def_file ' + pb_path, + '--output_file ' + tflite_path, + '--input_arrays ' + images_name_ph, + '--output_arrays ' + ','.join([node.name.replace(':0', '') for node in net_outputs]), + '--inference_type QUANTIZED_UINT8', + '--mean_values 128', + '--std_dev_values 127'] + cmd_str = ' '.join(['tflite_convert'] + arg_list) + with open('./dump', 'w') as o_file: + subprocess.call(cmd_str, shell=True, stdout=o_file, stderr=o_file) + + # detect the unquantized activation node (if any) + unquant_node_name = None + if not os.path.exists(tflite_path): + flag_str = 'tensorflow/contrib/lite/toco/tooling_util.cc:1634]' + with open('./dump', 'r') as i_file: + for i_line in i_file: + if not 'is lacking min/max data' in i_line: + continue + for sub_line in i_line.split('\\n'): + if flag_str in sub_line: + sub_strs = sub_line.replace(',', ' ').split() + unquant_node_name = sub_strs[sub_strs.index(flag_str) + 2] + ':0' + break + + return unquant_node_name + +def find_unquant_act_nodes(model_helper, data_scope, model_scope): + """Find unquantized activation nodes in the model. + + TensorFlow's quantization-aware training APIs insert quantization operations into the graph, + so that model weights can be fine-tuned with quantization error taken into consideration. + However, these APIs only insert quantization operations into nodes matching certain topology + rules, and some nodes may be left unquantized. When converting such model to *.tflite model, + these unquantized nodes will introduce extra performance loss. + Here, we provide a utility function to detect these unquantized nodes before training, so that + quantization operations can be inserted. The resulting model can be smoothly exported to a + *.tflite model. + + Args: + * model_helper: model helper with definitions of model & dataset + # data_scope: data scope name + * model_scope: model scope name + + Returns: + * unquant_node_names: list of unquantized activation node names + """ + + # obtain the image tensor's name & shape + with tf.Graph().as_default(): + with tf.variable_scope(data_scope): + iterator = model_helper.build_dataset_eval() + inputs, labels = iterator.get_next() + if not isinstance(inputs, dict): + images_shape, images_name = inputs.shape, inputs.name + else: + images_shape, images_name = inputs['image'].shape, inputs['image'].name + + # create-quantize-save-export the model, and check for unquantized nodes + input_coll = 'net_inputs' + output_coll = 'net_outputs' + unquant_node_names = [] + while True: + # create a model, quantize it, and then save + with tf.Graph().as_default() as graph: + # create a TensorFlow session + config = tf.ConfigProto() + config.gpu_options.allow_growth = True + sess = tf.Session(config=config) + + # data input pipeline + with tf.variable_scope(data_scope): + iterator = model_helper.build_dataset_eval() + inputs, labels = iterator.get_next() + + # model definition - uniform quantized model + with tf.variable_scope(model_scope): + outputs = model_helper.forward_eval(inputs) + tf.contrib.quantize.experimental_create_eval_graph( + weight_bits=FLAGS.uqtf_weight_bits, + activation_bits=FLAGS.uqtf_activation_bits, + scope=model_scope) + + # manually insert quantization operations + for node_name in unquant_node_names: + insert_quant_op(graph, node_name, is_train=False) + + # add input & output tensors to collections + if not isinstance(inputs, dict): + tf.add_to_collection(input_coll, inputs) + else: + tf.add_to_collection(input_coll, inputs['image']) + if not isinstance(outputs, dict): + tf.add_to_collection(output_coll, outputs) + else: + for value in outputs.values(): + tf.add_to_collection(output_coll, value) + + # save the model + vars_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=model_scope) + saver = tf.train.Saver(vars_list) + sess.run(tf.variables_initializer(vars_list)) + save_path = saver.save(sess, FLAGS.uqtf_save_path_probe) + tf.logging.info('probe model saved to ' + save_path) + + # attempt to export a *.tflite model and detect unquantized activation nodes (if any) + unquant_node_name = export_tflite_model(input_coll, output_coll, images_shape, images_name) + if unquant_node_name: + unquant_node_names += [unquant_node_name] + tf.logging.info('node <%s> is not quantized' % unquant_node_name) + else: + break + + return unquant_node_names From fbd90017d56f6abaa4934be1ebd97468e4aa807d Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 14 Jan 2019 15:52:02 +0800 Subject: [PATCH 081/173] bugfix: typo (labels -> logits) --- learners/uniform_quantization_tf/learner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/learners/uniform_quantization_tf/learner.py b/learners/uniform_quantization_tf/learner.py index 0a33065..f9eba7c 100644 --- a/learners/uniform_quantization_tf/learner.py +++ b/learners/uniform_quantization_tf/learner.py @@ -311,9 +311,9 @@ def __build_eval(self): else: tf.add_to_collection('images_final', images['image']) if not isinstance(logits, dict): - tf.add_to_collection('logits_final', labels) + tf.add_to_collection('logits_final', logits) else: - for value in labels.values(): + for value in logits.values(): tf.add_to_collection('logits_final', value) def __save_model(self, is_train): From 50877c26e9d52a1995b9cd4045164749bfe3fc37 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 14 Jan 2019 17:10:22 +0800 Subject: [PATCH 082/173] do not add nodes to collection in ref. model for distillation --- learners/full_precision/learner.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/learners/full_precision/learner.py b/learners/full_precision/learner.py index 4184a62..ae4f6fa 100644 --- a/learners/full_precision/learner.py +++ b/learners/full_precision/learner.py @@ -47,6 +47,7 @@ def __init__(self, sm_writer, model_helper, model_scope=None, enbl_dst=None): if model_scope is not None: self.model_scope = model_scope self.enbl_dst = enbl_dst if enbl_dst is not None else FLAGS.enbl_dst + self.enbl_add_to_coll = True if enbl_dst is None else False # class-dependent initialization if self.enbl_dst: @@ -119,7 +120,11 @@ def __build(self, is_train): # pylint: disable=too-many-locals with tf.variable_scope(self.data_scope): iterator = self.build_dataset_train() if is_train else self.build_dataset_eval() images, labels = iterator.get_next() - tf.add_to_collection('images_final', images) + if self.enbl_add_to_coll: + if not isinstance(images, dict): + tf.add_to_collection('images_final', images) + else: + tf.add_to_collection('images_final', images['image']) # model definition - distilled model if self.enbl_dst: @@ -129,7 +134,12 @@ def __build(self, is_train): # pylint: disable=too-many-locals with tf.variable_scope(self.model_scope): # forward pass logits = self.forward_train(images) if is_train else self.forward_eval(images) - tf.add_to_collection('logits_final', logits) + if self.enbl_add_to_coll: + if not isinstance(logits, dict): + tf.add_to_collection('logits_final', logits) + else: + for value in logits.values(): + tf.add_to_collection('logits_final', value) # loss & extra evalution metrics loss, metrics = self.calc_loss(labels, logits, self.trainable_vars) From a227bfd515e7d274a999c90e41342f06211b1eca Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Tue, 15 Jan 2019 09:23:48 +0800 Subject: [PATCH 083/173] update the TensorFlow version at seven --- main.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/main.sh b/main.sh index d476ea6..cd43e42 100755 --- a/main.sh +++ b/main.sh @@ -10,6 +10,8 @@ mkdir -p ~/.pip/ \ cat ~/.pip/pip.conf # install Python packages with Internet access +pip install tensorflow-gpu==1.12.0 +pip install horovod pip install docopt pip install hdfs pip install scipy @@ -19,6 +21,7 @@ pip install mpi4py # add the current directory to PYTHONPATH export PYTHONPATH=${PYTHONPATH}:`pwd` +export LD_LIBRARY_PATH=/opt/ml/disk/local/cuda/lib64:$LD_LIBRARY_PATH # start TensorBoard LOG_DIR=/opt/ml/log From 0b4ffc3a8b87c1116c651a13472b9d7e6039328f Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Tue, 15 Jan 2019 09:24:38 +0800 Subject: [PATCH 084/173] make auto_barrier() & is_primary_worker() as public util func --- learners/abstract_learner.py | 14 ++++---------- utils/misc_utils.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/learners/abstract_learner.py b/learners/abstract_learner.py index 86ebaff..9a32134 100644 --- a/learners/abstract_learner.py +++ b/learners/abstract_learner.py @@ -23,6 +23,8 @@ import subprocess import tensorflow as tf +from utils.misc_utils import auto_barrier as auto_barrier_impl +from utils.misc_utils import is_primary_worker as is_primary_worker_impl from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw FLAGS = tf.app.flags.FLAGS @@ -124,10 +126,7 @@ def download_model(self): def auto_barrier(self): """Automatically insert a barrier for multi-GPU training, or pass for single-GPU training.""" - if FLAGS.enbl_multi_gpu: - self.mpi_comm.Barrier() - else: - pass + auto_barrier_impl(self.mpi_comm) @classmethod def is_primary_worker(cls, scope='global'): @@ -140,12 +139,7 @@ def is_primary_worker(cls, scope='global'): * flag: whether is the primary worker """ - if scope == 'global': - return True if not FLAGS.enbl_multi_gpu else mgw.rank() == 0 - elif scope == 'local': - return True if not FLAGS.enbl_multi_gpu else mgw.local_rank() == 0 - else: - raise ValueError('unrecognized worker scope: ' + scope) + return is_primary_worker_impl(scope) @property def vars(self): diff --git a/utils/misc_utils.py b/utils/misc_utils.py index aa811ef..7388c8e 100644 --- a/utils/misc_utils.py +++ b/utils/misc_utils.py @@ -22,6 +22,18 @@ FLAGS = tf.app.flags.FLAGS +def auto_barrier(mpi_comm=None): + """Automatically insert a barrier for multi-GPU training, or pass for single-GPU training. + + Args: + * mpi_comm: MPI communication object + """ + + if FLAGS.enbl_multi_gpu: + mpi_comm.Barrier() + else: + pass + def is_primary_worker(scope='global'): """Check whether is the primary worker of all nodes (global) or the current node (local). From 36cf0ccf1376125285edea3f3e89a8fd8f6063a8 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Tue, 15 Jan 2019 15:59:32 +0800 Subject: [PATCH 085/173] bugfix: models not completely removed when making the minimal dir --- scripts/create_minimal.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/create_minimal.sh b/scripts/create_minimal.sh index 82305e1..36ccfcc 100755 --- a/scripts/create_minimal.sh +++ b/scripts/create_minimal.sh @@ -17,9 +17,9 @@ cd ${dir_temp} # remove redundant files #git clean -xdf # all files ignored by git -rm -r ./models -rm -r ./logs -rm -rf .git +rm -rf ./models* +rm -rf ./logs +rm -rf .git .gitignore cp ${dir_curr}/path.conf . # return to the original directory From 1fd0088e4f45cc9cfec3470c572bf366fc4ed094 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Fri, 18 Jan 2019 16:26:11 +0800 Subject: [PATCH 086/173] use to control whether BN layer is fused --- utils/external/resnet_model.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/utils/external/resnet_model.py b/utils/external/resnet_model.py index e17aab5..673b915 100644 --- a/utils/external/resnet_model.py +++ b/utils/external/resnet_model.py @@ -33,6 +33,14 @@ import tensorflow as tf +FLAGS = tf.app.flags.FLAGS + +tf.app.flags.DEFINE_boolean('enbl_fused_batchnorm', True, + 'Enable fused batch normalization or not. Enable this will bring a ' + 'significant performance boost, but may not be able to export a ' + '*.tflite model when using TensorFlow\'s quantization-aware training ' + 'APIs (at least for TensorFlow==1.12.0, the answer is no).') + _BATCH_NORM_DECAY = 0.997 _BATCH_NORM_EPSILON = 1e-5 DEFAULT_VERSION = 2 @@ -51,7 +59,7 @@ def batch_norm(inputs, training, data_format): return tf.layers.batch_normalization( inputs=inputs, axis=1 if data_format == 'channels_first' else 3, momentum=_BATCH_NORM_DECAY, epsilon=_BATCH_NORM_EPSILON, center=True, - scale=True, training=training, fused=True) + scale=True, training=training, fused=FLAGS.enbl_fused_batchnorm) def fixed_padding(inputs, kernel_size, data_format): From 623ec56c2699436b597f0d32070a1505f49741a6 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Fri, 18 Jan 2019 16:50:13 +0800 Subject: [PATCH 087/173] use Adam instead SGD-M for optimization --- learners/uniform_quantization_tf/learner.py | 27 +++++++++++++-------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/learners/uniform_quantization_tf/learner.py b/learners/uniform_quantization_tf/learner.py index f9eba7c..a3b1597 100644 --- a/learners/uniform_quantization_tf/learner.py +++ b/learners/uniform_quantization_tf/learner.py @@ -42,6 +42,8 @@ 'UT-TF: # of steps after which moving mean and variance are frozen \ and used instead of batch statistics during training.') tf.app.flags.DEFINE_float('uqtf_lrn_rate_dcy', 1e-2, 'UQ-TF: learning rate\'s decaying factor') +tf.app.flags.DEFINE_boolean('uqtf_enbl_manual_quant', False, + 'UQ-TF: enable manually inserting quantization operations') def get_vars_by_scope(scope): """Get list of variables within certain name scope. @@ -84,8 +86,10 @@ def __init__(self, sm_writer, model_helper): tf.logging.info('model files: ' + ', '.join(os.listdir('./models'))) # detect unquantized activations nodes - self.unquant_node_names = find_unquant_act_nodes( - model_helper, self.data_scope, self.model_scope_quan) + self.unquant_node_names = [] + if FLAGS.uqtf_enbl_manual_quant: + self.unquant_node_names = find_unquant_act_nodes( + model_helper, self.data_scope, self.model_scope_quan, self.mpi_comm) tf.logging.info('unquantized activation nodes: {}'.format(self.unquant_node_names)) # class-dependent initialization @@ -170,15 +174,14 @@ def __build_train(self): # pylint: disable=too-many-locals,too-many-statements with tf.variable_scope(self.data_scope): iterator = self.build_dataset_train() images, labels = iterator.get_next() - if not isinstance(images, dict): - images.set_shape((FLAGS.batch_size, images.shape[1], images.shape[2], images.shape[3])) - else: - shape = images['image'].shape - images['image'].set_shape((FLAGS.batch_size, shape[1], shape[2], shape[3])) # model definition - uniform quantized model - part 1 with tf.variable_scope(self.model_scope_quan): logits_quan = self.forward_train(images) + if not isinstance(logits_quan, dict): + outputs = tf.nn.softmax(logits_quan) + else: + outputs = tf.nn.softmax(logits_quan['cls_pred']) tf.contrib.quantize.experimental_create_training_graph( weight_bits=FLAGS.uqtf_weight_bits, activation_bits=FLAGS.uqtf_activation_bits, @@ -243,7 +246,8 @@ def __build_train(self): # pylint: disable=too-many-locals,too-many-statements self.init_op = tf.group(init_ops) # TF operations for fine-tuning - optimizer_base = tf.train.MomentumOptimizer(lrn_rate, FLAGS.momentum) + #optimizer_base = tf.train.MomentumOptimizer(lrn_rate, FLAGS.momentum) + optimizer_base = tf.train.AdamOptimizer(lrn_rate) if not FLAGS.enbl_multi_gpu: optimizer = optimizer_base else: @@ -279,6 +283,10 @@ def __build_eval(self): # model definition - uniform quantized model - part 1 with tf.variable_scope(self.model_scope_quan): logits = self.forward_eval(images) + if not isinstance(logits, dict): + outputs = tf.nn.softmax(logits) + else: + outputs = tf.nn.softmax(logits['cls_pred']) tf.contrib.quantize.experimental_create_eval_graph( weight_bits=FLAGS.uqtf_weight_bits, activation_bits=FLAGS.uqtf_activation_bits, @@ -313,8 +321,7 @@ def __build_eval(self): if not isinstance(logits, dict): tf.add_to_collection('logits_final', logits) else: - for value in logits.values(): - tf.add_to_collection('logits_final', value) + tf.add_to_collection('logits_final', logits['cls_pred']) def __save_model(self, is_train): """Save the current model for training or evaluation. From 8adba3c5de924609e68100127281bf4d26cd491c Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Fri, 18 Jan 2019 16:57:22 +0800 Subject: [PATCH 088/173] automatically detect all the unquantized nodes --- learners/uniform_quantization_tf/utils.py | 193 +++++++++++++++------- 1 file changed, 137 insertions(+), 56 deletions(-) diff --git a/learners/uniform_quantization_tf/utils.py b/learners/uniform_quantization_tf/utils.py index 78b9183..031c04b 100644 --- a/learners/uniform_quantization_tf/utils.py +++ b/learners/uniform_quantization_tf/utils.py @@ -24,10 +24,31 @@ from tensorflow.contrib.quantize.python import quant_ops from tensorflow.contrib.lite.python import lite_constants as constants +from utils.misc_utils import auto_barrier +from utils.misc_utils import is_primary_worker +from utils.multi_gpu_wrapper import MultiGpuWrapper as mgw + FLAGS = tf.app.flags.FLAGS tf.app.flags.DEFINE_string('uqtf_save_path_probe', './models_uqtf_probe/model.ckpt', 'UQ-TF: probe model\'s save path') +tf.app.flags.DEFINE_string('uqtf_save_path_probe_eval', './models_uqtf_probe_eval/model.ckpt', + 'UQ-TF: probe model\'s save path for evaluation') + +def create_session(): + """Create a TensorFlow session. + + Return: + * sess: TensorFlow session + """ + + # create a TensorFlow session + config = tf.ConfigProto() + config.gpu_options.visible_device_list = str(mgw.local_rank() if FLAGS.enbl_multi_gpu else 0) # pylint: disable=no-member + config.gpu_options.allow_growth = True # pylint: disable=no-member + sess = tf.Session(config=config) + + return sess def insert_quant_op(graph, node_name, is_train): """Insert quantization operations to the specified activation node. @@ -67,9 +88,10 @@ def export_tflite_model(input_coll, output_coll, images_shape, images_name): """ # remove previously generated *.pb & *.tflite models - model_dir = os.path.dirname(FLAGS.uqtf_save_path_probe) - pb_path = os.path.join(model_dir, 'model.pb') - tflite_path = os.path.join(model_dir, 'model.tflite') + model_dir = os.path.dirname(FLAGS.uqtf_save_path_probe_eval) + idx_worker = mgw.local_rank() if FLAGS.enbl_multi_gpu else 0 + pb_path = os.path.join(model_dir, 'model_%d.pb' % idx_worker) + tflite_path = os.path.join(model_dir, 'model_%d.tflite' % idx_worker) if os.path.exists(pb_path): os.remove(pb_path) if os.path.exists(tflite_path): @@ -79,9 +101,7 @@ def export_tflite_model(input_coll, output_coll, images_shape, images_name): images_name_ph = 'images' with tf.Graph().as_default() as graph: # create a TensorFlow session - config = tf.ConfigProto() - config.gpu_options.allow_growth = True - sess = tf.Session(config=config) + sess = create_session() # restore the graph with inputs replaced ckpt_path = tf.train.latest_checkpoint(model_dir) @@ -92,7 +112,8 @@ def export_tflite_model(input_coll, output_coll, images_shape, images_name): # obtain input & output nodes net_inputs = tf.get_collection(input_coll) - net_outputs = tf.get_collection(output_coll) + net_logits = tf.get_collection(output_coll)[0] + net_outputs = [tf.nn.softmax(net_logits)] for node in net_inputs: tf.logging.info('inputs: {} / {}'.format(node.name, node.shape)) for node in net_outputs: @@ -132,9 +153,86 @@ def export_tflite_model(input_coll, output_coll, images_shape, images_name): unquant_node_name = sub_strs[sub_strs.index(flag_str) + 2] + ':0' break + assert unquant_node_name is not None, 'unable to locate the unquantized node' + return unquant_node_name -def find_unquant_act_nodes(model_helper, data_scope, model_scope): +def build_graph(model_helper, unquant_node_names, config, is_train): + """Build a graph for training or evaluation. + + Args: + * model_helper: model helper with definitions of model & dataset + * unquant_node_names: list of unquantized activation node names + * config: graph configuration + * is_train: insert training-related operations or not + + Returns: + * model: dictionary of model-related objects & operations + """ + + # setup function handles + if is_train: + build_dataset_fn = model_helper.build_dataset_train + forward_fn = model_helper.forward_train + create_quant_graph_fn = tf.contrib.quantize.experimental_create_training_graph + else: + build_dataset_fn = model_helper.build_dataset_eval + forward_fn = model_helper.forward_eval + create_quant_graph_fn = tf.contrib.quantize.experimental_create_eval_graph + + # build a graph for trianing or evaluation + model = {} + with tf.Graph().as_default() as graph: + # data input pipeline + with tf.variable_scope(config['data_scope']): + iterator = build_dataset_fn() + inputs, __ = iterator.get_next() + + # model definition - uniform quantized model + with tf.variable_scope(config['model_scope']): + # obtain outputs from model's forward-pass + outputs = forward_fn(inputs) + if not isinstance(outputs, dict): + outputs_sfmax = tf.nn.softmax(outputs) # is logits + else: + outputs_sfmax = tf.nn.softmax(outputs['cls_pred']) # is logits + + # quantize the graph using TensorFlow APIs + create_quant_graph_fn( + weight_bits=FLAGS.uqtf_weight_bits, + activation_bits=FLAGS.uqtf_activation_bits, + scope=config['model_scope']) + + # manually insert quantization operations + for node_name in unquant_node_names: + insert_quant_op(graph, node_name, is_train=is_train) + + # randomly increase each trainable variable's value + incr_ops = [] + for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES): + incr_ops += [var.assign_add(tf.random.uniform(var.shape))] + incr_op = tf.group(incr_ops) + + # add input & output tensors to collections + if not isinstance(inputs, dict): + tf.add_to_collection(config['input_coll'], inputs) + else: + tf.add_to_collection(config['input_coll'], inputs['image']) + if not isinstance(outputs, dict): + tf.add_to_collection(config['output_coll'], outputs) + else: + tf.add_to_collection(config['output_coll'], outputs['cls_pred']) + + # save the model + vars_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=config['model_scope']) + model['sess'] = create_session() + model['saver'] = tf.train.Saver(vars_list) + model['init_op'] = tf.variables_initializer(vars_list) + model['incr_op'] = incr_op + + return model + +def find_unquant_act_nodes(model_helper, data_scope, model_scope, mpi_comm): """Find unquantized activation nodes in the model. TensorFlow's quantization-aware training APIs insert quantization operations into the graph, @@ -148,13 +246,22 @@ def find_unquant_act_nodes(model_helper, data_scope, model_scope): Args: * model_helper: model helper with definitions of model & dataset - # data_scope: data scope name + * data_scope: data scope name * model_scope: model scope name + * mpi_comm: MPI communication object Returns: * unquant_node_names: list of unquantized activation node names """ + # setup configurations + config = { + 'data_scope': data_scope, + 'model_scope': model_scope, + 'input_coll': 'inputs', + 'output_coll': 'outputs', + } + # obtain the image tensor's name & shape with tf.Graph().as_default(): with tf.variable_scope(data_scope): @@ -165,55 +272,29 @@ def find_unquant_act_nodes(model_helper, data_scope, model_scope): else: images_shape, images_name = inputs['image'].shape, inputs['image'].name - # create-quantize-save-export the model, and check for unquantized nodes - input_coll = 'net_inputs' - output_coll = 'net_outputs' + # iteratively check for unquantized nodes unquant_node_names = [] while True: - # create a model, quantize it, and then save - with tf.Graph().as_default() as graph: - # create a TensorFlow session - config = tf.ConfigProto() - config.gpu_options.allow_growth = True - sess = tf.Session(config=config) - - # data input pipeline - with tf.variable_scope(data_scope): - iterator = model_helper.build_dataset_eval() - inputs, labels = iterator.get_next() - - # model definition - uniform quantized model - with tf.variable_scope(model_scope): - outputs = model_helper.forward_eval(inputs) - tf.contrib.quantize.experimental_create_eval_graph( - weight_bits=FLAGS.uqtf_weight_bits, - activation_bits=FLAGS.uqtf_activation_bits, - scope=model_scope) - - # manually insert quantization operations - for node_name in unquant_node_names: - insert_quant_op(graph, node_name, is_train=False) - - # add input & output tensors to collections - if not isinstance(inputs, dict): - tf.add_to_collection(input_coll, inputs) - else: - tf.add_to_collection(input_coll, inputs['image']) - if not isinstance(outputs, dict): - tf.add_to_collection(output_coll, outputs) - else: - for value in outputs.values(): - tf.add_to_collection(output_coll, value) - - # save the model - vars_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=model_scope) - saver = tf.train.Saver(vars_list) - sess.run(tf.variables_initializer(vars_list)) - save_path = saver.save(sess, FLAGS.uqtf_save_path_probe) - tf.logging.info('probe model saved to ' + save_path) - - # attempt to export a *.tflite model and detect unquantized activation nodes (if any) - unquant_node_name = export_tflite_model(input_coll, output_coll, images_shape, images_name) + # build training & evaluation graphs + model_train = build_graph(model_helper, unquant_node_names, config, is_train=True) + model_eval = build_graph(model_helper, unquant_node_names, config, is_train=False) + + # initialize a model in the training graph, and then save + model_train['sess'].run(model_train['init_op']) + model_train['sess'].run(model_train['incr_op']) + save_path = model_train['saver'].save(model_train['sess'], FLAGS.uqtf_save_path_probe) + tf.logging.info('model saved to ' + save_path) + + # restore a model in the evaluation graph from *.ckpt files, and then save again + save_path = tf.train.latest_checkpoint(os.path.dirname(FLAGS.uqtf_save_path_probe)) + model_eval['saver'].restore(model_eval['sess'], save_path) + tf.logging.info('model restored from ' + save_path) + save_path = model_eval['saver'].save(model_eval['sess'], FLAGS.uqtf_save_path_probe_eval) + tf.logging.info('model saved to ' + save_path) + + # try to export *.tflite models and check for unquantized nodes (if any) + unquant_node_name = export_tflite_model( + config['input_coll'], config['output_coll'], images_shape, images_name) if unquant_node_name: unquant_node_names += [unquant_node_name] tf.logging.info('node <%s> is not quantized' % unquant_node_name) From 047d354b3bdd313d48fbe328db3962faf1b97073 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Fri, 18 Jan 2019 19:00:46 +0800 Subject: [PATCH 089/173] use Python's TFLiteConverter for speed-up --- learners/uniform_quantization_tf/utils.py | 43 +++++++++-------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/learners/uniform_quantization_tf/utils.py b/learners/uniform_quantization_tf/utils.py index 031c04b..e520ced 100644 --- a/learners/uniform_quantization_tf/utils.py +++ b/learners/uniform_quantization_tf/utils.py @@ -22,7 +22,7 @@ from tensorflow.contrib.quantize.python import common from tensorflow.contrib.quantize.python import input_to_ops from tensorflow.contrib.quantize.python import quant_ops -from tensorflow.contrib.lite.python import lite_constants as constants +from tensorflow.contrib.lite.python import lite_constants from utils.misc_utils import auto_barrier from utils.misc_utils import is_primary_worker @@ -125,34 +125,25 @@ def export_tflite_model(input_coll, output_coll, images_shape, images_name): tf.train.write_graph(graph_def, model_dir, os.path.basename(pb_path), as_text=False) assert os.path.exists(pb_path), 'failed to generate a *.pb model' - # convert the *.pb model to a *.tflite model + # convert the *.pb model to a *.tflite model and detect the unquantized activation node (if any) tf.logging.info(pb_path + ' -> ' + tflite_path) - arg_list = [ - '--graph_def_file ' + pb_path, - '--output_file ' + tflite_path, - '--input_arrays ' + images_name_ph, - '--output_arrays ' + ','.join([node.name.replace(':0', '') for node in net_outputs]), - '--inference_type QUANTIZED_UINT8', - '--mean_values 128', - '--std_dev_values 127'] - cmd_str = ' '.join(['tflite_convert'] + arg_list) - with open('./dump', 'w') as o_file: - subprocess.call(cmd_str, shell=True, stdout=o_file, stderr=o_file) - - # detect the unquantized activation node (if any) + converter = tf.contrib.lite.TFLiteConverter.from_frozen_graph( + pb_path, [images_name_ph], [node.name.replace(':0', '') for node in net_outputs]) + converter.inference_type = lite_constants.QUANTIZED_UINT8 + converter.quantized_input_stats = {images_name_ph: (0., 1.)} unquant_node_name = None - if not os.path.exists(tflite_path): + try: + tflite_model = converter.convert() + with open(tflite_path, 'wb') as o_file: + o_file.write(tflite_model) + except Exception as err: + err_msg = str(err) flag_str = 'tensorflow/contrib/lite/toco/tooling_util.cc:1634]' - with open('./dump', 'r') as i_file: - for i_line in i_file: - if not 'is lacking min/max data' in i_line: - continue - for sub_line in i_line.split('\\n'): - if flag_str in sub_line: - sub_strs = sub_line.replace(',', ' ').split() - unquant_node_name = sub_strs[sub_strs.index(flag_str) + 2] + ':0' - break - + for sub_line in err_msg.split('\\n'): + if flag_str in sub_line: + sub_strs = sub_line.replace(',', ' ').split() + unquant_node_name = sub_strs[sub_strs.index(flag_str) + 2] + ':0' + break assert unquant_node_name is not None, 'unable to locate the unquantized node' return unquant_node_name From d383837674a890b2b8e13812315fe6d3d8b90526 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 21 Jan 2019 09:42:42 +0800 Subject: [PATCH 090/173] re-organize 's implementation --- learners/distillation_helper.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/learners/distillation_helper.py b/learners/distillation_helper.py index 02ebd90..2cd8468 100644 --- a/learners/distillation_helper.py +++ b/learners/distillation_helper.py @@ -48,9 +48,8 @@ def __init__(self, sm_writer, model_helper, mpi_comm): # initialize a full-precision model self.model_scope = 'distilled_model' # to distinguish from models created by other learners - enbl_dst = False # disable the distillation loss for teacher model from learners.full_precision.learner import FullPrecLearner - self.learner = FullPrecLearner(sm_writer, model_helper, self.model_scope, enbl_dst) + self.learner = FullPrecLearner(sm_writer, model_helper, self.model_scope, enbl_dst=False) # initialize a model for training with the distillation loss if is_primary_worker('local'): @@ -112,17 +111,18 @@ def __initialize(self): # download the pre-trained model from HDFS self.learner.download_model() - - # rename the variable scope of pre-trained model if os.path.isdir(os.path.dirname(FLAGS.save_path_dst)): shutil.rmtree(os.path.dirname(FLAGS.save_path_dst)) shutil.copytree(os.path.dirname(FLAGS.save_path), os.path.dirname(FLAGS.save_path_dst)) - self.__rename_var_scope() - self.__evaluate_model() - def __rename_var_scope(self): - """Rename the name scope of all variables.""" + # restore a pre-trained model and then evaluate + self.__restore() + self.__evaluate() + + def __restore(self): + """Restore a pre-trained model with the variable scope renamed.""" + # rename the variable scope ckpt_dir = os.path.dirname(FLAGS.save_path_dst) ckpt = tf.train.get_checkpoint_state(ckpt_dir) with tf.Graph().as_default(): @@ -139,14 +139,14 @@ def __rename_var_scope(self): sess.run(tf.global_variables_initializer()) saver.save(sess, ckpt.model_checkpoint_path) # pylint: disable=no-member - def __evaluate_model(self): - """Evaluate the model's loss & accuracy.""" - # restore the model from checkpoint files ckpt_file = tf.train.latest_checkpoint(os.path.dirname(FLAGS.save_path_dst)) self.learner.saver_eval.restore(self.learner.sess_eval, ckpt_file) tf.logging.info('model restored from ' + ckpt_file) + def __evaluate(self): + """Evaluate the model's loss & accuracy.""" + # evaluate the model losses, accuracies = [], [] nb_iters = int(np.ceil(float(FLAGS.nb_smpls_eval) / FLAGS.batch_size_eval)) From fdc2376d63e5dbd4ffef24ee92126d1620649ed7 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 21 Jan 2019 09:49:59 +0800 Subject: [PATCH 091/173] add tensors to input/output collections regardless of --- learners/full_precision/learner.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/learners/full_precision/learner.py b/learners/full_precision/learner.py index ae4f6fa..3052f52 100644 --- a/learners/full_precision/learner.py +++ b/learners/full_precision/learner.py @@ -47,7 +47,6 @@ def __init__(self, sm_writer, model_helper, model_scope=None, enbl_dst=None): if model_scope is not None: self.model_scope = model_scope self.enbl_dst = enbl_dst if enbl_dst is not None else FLAGS.enbl_dst - self.enbl_add_to_coll = True if enbl_dst is None else False # class-dependent initialization if self.enbl_dst: @@ -120,11 +119,10 @@ def __build(self, is_train): # pylint: disable=too-many-locals with tf.variable_scope(self.data_scope): iterator = self.build_dataset_train() if is_train else self.build_dataset_eval() images, labels = iterator.get_next() - if self.enbl_add_to_coll: - if not isinstance(images, dict): - tf.add_to_collection('images_final', images) - else: - tf.add_to_collection('images_final', images['image']) + if not isinstance(images, dict): + tf.add_to_collection('images_final', images) + else: + tf.add_to_collection('images_final', images['image']) # model definition - distilled model if self.enbl_dst: @@ -134,12 +132,11 @@ def __build(self, is_train): # pylint: disable=too-many-locals with tf.variable_scope(self.model_scope): # forward pass logits = self.forward_train(images) if is_train else self.forward_eval(images) - if self.enbl_add_to_coll: - if not isinstance(logits, dict): - tf.add_to_collection('logits_final', logits) - else: - for value in logits.values(): - tf.add_to_collection('logits_final', value) + if not isinstance(logits, dict): + tf.add_to_collection('logits_final', logits) + else: + for value in logits.values(): + tf.add_to_collection('logits_final', value) # loss & extra evalution metrics loss, metrics = self.calc_loss(labels, logits, self.trainable_vars) From dbccbe290c8ab8a33d9ed9b89cc5b329d15f5efe Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 21 Jan 2019 09:55:00 +0800 Subject: [PATCH 092/173] remove debug-only code --- learners/uniform_quantization_tf/learner.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/learners/uniform_quantization_tf/learner.py b/learners/uniform_quantization_tf/learner.py index a3b1597..da42766 100644 --- a/learners/uniform_quantization_tf/learner.py +++ b/learners/uniform_quantization_tf/learner.py @@ -110,13 +110,6 @@ def train(self): if FLAGS.enbl_multi_gpu: self.sess_train.run(self.bcast_op) - if self.is_primary_worker('global'): - for idx_iter in range(-10, 0): - log_rslt = self.sess_train.run(self.log_op) - log_str = ' | '.join(['%s = %.4e' % (name, value) - for name, value in zip(self.log_op_names, log_rslt)]) - tf.logging.info('iter #%d: %s' % (idx_iter + 1, log_str)) - # train the model through iterations and periodically save & evaluate the model time_prev = timer() for idx_iter in range(self.nb_iters_train): @@ -147,7 +140,6 @@ def evaluate(self): """Restore a model from the latest checkpoint files and then evaluate it.""" self.__restore_model(is_train=False) - tf.logging.info('global_step_eval = %d' % self.sess_eval.run(self.global_step_eval)) nb_iters = int(np.ceil(float(FLAGS.nb_smpls_eval) / FLAGS.batch_size_eval)) eval_rslts = np.zeros((nb_iters, len(self.eval_op))) self.dump_n_eval(outputs=None, action='init') @@ -293,7 +285,6 @@ def __build_eval(self): scope=self.model_scope_quan) for node_name in self.unquant_node_names: insert_quant_op(graph, node_name, is_train=False) - self.global_step_eval = tf.train.get_or_create_global_step() vars_quan = get_vars_by_scope(self.model_scope_quan) # model definition - distilled model From e120202b1891c3b23e25e0487164e5a87e7a6bda Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 21 Jan 2019 15:52:22 +0800 Subject: [PATCH 093/173] add a unified conversion script for *.pb & *.tflite models --- tools/conversion/export_pb_tflite_models.py | 391 ++++++++++++++++++++ 1 file changed, 391 insertions(+) create mode 100644 tools/conversion/export_pb_tflite_models.py diff --git a/tools/conversion/export_pb_tflite_models.py b/tools/conversion/export_pb_tflite_models.py new file mode 100644 index 0000000..4a1f366 --- /dev/null +++ b/tools/conversion/export_pb_tflite_models.py @@ -0,0 +1,391 @@ +# Tencent is pleased to support the open source community by making PocketFlow available. +# +# Copyright (C) 2018 THL A29 Limited, a Tencent company. All rights reserved. +# +# Licensed under the BSD 3-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Export *.pb & *.tflite models from checkpoint files. + +Description: +* To export compressed *.pb & *.tflite models trained with channel pruning based algorithms, + set to True. +* To export compressed *.pb & *.tflite models trained with the learner, + set to True. +""" + +import os +import re +import traceback +import numpy as np +import tensorflow as tf +from tensorflow.contrib import graph_editor + +FLAGS = tf.app.flags.FLAGS + +# common configurations +tf.app.flags.DEFINE_string('log_dir', './logs', 'logging directory') +tf.app.flags.DEFINE_string('model_dir', './models', 'model directory') +tf.app.flags.DEFINE_string('input_coll', 'images_final', 'input tensor\'s collection') +tf.app.flags.DEFINE_string('output_coll', 'logits_final', 'output tensor\'s collection') + +# channel-pruning-related configurations +tf.app.flags.DEFINE_boolean('enbl_chn_prune', False, + 'enable exporting models with pruned channels removed') +tf.app.flags.DEFINE_boolean('enbl_fake_prune', False, 'enable fake pruning (for speed test only)') +tf.app.flags.DEFINE_float('fake_prune_ratio', 0.5, 'fake pruning ratio') + +# uniform-quantization-related configurations +tf.app.flags.DEFINE_boolean('enbl_uni_quant', False, + 'enable exporting models with uniform quantization operations applied') +tf.app.flags.DEFINE_boolean('enbl_fake_quant', False, + 'enable post-training quantization (may have extra performance loss)') +tf.app.flags.DEFINE_integer('fake_quant_nbits_wgh', 8, '# of bits for fake quantization of weights') +tf.app.flags.DEFINE_integer('fake_quant_nbits_act', 8, + '# of bits for fake quantization of activations') + +def get_meta_path(): + """Get the path to the *.meta file. + + Returns: + * file_path: path to the *.meta file + """ + + pattern = re.compile(r'model\.ckpt\.meta$') # file name must be: *model.ckpt.meta + for file_name in os.listdir(FLAGS.model_dir): + if re.search(pattern, file_name) is not None: + file_path = os.path.join(FLAGS.model_dir, file_name) + break + + return file_path + +def get_input_name_n_shape(meta_path): + """Get the input tensor's name & shape from *.meta file. + + Args: + * meta_path: path to the *.meta file + + Returns: + * input_name: input tensor's name + * input_shape: input tensor's shape + """ + + with tf.Graph().as_default(): + tf.train.import_meta_graph(meta_path) + net_input = tf.get_collection(FLAGS.input_coll)[0] + input_name = net_input.name + input_shape = net_input.shape + + return input_name, input_shape + +def get_data_format(): + """Get the data format of convolutional layers. + + Returns: + * data_format: data format of convolutional layers + """ + + data_format = None + pattern = re.compile(r'Conv2D$') + for op in tf.get_default_graph().get_operations(): + if re.search(pattern, op.name) is not None: + data_format = op.get_attr('data_format').decode('utf-8') + tf.logging.info('data format: ' + data_format) + break + assert data_format is not None, 'unable to determine ; convolutional layer not found' + + return data_format + +def is_initialized(sess, var): + """Check whether a variable is initialized. + + Args: + * sess: TensorFlow session + * var: variabile to be checked + """ + + try: + sess.run(var) + return True + except tf.errors.FailedPreconditionError: + return False + +def apply_fake_pruning(kernel): + """Apply fake pruning to the convolutional kernel. + + Args: + * kernel: original convolutional kernel + + Returns: + * kernel: randomly pruned convolutional kernel + """ + + tf.logging.info('kernel shape: {}'.format(kernel.shape)) + nb_chns = kernel.shape[2] + idxs_all = np.arange(nb_chns) + np.random.shuffle(idxs_all) + idxs_pruned = idxs_all[:int(nb_chns * FLAGS.fake_prune_ratio)] + kernel[:, :, idxs_pruned, :] = 0.0 + + return kernel + +def replace_dropout_layers(): + """Replace dropout layers with identity mappings. + + Returns: + * op_outputs_old: output nodes to be swapped in the old graph + * op_outputs_new: output nodes to be swapped in the new graph + """ + + pattern_div = re.compile('/dropout/div') + pattern_mul = re.compile('/dropout/mul') + op_outputs_old, op_outputs_new = [], [] + for op in tf.get_default_graph().get_operations(): + if re.search(pattern_div, op.name) is not None: + x = tf.identity(op.inputs[0]) + op_outputs_new += [x] + if re.search(pattern_mul, op.name) is not None: + op_outputs_old += [op.outputs[0]] + + return op_outputs_old, op_outputs_new + +def insert_alt_routines(sess, graph_trans_mthd): + """Insert alternative rountines for convolutional layers. + + Args: + * sess: TensorFlow session + * graph_trans_mthd: graph transformation method + + Returns: + * op_outputs_old: output nodes to be swapped in the old graph + * op_outputs_new: output nodes to be swapped in the new graph + """ + + pattern = re.compile('Conv2D$') + op_outputs_old, op_outputs_new = [], [] + for op in tf.get_default_graph().get_operations(): + if re.search(pattern, op.name) is not None: + # skip un-initialized variables, which is not needed in the final *.pb file + if not is_initialized(sess, op.inputs[1]): + continue + + # detect which channels to be pruned + tf.logging.info('transforming OP: ' + op.name) + kernel = sess.run(op.inputs[1]) + if FLAGS.enbl_fake_prune: + kernel = apply_fake_pruning(kernel) + kernel_chn_in = kernel.shape[2] + strides = op.get_attr('strides') + padding = op.get_attr('padding').decode('utf-8') + data_format = op.get_attr('data_format').decode('utf-8') + dilations = op.get_attr('dilations') + nnzs = np.nonzero(np.sum(np.abs(kernel), axis=(0, 1, 3)))[0] + tf.logging.info('reducing %d channels to %d' % (kernel_chn_in, nnzs.size)) + kernel_gthr = np.zeros((1, 1, kernel_chn_in, nnzs.size)) + kernel_gthr[0, 0, nnzs, np.arange(nnzs.size)] = 1.0 + kernel_shrk = kernel[:, :, nnzs, :] + + # replace channel pruned convolutional with cheaper operations + if graph_trans_mthd == 'gather': + x = tf.gather(op.inputs[0], nnzs, axis=1) + x = tf.nn.conv2d( + x, kernel_shrk, strides, padding, data_format=data_format, dilations=dilations) + elif graph_trans_mthd == '1x1_conv': + x = tf.nn.conv2d(op.inputs[0], kernel_gthr, [1, 1, 1, 1], 'SAME', data_format=data_format) + x = tf.nn.conv2d( + x, kernel_shrk, strides, padding, data_format=data_format, dilations=dilations) + else: + raise ValueError('unrecognized graph transformation method: ' + graph_trans_mthd) + + # obtain old and new routines' outputs + op_outputs_old += [op.outputs[0]] + op_outputs_new += [x] + + return op_outputs_old, op_outputs_new + +def convert_pb_model_to_tflite(net, pb_path, tflite_path): + """Convert the *.pb model to a *.tflite model. + + Args: + * net: network configurations + * pb_path: path to the *.pb file + * tflite_path: path to the *.tflite file + """ + + # setup a TFLite converter + tf.logging.info(pb_path + ' -> ' + tflite_path) + converter = tf.contrib.lite.TFLiteConverter.from_frozen_graph( + pb_path, [net['input_name']], [net['output_name']]) + if FLAGS.enbl_uni_quant: + converter.inference_type = lite_constants.QUANTIZED_UINT8 + converter.quantized_input_stats = {net['input_name']: (0., 1.)} + if FLAGS.enbl_fake_quant: + converter.post_training_quantize = True + converter.default_ranges_stats = (0, 6) + + # convert the *.pb model to a *.tflite model + try: + tflite_model = converter.convert() + with open(tflite_path, 'wb') as o_file: + o_file.write(tflite_model) + tf.logging.info(tflite_path + ' generate') + except Exception as err: + tf.logging.info('unable to generate a *.tflite model') + raise err + +def test_pb_model(file_path, net_input_name, net_output_name, net_input_data): + """Test the *.pb model. + + Args: + * file_path: file path to the *.pb model + * net_input_name: network's input node's name + * net_output_name: network's output node's name + * net_input_data: network's input node's data + """ + + with tf.Graph().as_default() as graph: + sess = tf.Session() + + # restore the model + graph_def = tf.GraphDef() + with tf.gfile.GFile(file_path, 'rb') as i_file: + graph_def.ParseFromString(i_file.read()) + tf.import_graph_def(graph_def) + + # obtain input & output nodes and then test the model + net_input = graph.get_tensor_by_name('import/' + net_input_name + ':0') + net_output = graph.get_tensor_by_name('import/' + net_output_name + ':0') + tf.logging.info('input: {} / output: {}'.format(net_input.name, net_output.name)) + net_output_data = sess.run(net_output, feed_dict={net_input: net_input_data}) + tf.logging.info('outputs from the *.pb model: {}'.format(net_output_data)) + +def test_tflite_model(file_path, net_input_data): + """Test the *.tflite model. + + Args: + * file_path: file path to the *.tflite model + * net_input_data: network's input node's data + """ + + # restore the model and allocate tensors + interpreter = tf.contrib.lite.Interpreter(model_path=file_path) + interpreter.allocate_tensors() + + # get input & output tensors + input_details = interpreter.get_input_details() + output_details = interpreter.get_output_details() + tf.logging.info('input details: {}'.format(input_details)) + tf.logging.info('output details: {}'.format(output_details)) + + # test the model with given inputs + interpreter.set_tensor(input_details[0]['index'], net_input_data) + interpreter.invoke() + net_output_data = interpreter.get_tensor(output_details[0]['index']) + tf.logging.info('outputs from the *.tflite model: {}'.format(net_output_data)) + +def export_pb_tflite_model(net, meta_path, pb_path, tflite_path): + """Export *.pb & *.tflite models from checkpoint files. + + Args: + * net: network configurations + * meta_path: path to the *.meta file + * pb_path: path to the *.pb file + * tflite_path: path to the *.tflite file + """ + + # convert checkpoint files to a *.pb model + with tf.Graph().as_default() as graph: + config = tf.ConfigProto() + config.gpu_options.allow_growth = True # pylint: disable=no-member + sess = tf.Session(config=config) + + # restore the graph with inputs replaced + net_input = tf.placeholder(tf.float32, shape=net['input_shape'], name=net['input_name']) + saver = tf.train.import_meta_graph( + meta_path, input_map={net['input_name_ckpt']: net_input}) + saver.restore(sess, meta_path.replace('.meta', '')) + + # obtain the data format and determine which graph transformation method to be used + data_format = get_data_format() + graph_trans_mthd = 'gather' if data_format == 'NCHW' else '1x1_conv' + + # obtain the output node + net_logits = tf.get_collection(FLAGS.output_coll)[0] + net_output = tf.nn.softmax(net_logits, name=net['output_name']) + tf.logging.info('input: {} / output: {}'.format(net_input.name, net_output.name)) + tf.logging.info('input\'s shape: {}'.format(net_input.shape)) + tf.logging.info('output\'s shape: {}'.format(net_output.shape)) + + # replace dropout layers with identity mappings (TF-Lite does not support dropout layers) + op_outputs_old, op_outputs_new = replace_dropout_layers() + sess.close() + graph_editor.swap_outputs(op_outputs_old, op_outputs_new) + sess = tf.Session(config=config) # open a new session + saver.restore(sess, meta_path.replace('.meta', '')) + + # edit the graph by inserting alternative routines for each convolutional layer + if FLAGS.enbl_chn_prune: + op_outputs_old, op_outputs_new = insert_alt_routines(sess, graph_trans_mthd) + sess.close() + graph_editor.swap_outputs(op_outputs_old, op_outputs_new) + sess = tf.Session(config=config) # open a new session + saver.restore(sess, meta_path.replace('.meta', '')) + + # write the original grpah to *.pb file + graph_def = graph.as_graph_def() + graph_def = tf.graph_util.convert_variables_to_constants(sess, graph_def, [net['output_name']]) + file_name_pb = os.path.basename(pb_path) + tf.train.write_graph(graph_def, FLAGS.model_dir, file_name_pb, as_text=False) + tf.logging.info(pb_path + ' generated') + + # convert the *.pb model to a *.tflite model + convert_pb_model_to_tflite(net, pb_path, tflite_path) + + # test *.pb & *.tflite models + test_pb_model(pb_path, net['input_name'], net['output_name'], net['input_data']) + test_tflite_model(tflite_path, net['input_data']) + +def main(unused_argv): + """Main entry. + + Args: + * unused_argv: unused arguments (after FLAGS is parsed) + """ + + try: + # setup the TF logging routine + tf.logging.set_verbosity(tf.logging.INFO) + + # network configurations + meta_path = get_meta_path() + input_name, input_shape = get_input_name_n_shape(meta_path) + net = { + 'input_name_ckpt': input_name, # used to import the model from checkpoint files + 'input_name': 'net_input', # used to export the model to *.pb & *.tflite files + 'input_shape': input_shape, + 'output_name': 'net_output' + } + net['input_data'] = np.zeros(tuple([1] + list(net['input_shape'])[1:]), dtype=np.float32) + + # generate *.pb & *.tflite files + pb_path = os.path.join(FLAGS.model_dir, 'model.pb') + tflite_path = os.path.join(FLAGS.model_dir, 'model.tflite') + export_pb_tflite_model(net, meta_path, pb_path, tflite_path) + + # exit normally + return 0 + except ValueError: + traceback.print_exc() + return 1 # exit with errors + +if __name__ == '__main__': + tf.app.run() From 51b31ff2addec29947a3ac88c36cebed7a72ab8a Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 21 Jan 2019 15:57:37 +0800 Subject: [PATCH 094/173] remove unused FLAGS --- tools/conversion/export_pb_tflite_models.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/conversion/export_pb_tflite_models.py b/tools/conversion/export_pb_tflite_models.py index 4a1f366..4b5f756 100644 --- a/tools/conversion/export_pb_tflite_models.py +++ b/tools/conversion/export_pb_tflite_models.py @@ -29,6 +29,7 @@ import numpy as np import tensorflow as tf from tensorflow.contrib import graph_editor +from tensorflow.contrib.lite.python import lite_constants FLAGS = tf.app.flags.FLAGS @@ -49,9 +50,6 @@ 'enable exporting models with uniform quantization operations applied') tf.app.flags.DEFINE_boolean('enbl_fake_quant', False, 'enable post-training quantization (may have extra performance loss)') -tf.app.flags.DEFINE_integer('fake_quant_nbits_wgh', 8, '# of bits for fake quantization of weights') -tf.app.flags.DEFINE_integer('fake_quant_nbits_act', 8, - '# of bits for fake quantization of activations') def get_meta_path(): """Get the path to the *.meta file. From ad4679dc314849a044c1997d3aa1a6f3f9b25794 Mon Sep 17 00:00:00 2001 From: jiaxiang-wu Date: Mon, 21 Jan 2019 16:20:30 +0800 Subject: [PATCH 095/173] change data type for quantized *.tflite models --- tools/conversion/export_pb_tflite_models.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/conversion/export_pb_tflite_models.py b/tools/conversion/export_pb_tflite_models.py index 4b5f756..cb5dfb9 100644 --- a/tools/conversion/export_pb_tflite_models.py +++ b/tools/conversion/export_pb_tflite_models.py @@ -285,7 +285,10 @@ def test_tflite_model(file_path, net_input_data): tf.logging.info('output details: {}'.format(output_details)) # test the model with given inputs - interpreter.set_tensor(input_details[0]['index'], net_input_data) + if not FLAGS.enbl_uni_quant: + interpreter.set_tensor(input_details[0]['index'], net_input_data) + else: + interpreter.set_tensor(input_details[0]['index'], net_input_data.astype(np.uint8)) interpreter.invoke() net_output_data = interpreter.get_tensor(output_details[0]['index']) tf.logging.info('outputs from the *.tflite model: {}'.format(net_output_data)) @@ -372,7 +375,7 @@ def main(unused_argv): 'input_shape': input_shape, 'output_name': 'net_output' } - net['input_data'] = np.zeros(tuple([1] + list(net['input_shape'])[1:]), dtype=np.float32) + net['input_data'] = np.random.random(size=tuple([1] + list(net['input_shape'])[1:])) # generate *.pb & *.tflite files pb_path = os.path.join(FLAGS.model_dir, 'model.pb') From 62f0a03d76476f3b478c04bbe5615313cd8b5623 Mon Sep 17 00:00:00 2001 From: Jiaxiang Wu Date: Thu, 24 Jan 2019 11:43:31 +0800 Subject: [PATCH 096/173] add QR code for joining the QQ group --- docs/qr_code.jpg | Bin 0 -> 60338 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/qr_code.jpg diff --git a/docs/qr_code.jpg b/docs/qr_code.jpg new file mode 100644 index 0000000000000000000000000000000000000000..644fb1c95ebd195109506d63f3d7fd89ed7cb18a GIT binary patch literal 60338 zcmd3Oc|4Tw*Z)0aFS2FLREQ6uY{@oM_K&oDkaa5Ak_KfBBa*U4X(2{Ql$gpEVeCt? z%M@jdu`ijijxo#c@_C-;`>npe<$0dpKhNzo%sq3@J=b-ebKd8?&p8*?7uG1W)6&$! z6oSDZ$Q=AYEFyH)B*fbTf~>6}B?yAJAvTy0#15Xpzz-DX4Q>1TIRxzoe<6rH2lmf* zbJ+glT{h_)_J2Gd-@1^s0O_3cLR~{$^+NgW(NH}I=^VAN-nMl&aQyvT{_m$d2X-fZ zO@ZERn~CCiG*3Mm$*O|{x!K0qq}gHnAT~i5yC96!1i`_GIADJte}4@8g0Zo0E(Ut z>b1b2;E>R;n47V;;%?uGPrG+N{lUXWk29X-KF@oRU-0tPyOPqf@{0GBRSk_z%`L5M z?H~L42R;wtzkD4cj*-SECa0!nW+_X{E30ek)D7C!yufTN4-D#$3ps<;VX-q>xWs8Opd_YX>2lr__W%a1sR3~ zGd43>P|7J5v||U!W@EV@xA=`f4~sHveO9Z(f^OZS8Je6#q$Ud#~uebjHn6D!l)53y8=SJlFSA05rL~i}KcIT6Sn>eTb=wtKl zQJ+~yNby-br1+|$zosa0{ZN#n;`cfO^_di9+gJ(taMKcz3q@aeO7Md(>| z<`^zT3(fZB%eNUpV`Uff>4rcLNO7B+k9;U+I%Tm%Y$z}9{$NIk#bmO!vUDZv3wz?n z(8(yn_s`-ty2lXBi$WpCnl-$_+-siPHIu#~c`+5ETB-j!9$^X5)c>2Z|L+UvjlS}$ zZ$8;l{i_LOCK*3QjyRigpX^fi<57U$DP|w*PFcmU8Rxe@H$naJ!TZ?8>_XBF|D@vV z+7;N`fPH%gyfsB^Htch9mO)We^5mqV^Xq_He6q1RpPqYiB`eSALWUs^M$5SWv2JH?!do3z~jhIHFw7 zg1k^0Lh~$WUKY;0lM!X8g9QK)dx)aIyn);}gaUT0+|hapnLhv{r_ldMxr12YWkKx; zXyy^KUr&+2r;Om-EU4iT^&Sf{!qOjA08F}%{`EY4Jy7Id?qsVaaFdDXoA(^RmJ(D) zhO{4;E|2;a#K?`BThy(Hu6G8w8iIk#IqR6ip+Xx!ZgIEzpW{ALsrxE<&ZezZP%`?Z}Zkkkj zSTiqMEUTT03q<6pejt-tTqXO5{~DY(?7sZE0NHtG;|dq|I5T}!pbovU^9-~01v6ct z6}h3XjK()1XrfTKiR)u7piZ|R9yGhk`LOE9U%D7WSo-<9eiv7;8rZ%X0j5zoVs$}` z8Kua+Olpc0qHx{Jk0&(kB6x(}ui7!aanjGGtur~|58D(onv}#<}t2uUO6gT9J@V3tYwLz#WG>%1hG~9@D8( zm;K*(0CDL+WKN%Vj3+E;Tm>vUo*TC!i~KVrfoZ}pW7c@Epk9wC9rW*CYfml2dK=19 z5^2Q=hcTBa$DgjfyZ9<9#7_WuX*phz9c4uR@pdc$m|iEpK&kU(^~)nKmx|Q`#`&3L z!@rwN30atFhTFWBxlt5RXas+h)E}(sM?hvY;EMSrEl`7GA1APcFx;2>=B%?Qj(~+I{%kbV9kXz% zs2YfdoI3$-;Och!oc&B3JW+JrKjeZ>M0PlK|M!MJN&Y%roc^vAq{R&(EgX&feoN`A z&DQ)_(3xU%jW0b9dxf%2zavRtggB$8ON`)IW$P6U0vigtW}9h1np4%Pwkvv)~|MVD9I|C=@%st$bCZS z9&Yp~fVN9H0^uy^rgA-QWm`WBBJWaT?yEJY5?;&<*(atATb17$l=T)4^?TOxA;Iv3 z2rOIq0|;&J-eX1q*|6a@ZOvLF{iv&qyJr{zZ9Hp<8%XVi&D=TM3%$S`frkay zKuT+UEEyH2fNyzqB~e9kBhu_)*1XtaXi~pnI`Jbbg-!L{tIHOa(WJ0>nNH1Tl0x-6vpux8ldeq6t>*A+<1AMY8*ESW!_}LS z9P@Pc$Gb~|yYPuIyq>XgN&@H3O4~mPHzViA&Q}W*x_tH4K4~p2VIffSz<9fZW z_2)~zFD@-cERj8-WW^o48wnj z?8Pkc^daV|aWro1W4ay-lH~*|f&|DueIW{AgeLNmQ4AQMfOHN3GPkkGwd&q$lAg)M zDw@L~MNiK@-KTTP&B$_l)+=bTb`Bd?dj#Env=T%t2hr1PQL5qF0AwyQzwc-IATQyY zkubzKz|sl}Tn2uq6t?mBCA!UYidltY-t?KlM$tH93aODyUwfOf57#~n73u%6m;J&M zwz^UKPgOrTzgjFK3pegKfSy}sLEc5sHH0u-g_s}!o2>}awY@KFCMlYx9MU`RSj4a1 zSlYfRkft{V%qq7L9|p^uADgia@BFYvjEVS`xU!1n0MLO-VaDXMAQ*bn7Yak(&{rTj z;W%f+>>W)K`#QXZ_1aNV8i!7$SvaTp)by7ch?%JlStpe}z6bjxD0lsBzn)Q-#wqG` z3rYSs_h)y)7vDDA-N+33@#WByizI~O7*j6%I0`!EuJk2=P_CJfa`@x$L@eUlMIzS=SL|?y6j%_5rxxK;M!UuSUtqK4G5UY=BFHeRcIc1ly}94Mwe@wVBz^c$ zdEis0WSWJ3n$2mSHmPSG>s4Yo>5!nG?EDyQ{7h9QQ*w}O^D5p(}t~97dTdd7QzZU4#1r^f||h)einr| zv*!xgqqz;ZwN~QlP&>UG<&$4!G&5>4t>jF$ExmS++VSLytPYlwf}kf>-M*c1S@?34-yv>aq!-d?41yii|zUHEBDMHG&CP3+|~FZtiG%<~Wm!^2Dpp;u+yp z!2yekyAapuNXH4E6&wgM?pjYV`~ZnH*4lV#BsGMNrCSv`F-olZRg0gtBh|%S@UjIT zqt%taFn5GFH4WJ$ja7fv?D%kYn^25gPnD|qsC$$&{&hI;ZmArIxmnbVYo`%M33PjfuhS;m2+n&qeIh_@Yu}slmnk{%TYcpvlxhm=I8E{E& zS_JNz{hCih z=sg5W^BF*UA~yIgv7prkH3m1nUuGXz05nJ^U>$HP4msO#R2zL^u&NybbRHn@cx+1T zp-Z}vg`|XpPW3V60!c-qGzI?Wu~1@SvT}!4j=&`<4{$mJMwK;vu=X=HzLpK$w$6gC z+(T<&%RmwUAV%GY=JmNa>s|Ch%W*-}er^yzl(=q^1AgG^GvpUi67sBNLW{${jMbcH zueyAy((X?ob?(jxG2~umFJei3hOj!tg32VL3^Ph7=vXbO`|sX-e*Ht?E>Sl5n&$=1 z5NMp@A{x#|e#o@kpUQ$33UJm|L1&(Zi* z95d080mpXz_Ck(8Ec5@ylIDa-Ncw%>V~jS$ilrT4o&lDu8pwdC#!O6(GVH{DvH;(P z*lgXv&wP)(V8a9+cm&ZECA&pXu)Rn&Frak68@?Fg!Q@mT1MR1=H_+pjTNn%n82s!( zl)m!+355SkBnbcC187L+2=)z*_>>Mjli^_&wEhW64~R0*Lz2aAR-X#1x>a*Q)ziGm zI5ACir;t&@!QB?G5!5pTe#Ep8Zk>i=9&tv|H4!&i&=g4dGB*-fP%_ZT9Rfte1_dBp zEeNzp_hDbr4cSzGU{2{E<=u~_d_)j!!T|*3Sh66(430YCk0bXmf%;;kSkUIA+E)JS zx4WL?dQA;(kdY?mWQ8NVR7_g>Rd*ADJ}qp{`zL0nJGBy)2s&ABXSRe`Wh1FWO}-uv-ypaFqY7U$Nw|u1t%fyly&n_BfFN3E4523 zf@Fhui_*VLiTjzIRGtzyRG}Zsz8LLd`&jPWYyE=IZKElbuEfpb=(fW1Ah2r$@^44V zIRKY3j#%wQXF(A_C3vqBTN!NjOcrFll?-f(-0L$`(P$7Ft~VOmm3gj5zCPC0UB>6Z z_TfDKZ?V#z()M^QN^tAqhKw7qd_~-fLJofl5!r;4pvJX4sIn_5&}uGC{phG6?UPnu zK5#MOyU6~VfW)g^l73zMJJKLwm%C*3>w zyQqLP45-Kxxc6@!d|b^QvA+}lIz(RlrL0j9-S0Mejr#jOry{#}c~p7C zhdO7B1ZN0*be`vgV=ujFmzw>rjF}TU?=PJ18LfmrDyF?F>vSa;_ou4LtL|*> zc8Vv!m|g#e?O2q`SGpaAvHDHh-Z58SqwQ^5?O(D1mGu%l*~Zv{-_k0d zD+_wc%TUV^xLb`LQ}HC?ZaX!)^1jy%%6==?u(NOvVSn)MzPKy?bB;Kk+P&1)rpQAH ze-2_hn@f%ru(5-H z#(%=mB(Qx|^Z`72{BQ)&D0M&;JfaM97+HvMo>u^dK7usOT($wo^P;&fg=J?d-1Fb2 zmmAJrI)3%mw#WoW)kZX#)d*cbXB(AdY!1*xYJki9_adTFcEc^&{iY}x zaeHsZ?2m7c9j#8j*tcIVisPfUkv_#ptac9-ZASIUK{r^8-sa3I2yN$jBJteNDcT|E zqf2PrLI!z}>WmNs>#a&ypM*2_)xq822Z_23i*55zpHc{x zwwGl=+bLt+8_YKV1M>g|UV{OAY#>sBYc>u7Ab1eukwia+#5B#R_u6P_3_+-@DF=qr1 z$7g_KS=<3D?X>6FHj6KSIMQ1{IhSg~3`YSr_6Rs2P#FBh{}}ClNdIFo4!**J+++a1 zD4oj`S#M0CZNoB{nZU`}14s3nm;3FGP!N)wUn`C7kw-7C5ab4e(GMDA6FO|NA*I3XskPl1u>Sz{{A*ecmsNwgl0kE&A>A#vS%@#hj}5^ z)>%-gPdy#zqU;M`qUWVRd8X`pxIH!QId#Pw_tnpa>q|9*8FC79xT8h81buDtJ#rhm zZDuP5F$E6COa+Qe__wm~R%uuS_H+9}wenhw*}eJ)@$qLt>8iC(f(1R$u&?R`DRF{F zB@4>70mI2K>;!0`k6~_T(XN_6k!}$gQ_N5VW65h%tDC-r-WviGL71_ZmUtJIRHmo1Pxw7_>FDD7|-M1>24Nb%ru% z@>_>U_zZ7d^j>}V`cO4#Eq*JG{}X67(95135duj4$X?`<+6-oOg+PPl2&Av1Z14d% z!-ue-mA%Nk)s3gWJD+%_Zlea3@>TFrM23u!!eA~{YI{3r+M)M;MD`VPS*p*%Ry5ja z9n3ryOjuo3W8U&vDgMQ=|B&zAYYnnz;-NMVY8+7H6zyZySuKc(41AS0Ziyd&O)-e-bK)zF zWCAHZOp6gL$l3sXHB|}cLq>!)+);M=ttd6Es8qfVMF4}C} zBxGtq?7(S9uUaSN^FMT@*OF@;pZ#ap$StO3y$1dGfP~;9k(j#x5+3~FYWojE!uDr? z@}AT`%*pH@hJ-r-7~Ln-Ka~ysPkUt?*-A9j{)g3S{)fH#tW3s3^p<~E?Wxhs>o-US zJ2JGPOWKusHuN<|hTyX(Ud4Yt5UgJtjJD~sfg%UhOF$aktqk8>*(wm#b{p&nKLkQs zt1&e?wUq^-#{gq4V0U9mvbVyQKejR%2!L?K!^ElPMGNW7e@O=Zsd~>k?CG z_uzeL>M&{lIcEjBDj^QJ%urxZAF`l3uZymZ1;9f?sWy=h5@OqjeK@4=Ch)EFX4%+WgPTldJ`0k{{rv^3A1ImsCoZ$cuQJpcbuVVrzTF!? zr2h2L9~bzf`9*{W@GwyNuOQ;-EC^Unq&yJ17yDSvr9KtJ*_t;|_jcDby;c2jV&Foq zAyxEq$l)>7n$eIe=X%pPonF|Xk4Pk++`yKCFE*WJP)>`ly_ zNZ3f`EhyV*G|^IShO6u^%ey@k_&}4*_Q3nAF?F&Vcnjj9;EdrP{kICAwYgUu zXn$Y0_ghWKxgDY1KX(m;Jfv%KB_pQNQy4VRQsSx>=(irM{=Fk=SWf_t(Fx=uaR2ft9#| zImr7w`MRJj&eLXt?GkQv2}9!u3vPyQ?9{^0))BOwIRcqzVvfE)h!b9;7yCi13mE^e z=VaPSmNM~hvR0Aq4ttvtvx-V1%G(Srd7HPFBBp6tiQ7cL?a45dEPZHYGXH9~t;T8CusdAl2z9rJv! zCm8z}L9_@r0eK8@5bn^CbXzE*0{Hbd-C<@W=oJA3{{Y>JA`l&zV6|hn%FvMo`rvZv zqaWYw6V)j7foeOA=i|3M zaevb7Z8v%jDcp0tYQ%9}tN{U=&OU-UmEz!i-|@ML{sX>_2VDV0Gp>5B!(_LXNEvcO zOa6YG#?gvDxKl1^j6c21=m1z~trduc>MCC}pCwF1-ebB(f|f5xv_ zF7qK0U17NDTP=gDLr!;g$#F_>I&?wpu1Berzz%vUU>`& ztb$$#iYBa!Zcoh`Lu{K{8}hjOpi`v0P(vd1afMjm=q>59i*1H4CEcvxGK3{?s7G_V3RHSU>l|DmO9pXT@m2H&s0Qf5?~LCA8!{3o=!@aG#`P z++vg?aQPYTFpj81F=KF`(ByqKOr;d6W%VgQ9GwVsZV-pW=LnqrwV*P|qL!Yh&kH?ZP# z_VSgngY(%VTJg5u)&CfphDmITiRl@OGPs31qVEofMq~-d7yz@@dIkS&AlS-k72*Qh zdZVuG#AKF_{_@x1_5<&ZCjMBEeBg0`NA6ylja(1c)c3#<;2cs3Tbytc#L>!&2&zdq zz&WErH9A)c@VUi4Esp$g`BW=AT>rdE(T_bP8RmKMB4%cP$*I4KGUmE#0)_1k7p9;A z?ZYzZNI?5Q8vkZtGkis~26(nR25zwx`1q51&Cqq+{34iy zEudB*7*L&yxPg3J?s)6zi#GTEV)^nQDXZi2H+-%tp~t@a5p z#LLK^A<HE)uoZ*VhjV3h`LD2RkRn zZA&$h&^(2FI;3_>Uu%DgyIh7#riN3K#X(aGc`k*C90B=X@?0|n5%jnbpk3bpk2qSb zLAv>lin8m^AplTdo9}k8=jrK={Tr`l$Z$%dU@SH*hhc(;;m3u z^Y4Bk?iY;Vr;KD=kU50yKy*g|ulbBF@&)mlG7(MVa6pghqv_U{>rou9l~KNu&63-d z_VB(kfTVe$2k{%5q_zBlnJSM5w7`ME=E=K>r71LK#ovS{cXgxww^vqRWZX~naI7NM{&&;{ z$%e?}Mj~pv_{l7#wya#H3klGLpeTe+VSgk)K(D?nn3=w#oJ(EJ|KeS%ToQMpB7R8w zt*Nq*5?lCJPVq2 zUL-)x#tIj4933jHF)Q9vL1`8TR31NWKk(_oX$yrkYv|g5eHXlRSK-Tw2H?Y{ z3ksi8K^!GVI71r-r_^b7q7p-jjH1U&sxd;G(9?O?F3Zh|hFSx%OMdK1p;MM?-;oRN zD$Gppv%P*>uqWlD7yCe=vP&P{xfgZK^KO&co;0D+i^Z$TPyC;Fyg?H$P{o)rcIfq0 zFVM@EFp8w@067Q3=Ny6EAUyU5dX)1Cw?2xdTtheNd;wJ61Gz~?P^;F|sAp?Az`LI@ z42C1(r`xbkVDG?ri|Aa>hiCl=HD9y+`3DpCtAkvN+l=cWq*J(aZ3$&C&gTVH*YYVo4UT3*_Lv1BixCYC0xJgSvae`c2BAjez91zJpE5;nQte#d?`4l#rH+VwV1!zyB6|XxC`ZB zY+wqz%NOsktg=swa`JA;MxTzpSGX~)Ork+FS!+fPN^?Pb*v4hw;axq?jT4(ZFT{3f z2RDaKHL2a!2%(Mjg}xr_pT9lJRv9teW>D91$l)NBy`|z{UX!H3aO=KXrh-BeXD-QlMz?J|176}Ax(j!-a2Zc*rKe=5F(O*& zb1O5?2L;A zi6JdP#x0u#?bMftM^_&mlC$TWIc}lwsN)TNg9KqL$CRfriP<|Tre!4Un!anp2?aZ< zttOAhK9oHoAv%E|7gq-Y5AM{xxdIxp?Ppv;jJ(GQPFn*!*9hd=>B`vM=$>86Oomp( z!xtgT^$)^)i?Rl>-XcAX(gCvO?8>jAjAe2tMiQ8vkz&KtwWXmYpUG0~-qLdi&aYpb z(tlHoYDBX8>K`c{^11V>UBl60lKa!%V}7O@)uf^@HTnT08_>f`-ArZ?ocwmH@5Q(f zu}Le1eZ_ZWD^h2IiHkc&rtjpLsSjDY2}#61Xbtz(!Jhp^y|K6H=uq#1kZ#+Wx9J|y zKDREW^4s3{u>-n>-WjQ2Lz;84@_udC<<%M{Q3G=SLpk@uWPrOm20FUlqwm=_+pc0-&|)EOj%3uA(k#o7SyXA$i1P`F@~E#m zqD$^j1RUf>U5AHTpBGT7hi!)U=vzKO;aUu)5Kcoh+S9k%_EOq`o>AL zG{$$8zem8W97hD^Q|1$OAF-f_$x!;vt0m%J7YD1&#^Cf6T>>Mr6I7@q*HB+unTD%a z+7yaDp0h)Ny4#z04JFk2N`E0QCgiR7(Ku}uRLy&0IdP%I-vRy0>et3o09sbz7$M)# z)8*JMUT8Mleau@QIVP19Y0gn{VrS_23fNn=)w;P9%Zb7l{lEd#xb~{8)TmERj{F&$ zrql7qke{E~Q}KdTBoFoBv2wADmMniCzJo?fPi+qv@42{M2bcza<{NzW45OD`xGaP_ z0oLg>=0^(H7;%aOB)n@x|MSz&9s@ExjZn;m7U`W5nMV#FYMj;R8c8V(P7rM4*jZ2l zHt$mQ(}FKEb^(_p@3*5+yAHSS9V$$N+Slbkk#3kN(bCF{0p&l!4ZwBrJ&yYyRjDSq z=jJ{(c_LURow!S|z?t;KQ?wg|G0kK(1}ESqi?etq->axl-`(GH@0@?U%l!KZ+w@z; zTs`s7GVV5-{~R^yz;S$$_nO*aDK~)+onL274^^xd!%e6>p1E_o9IIygEH4-Dd$RW? z)S|E>aS^mz3-hWS|U5pHT=u7J6bagiI_9+~)(Cm8K zlCbB%r&b6yYYhdKf{0P%-yi7z`VN|B_X9=O33MfN9AQc9y}v@ zs{0-YzI=$ZDBQolI=_C=Djt^4v z%}@a;+zz5o?n=}dSV+fjC;@^*KLjefgCJ|_>W7{#!)AgwlCZ%&O<4ViV(|EUEMh89 z*ZM~EoFeT0ikJJ#g!hS3B5eIS?;0ldjR{42h&I*g!zh%$*K$`Ar`qgi1uH?F=5mC{ zggE$;p%-p-9lnh1c|{?JP{kTt#mN((CVfqcdjU0n-n(e>baQU1=`pA&*`PTb_%uGZ zF*t)0;ApMeR_YOk`>AWdHEB~9EM5Ie;B!HxeELGQ)ae0i+4Y$~70WYPO|93G{rIHa z$~mrg&(8s_$e9l&R0sek9-@u?9p`PF0Sb_!??hY_*UrklB;@3DdtXGnNK)sMY-6nn z0=e99FYrYdkeicRLAD|N*J2A6D9u|yh+;q_M*u5E(OSKNA$QkR-NK;Ip?M~9>wz_L2y|QL6i9HZ->>T`2#%MJx-{I1TdcWp( z)4f{N%f6}nM(1N%#Wi7ICs`!AA)guw9B>G3@g?BI2;v#KK8Virk?R8(kc|rGa*_c7 zU1|*2dx|Xpy?U-wln?bxNpe@#i_WJG@`at6@e@|N>lF8dH{oxE1cO~qddym&GCj>u z4;osaGEUrGha+Kk~rk3aKHLYqJd}{Q8_H1}Wsc9>j&Oy=nJ& zr>=OK@3EZ3xJ;eB=qPFzHKAoh=^eQ>9tGkQ>uPnLqHl`!OC)bGc_V%GnEC}H894IU z7H>#ZJ_fpiBe2YPN02G`8Cq?ZfVD^N328OLm+jy)8@O2U?oDOJ(1_??`*fZK+|Ef? zfEh_=Kr1#wW-nqudKSHDS`DvJevO=vLQbcKD z4Lv@&c^hcsF;vu<1QQA;=Jkn0%GYnr8cHO8?)uvji_I6#^vam>fHa>Dv^~ux4~hUu zlO3W&0*&)K(;q;b?Q%-e>LywwY5dvFl7xkexwiCi+zRsWdE&vzI1xY_aNx9l#j)R?<8SidIBi7_22uzgVmIHtsK z6LLkUl)mMeeMUeH?C5g6p^UF2&1>pz*YcGVJ->R>BlQceH%y?=-!pL$9lw!7Bbn8F zIvajfiZ9}*!O_ollXB_C0(W;GgF@D^epIK?9mneIDteILO#H57YoFQmLz|617&hnP zd8^FeNcfQ#$}UH_OQUwW1PDAZn0t0>a%fx_rDV?Kk|+gFhuo-z=9x(H1T*2vob*G# zQp@hFoHw)nom-A;`@)zSjmI$(gcDk;Tb&b?V+Q=_9dp5syFVoyc<@>WXL$UbA;b!|xtXMVW;W5yY%&6TqpIZ9amI@!CS> zKtEz?7QRWvkxrXLxa)2B*+?f%7xU0>bC@Q!*7VE&(yB4}CIze?M9lwTQU{&DAPWml|6$VzVY=ybm7p&N4=A*iHyk+d5nFOikytS|Y<)hzBDfim=^R8|T__r**PnK70 z%{i5Zh8e+~(2XX*f8D4+KS9%NuLsAiJ_$cCw?tNQ-t4?arlj(&$Z%F~u3pz%wif|_@*x7$lcf;ry-VY5HjUvk&oMJUlc z*J3}ER`#u=qBH~2vz_P9ec3R}3xD_OZ0xDD5a=Y=AfV&DuE71~_!6c-`c>3h?e2on=0yra_FH>|4RiBV>9S$2FCYax~ z-2do)dh|h#E*%|B*i;zuCd!b{1>N9T)VX`Lw%=#MY20uolxwQ4ZrDYPu4x~Ko|X#U zXx*xe;aYi!1Ny>i*Lil0*_0g*I)1Diy8J}dI{Z50AqbDUUlZ0Ba1Buit3CIfjiqM~ z%AKr(`)4epr0J*n=zM5SOoWd?qm5X`b`w9&vG;nnSFlI>Z%9VG*B1o4MEBL6W#qj6 z@gv+UzpPr_lH&!B$pP51R9}=Xh+OXsc{UAO=<~gB?DV<%N%!#2x*m_&Rxf@f4srmM!icA4(4 zYL|bGLs-y>0Xa%}iCj?M zyRz}7or%U$DaYbhF<+^>wdc<2rqB+Cj1EtQXTCw|d`}#U5(A_MI}_IVAW!w_rFI{G zkxw~>=1Gk^b>^BQtp7rnX?AbW;+RY|YA@sE3v~AYVx59+i87S`HRbt`+~1-d$xYdo z(YzvLBQ>nAccEpzMA1Nr{04n}YgS^ccF5!{Kp zMO!HlT9@8{?SBuh+RWWJ$pz$Cil8U0Ay-E+U^mV9FX_`_1~Bw-6iAR^w*e>k4lhf- z%Ct1Sw~G0@R1^;-Ixa4?Tp~4 z6VFf!FWzsW?p1tLUfOW=xcWXB6y{4)QY87`7COuvoLM(tzt_rEB2uUR=Hv+h;|%H& zpHt=&Mux(}D;jUD+zj6rSJXQOCkhUo$MfU#gW5`EZ3kY&;cu2^8Jcb71{jc^F z=!RIbSRS4MaxL?q@gROxEt&udbKQBZ%-d<16wd}no0xDj&WE|7(Gn^`Zj;)i%4`$; zQU6%cW@<1JN3O;4SLV%c9+z6%hzX$2o*ei8k^wPfv0XM2=r#hN(0E2SGwITb4HfCC zzX}2j9D=!#OG`8TO9*V^=TtB(pNU8Q|FRd}1G9M@^b26w6eEm}M z`YqG(BN8%?a|HTm%3~=%e|Lm68T2L9QWS0;(5kWW98$IIn3uBOYC4JFfPysb)5D!8 zx89C+^5akThQB?4-?x8R%x~ya<_j}FbY<+$}J7W6KcDTYrm z@I^0zB!at<%;s;t5_vqtv!x2(Z0)MXK`m&ytQw=AeRKo$=tf}U#dT?lD^NCGu{zg> z$n{OI6E{a7UU|0rYX{SKsGAWO2b{{K4UA)1e4|6n+ZaPD#sk7J z8H$;ovad>oV&D7n+JM>)S$C`HY3-L@>;=PJ;fmz$MEwI9EyW7U8Q#K&+RK89ArfcJ zR3npvY!fAjo!@izV!d7Ek>q?>@S=zNkG+(p^qxRlZXgaE_fpRiA)I!II@@sHWcfh8v8IOwZQrP zirt8q8S;?eJ16}svRWip9x_N@8c?{*BnL76mkj%p`u1|45HQ`$Cu^;^S<<>H6jH|%ls#B69-FahO_2RwD@0~Hm@e_Iiu|95%xTS;Mgw<66<}GDB zQ!QMdydM4ffU2gk_S_v8cxMEOHz(0gxye`6{aw|qC23Ubph>rEz>`Ggz9@usA#pJo zbh;O#=Mv#Gs1Ql6tqz^@0U;rJQwbD?wuk7MQj4A|Bv#u`N^N|7hTdCUIxIN!-dW~* zXpVsTuX~rCRNE7M)rh;&LE#K_GdO{8$QRcq^~pNzVnxS|?vHx-3_t^Dl&K;t10;0j z3)mOJlxm(789diOjDW@i1M|+aVE^HG`1K-Ju*XFAR$0F9rpY_y{glc87yHx|t)&XO z8dfF@lurp5CeSlHHUr_cNL}tl&*il)B?GSdv$Z&*6PSBKVp`gLWy&w?8>qDTLpy-l z6>1nAr4#lay>$OzT#*;-OR%jeGW%jV@sJzCX3N?nG5=?H$J1q>ouzy9Pf!#R*QGu< zSZ72$+OV2Xy2ELvKY6S(Od*abNHuHJZIYaPq-xlU`rxr~Sgk9M-<0cu7kmHQZ%B`P zRyJORsYi0m4kdNH3{nlBd+F+}-EpJJ^nl-?{{2zEP6PfIfyM#o)nM}pq@rv zG~^*|B0p`K4dJeXe}uIeilr%;b6aGz&wL2_MC~y z=HCHD1pi(7+$w*9f1$zpdB;-8*kY_!ig?Hs!k@i@GG2>99{ly3&B-SEm>|*FYbUvy zWJgg=P=N0!iJ7T#eVwjm6GvF4AQ&5$R+oG5li=QaN!DDasJK_HV<--#PWNrf9t#@W zng5HoH;;$%?caw-LPSFLn2J)_%1#&&l0;-*rm~xmkllzv))0y;MIp;n_I2$0u1tin zX3vaujG6g7r_cSn?>pVU@BMvV&mYepy}-sN)?H^}q5_ z=i>O{r#450f}5RZpImdHI>}POkNA$y%-L~dl-qG>lc%5Sbmbot3hAMqYJ>t?WDtkF@rOJ=&7;id?W&j%ny$Wg< zcQcQ59k>n7(3*g$zA#_D@(@KG!MG&_Z`;?_YG*ricxCWjX>yZd9{8#3@*JEjhC& zY}*Uu;h-!=snG?fwz~3KQ0kILFQ+0eLP#Ab6*lxvQ#6bzb1Rc)} z_vXJK>k>;DTe~nyl+y?f738luBUf~;KP!58B++JK>XHfT<&5{%-Wi|@WI#a+fp{bu ze*NH#wd5o=gd#KGZ}4FWgKPX`ZXp=Yaxwk{|HCU7j1&MZ`{Vc2k?rUYNX!s&fwnFT zl@2>Nw6OKv8`@}`E`?{Wi*`Dq>eY8@@#Kv_J4(n{#LV*QTF?Y{m^$tW=A53Gw2TCY z(%YL)q1_Y!8t3jO(M=Hw@7`vwty+z_9Nmk|6UsxK_VK?u)1w~j{|WUV4pb2XbR^Bt zHTbabXN8~7N$j1rYS1*Ie5lRW8eH}8<63dnVE0X58UMbQvK1e5j6uiB6{n~%7n3{V zX8?cRvuH&#u(`$b5bc;;iLB1JXQ^*u(ps&5r2a36hhe91Z=ThAO&`f`@56F*%ve4r zc)nfa{Ae^iEYr7l3OZP`Y(en0R`Yx(f4^q;CMW|2Wx$=bs)TRIRh48^buUlm85zYZ zhBR8qX6U?fG@G(Me&t$p+CYXzpX;1ei|0kr{PA}sZ%aq-4F0EX&JwPhF!D;*Aw!8>{eAZ;djB9fiF#k8zYN7!u#x z2X?QcPFgnMV~z$~Y2Z$AmLD~3p6O_;7`cztu_LmNUFzYwMUFV7T(YJ?}TxfBVYdqoK!O5@yo#<46pceiFL@A#~OT1k(Ad z*fNSvHgJPLuymH4=tW0-2{sBixVv2qX5YdZRwdxa2qRIPUIx4s!^dq6*eAHR;0e!-cocFBVjQ@G&q*`6va8K`!iPKgbLc;#?ibl|tPIKwCf* z6s`h?NZ>&oyrzc#(Ow$QkHm=wuhtyR>c}=5Q|j!va?A4T1)4--F6b|eBY>kn^6sK& zknJgGz%PO@UKsc0D8#Yd_*QCOtFLp*15r7f_EIDJLR{o=`PCE%VLLlU>vWv$vj~Ob zhA$_B6^^U>X0$MeB?VFiD2a+?SMA+>nQ>Wlb$#2KYirzsGt8r8cx#V|-}v(Tx#ow- z)MwQ`Sv&#n6T64B1jU7CAO}L?3wGzzmkgAeqeY^>y-BkE5$}V_hfTfN1n52sB(?+q ztPWm5cCCiDe|2;xxoc5hYPkGaU-QFN_IyOqva=_vG`WN3>v21OmW)DQ`)2gB$I6EX zMYn{_M_kPOp3obs>roj!z;B|sLe);>o#+1E@Oe`nk|E?YX){v@!oDUA1i)IA17i?NVV@0@!#`__-bx&H$<($t zG2#o?X1pLI4>N-_|NiJ4e!oC?)mw3fwDO+|xmxi$Q%he%EQnR+f=k$<)jn#UaXt%_ z^vz}{}IB2+-eD7uLY0LiQN8?#q84)cJTCnn}-=vt>_GFfApM7#Ycaw#|{)%oX z=L=>=!Vyy;s7-OqnDyRw`i$N8*2t18u~jz?;;d*`!H zQUs9-*rPhV1=c^^tuJ2mu!jk}At3GwrqA4V%Md?;Ho$D6?iTUZ%SetaRL|>#+Ko{y z$Fzc3DyTtv$?GI@-2~0F{o%v&bHpY`CQ6p;m75;whEN4sQ=hzbyizXvUqVpl|`SfEbMfgu7@*iu9xv3|GQ`3@=i}Uefb0F59)X{_vW`=7~^Wq55%SgBmGr-?K0&La&Q-NFIES#;eb?xbN5r3I(~D!4JE*NM~^NIOezSe2SDmsrj(= z4kQ1iW5eye^gk=H(a81NrOZ3PNe^7Gz&@$5b)O8lw-ssbQEC6|?f2Q-CW*2~u<=ah zTJ$F=aU$J*J_Xzr!|-A~Dgil_B@N(}KfupGsLnual)>_#Zw4V1AcURmFc6^Dbx~?S zFAUJFu^-+T?JWLrec(FUvp?}(x|~4D4tx|!(by9w;&jQpqCR24ib~88q}%qHy22{S z8Y(t=^t=N0lh7SE6>iuM;u`s|7}(kaqk)gKl(axNmyiChR#K)a z_8^|}G5mz`$q@p?5o=Q&zr0%acx~TQm;G(3UH`i~I|!zNVCX>!=1M(ule>)6M=~%s5%=K$^+0Rm1#Ad%Ru3e&jo^u>i$c8D+TceCA1Cz`+g{~y2gOm2qKEE&6*m@IwLQS4@w(tp(LOV#pNdUqK& zL(%Nvq}0t&Jc;ZoPzcEo^mv~YATjsAiimomsYz<@yuHlO6~n(~N@BF{Mhu=j^rNAk zBpm-ofopnmk9+INg4EvaDuA?*;$H#v#?7!M%ISfctGr_l<-(pkd@ z4ybI4Oz!+_{CUiR>?i331?AW8{b9Iw=BkI<5d0RK5?V1@JJwd7Q{yu8EmhrBzw#3p zl-)*NdE8#fD)g13X`iz)7wGK9mdrJ^8DS^=M4a+?-bFzmCy2M+Fqmz`-uY*sCfq|g zs~Y{@yf<)N=gFq6JjHO4Jrd>6kEn7V(OEB_h(=e6$Xvo0peh*%bS>uCx>6Z;6<>AnoW;nN=LI*XQ_9MnwDE^ca7TyVcfYn$@z( zupsVN_T%iBJJbU&k}Lrc2_r3zxpI(pyGI?ndmUfB|CbHhvKtHd65da%$}uTsNB-+eadd+Z~HeNesH|u4%Kd=kG*Hkc$c*wcZ4@rcKEnq*kK-`>^6vCV`~wVN^olB zg&Qp&8CJR>!d`_fTjrx?imNcL8)P0sQRf#BTb&b{lE*1+- zk}wpC?B|(Zht9x_4o)y^`9es0;1jH8)`-*=6g3iJ#es6pn_l2f!2L0H2us4(Kz6M0 zPeDK)kwKmzEl__Cltqlf!u@p|ecp|q{d&Rfb8+h%96!wAe)<|0O*kZ@ogu?uaFcA;sUvIn#6@(jm4N+~mVK$OKM2v| z?}@+PR9{u&tNtyjTvYI=oQJ0#HPG%rju(kjcqBE!hv~GDNxHa9X8rx=08Rq6d7vf< zF*^=;8ZHfNjjLTB+>&2i75QZseB?#+=N7g^^^+b95cU-?&YA)vA98`MO(UQAw-h0c{5t#Q!fb9_7bFmLD)N>#?d z5qqW&Tm@Jg6u4$J_QB%NTvY0Byl+IP#W%KV18dUH_1eoG*0fMH+>J1Y>`e#BA6Tn8 zSwiZE?~DBMUrSac*n=lHq68fW+*#RTn=#eMCJm6oZPM?@Y0oL_VP3ewo$16T!glNW zmGhoAB^oX!d91g+d}JIF|2mT~b_7e&$OnxXJd7xzK>WG*TJ*iA+-a_mUo_s>CfGBt ziNnKU(!%-Qi;Vo-kKEDFr?-8uv5B%vmlTn>cI;@L1zF=f^&~U!Vaodst?wsDcb*dN zWi6&F<9;C0FPH1+Z@KWZNRd88A4;^kTrYxDm_bgx*@`6|H3!iSc-A~S@GxcO3Y=!x z4~?ocG0PmE^302u8Z<5izNGs37lfmRc}kS6fzvD|;X{-2!+2GWYjme1gZy^b*G2zx zih(A88z(z)>oX`GZOOhj)2HAUke8w3Nw9ZD^{b z?#q+8VDAQI-rT9M3RDsd7`Z)4h%5{wJPdoOEj-s!009rG#3Y+VX-xF^V{Qf$&r`-d zNL;UP8=0mMM4iC!L__@lg5)}JRu#^sgk}lrx?S#SA2*pfi2-1MLE85-?Z0d+-bYF3 z^y#z#0uL70GauA*4|#v-qVWay8^ualPsK8WMj$xm;kqDlN|Izi;pM&QD-V^Qc4RE7 z@R#GoC|b&0@6?9vjs@&d?P8p>9S0oEJY-lHk9<88U|vbm0fnz{Ob_9SO+0yEj=itQ zowPi#=IUIsY*$_VFz$ndF8WaQ^0t=?eRD9#`Y;B^&$V{a*SvzhEd>=lIBIE8qqj= zVTT0~2z@@Ux2$+Y(4c?zZpc;vl=loQv?y}R@_k@+P}Wq}^$9+Hdq*zuek#iie}YNJ z=3~WC#K_E9_L9WzwcqUl;NGpwcC3O%yGW7G9WE<3d3TS;gXSgME@mQnCw91o8br!; zo=`4AlI7!33a>}J+>mt^tBjaJ#75su!gd6xOY{1l0p0$0e?goruq_YMomxJ#yqqiU z)g7z}4pJt5COA&Yi)6-a`aQObrk7KUr>h0t;WwLr8bZ$Ufn;Kz{@^P6qaa1#wb*dj zr=$zWY3MPN7VBKIGbOSthxm>v$$Tyo5 z`tWe5WORdj6j=f>_Q&zrz(PqfwKztjN8{sd9%3V3k9;i}58`)UPJwh*5Ta)yt9?XW zJQ2^&yf)>d=d!&Q%*@x~CXLko?5wL_I&?c&`s+X=*E&^P@GM!#&Eps69nGrZ(%XgVFRlVY|Ct( zeVF)FaH$|4nEqVq;rwSX#+t*;p9cz`E4PslRUp|qOBl~wpz!0R-={}9C4H(*Y46~w z7qNL^zkcX5<#$}ve>C=Ac?&dOA{exA_>%Bai%vG@q))Nc$=m`-gr5`@QApi?Za?qa zuX~)Ep@2bLT!yky|&OpSW~*&o1a9B9NO>EDefgy$1=w&X%L5WbBzinM8ScFC zc=rLPYhtl{UlDP-#5LB;=yU$aBFkbzD+Yit-Os%EV046@3sk9gs{Yvo``fMlhMdYD zn=mugI9rnj$#s{@l6oEHuioPl<)z&ploL?IL|t`t^Ss#L%&RS6e*MqZ9O zCAe05vYhlXT5@qez7fnk#=J*u3G8?PhP;c!W&M7E8caD)4easAQX{dI&%x?P7wvZ0 zkDsS+PHMfeVZ;=BFUg<7*%gCOYRy{C5YX1D1wo2F0*;MgNQ1%Hau zb;j-82>vM(p>FyphT#)=IsNL-TGj|ip{I|%O|W)H))TFIiA#FAAGFx7BSyL9IJcWK zhz~MmtVI9dxcQw>E-=-QH_hN?aY9E;btq@+O7;QsTyK>i zey%NhKSa#|PpO#&e!O>3)#&`m>$CJ!a)yF9;ThBl6BwJ?f$xCr(^$(k8Af`kEl!++ zQ6io8l7e8D#`$F(X7vlQgO4;E_B?Dngf9M>mGk|()AaVw^GCam4G4U+XBd}I_urXD z?s9wT5=medKtuW-pmup+HXN%M$;er@e&yuFf?O@-AZjzi|1BGOj4Y<)ZYxXqsdBaW zxU@GPQvLFxQ)_iBP`cIsK-%Oh`yaF0!Y%2Xbzj$3OPw#gv`{rZ9YagiF27wU>Zh7v z+Amo0Cf^bwXjyaKb49{$UHC(ck#(>zU9-Z@wr-sLgLm3p<@Br>Rq}JG-vdjB7)k}F zxPFz%E%m>rk?(SJC40&D9bVS;v-9XGj!cEW4EdV7xNGcg`Zz`BeaWgXXu=2)J5(Bk z>gb%l<~HAwNPWpt#KZPXnkQxzxEGVuXJELL$?lGR_d_~COnkA{42+}g#I;mm$%?o( zNEF3BVk=oa?uOi0_T7*%oepJitfP=xpg1#kIS!GdTV8m#x`$i z9+NqCLrizv488o>y_M%2s=%}`{Z+ew9j(e3zY%M;{^=8e}h=(QW6^wB>gqb6shr(nlH;_m|n>z21&0V`^8U@r{6 z0|3!ikn?9D-q=tNXR9C0DV8lQiOp`BKGfAUH$Q&-azQZu5`+&(nvl_Nk^QO|P%soC ze1mgZ%m!$gR z1nA|m=*W{1J`M9CKnsFFcoa6JO4u0OUozyj6d1X}&%XzZXn&7fwJal=h}&d?5g zRxrPl4jn4_k>&n#bhL>v^?v&}4~^WFDjPkTwW#zffNNuR5ThC|*V6x{qn3Ie%rz@1 z<58>!KE){TlJ}Y_7&f(b2(>U1Y1EaLK^BZeJ#q(adzoa=wkIZ*CEAzwRwbT}b*mqn zrxqHLqFRFDYL&rF_BBIy=JtDg|GsOZ|FTN?xLYVceB|v^6?`@Qs`DAy+u@oouRRI2 zNnE#{&aTok7oo~>Vz~9}I@WE>()MQ=Nj>|iv zmFM2K5;wYdK?k(YxuHrGp(v;sWUfYOnoqltSm4_m-Xp7twL4q zl%GVw%2x#inH?9y?n4$NblIojTs7>O1c|jF4l3`6Laf#a=H43>uPXqq)0P3 z*{1>cVtH$+q##)Ihm>l4%HutAH zV8@ZzMAYR$sp>*kgYrR}XnE)BgF)SLDRgfs7rxHj4# zdFWI434gU|BwJk^s)slGh2%93X|>wxN}6SeYC_0%EzI&0Mu6sBw958affh7eda8=E zk@>kY|F^=eXvNWj;a)yuKtr2epjFsg6zcG81jivhqmermywmqjQm<$(co*Wn8D7XM z^>CkFP$(HYv(g!u8^n5)RV#p7-v^kSI6VQBOFpz z5m|@cc19`Fbd$C(sIeFjViSE_ciyOnMHYaO)us9o3lt-&F<}Lft-X8!;xHc-ve20J z(ShgImsej;8N2ihgiQ;_WK4Z{9q%k=7xtqFmPoIcURXgn&)eK|?IoJ_vA7q4Yg{ll z)AtkV5*{5P@~faXHm5vgkbQ&O2SWXVt`1>Z$E|sq|8bTu>Y*{oth@w$$U(s(I3U!A z^M$QEHEwZtxFnq?>gDd6l=aI6pAFwFuIJ{=KLWd1)cyzB*BnsvE;kS6da@%jQ|<$RXWr!ux8{{^w2+ zcVGG~*Sv+H;HP&u)xHb=6bA?E1i-Mna_VL+vNk6zp1etYt>95tLI)}babQR;Poxnq z-i&1@6zfnrhi#uvZgAWbymxD^R~~A%v9v%Ep9(N&A!cmfiksV|m}~8|j#e9U&(jUp za#Bu+$0`ALyz>{tT7`ey9;a3>Y1$<9Lbr0p2v+BkZG#CW!EvF_Cs&<%yRK=L|Leuk2u`Glk)8e{;Yv<&gNgp=gj^nkl)O29Z zlbWCnz$k3jL5UH7-rkMY5m7HZn8U2j5WEV^2}8}QEIEtW9kTIzSVjg$FX|@4TKl-=Af280+gVvdiLZ1$Z-O=4ht|J^(Jx3O7!b$X$G5nXOZY^wxhqZV3=A>@CYlAHA?5cJ;D_ z240TtaDH%3##_|B!Q{m4Q3B?HtLeG*iST1GO@oNlEtgex?#^Si6JhFAjY^yNQ0p56 zg9V;l;BgIAwD!t!yx8H)(0u0#2TakaE#Vh}vx)!0H{agYWW5>0ab3DG z1|sz|fb(#8=d|qjP9%v^`Xrh;yAu%paUb7A@lV%;w7?ntbthM)JuUBEovq0~vOYWJ zbxF|eN$ZJz+IcYI2F6P8bmU#Ly+`Lnc|I?xNbiQha~>k|bayn@Gmib(0@<nNFZiH>*!y$O(&n)O(`FViF*vj*mkYtY3$u*$YkS=Zi)%5@BX z;$n(d1B!alYnzys%|>5ntCe-m?v&yK?nhVFg~9U*nNaNaKklhk_*!$66f54T^A}`^ z)H5<`y~ePUCOcysTxzNDL%T#)*>kLe(eGV7R)B_EeiG=&6doWy{SDO>A>0%Q8`E{2 z{GnMMmt~!+A(5TemMDG_Q1{=4r%>Pdx(*xs05(cAMVbg#q49BT&DcFh%oxT3Qq?xA z_g3Zyx@>ffcL2uukJ-K=jr17&C-3k17uruv1_R;E&GiUUFXii30DxL@-6LRU;GU$j zNm_OMjuJOpM&d1sPbTcm|8k%n_*l9|^63F*;K#0U7^Pp9lVgJOlb;WdjBF-r!JU~d zj%Z5uz%U@5^j&~esao^Z8 z?t_h{WRS>U{~>uvU`id+o3QtFuzUO#VncGzzyP?!JoMTOGCE)jH>#OjpQ0H5*-oZY zOGt2QO@&`~PXpO5v~MlfLC@h#ojq=i6vg934gfarZ3NN!KiN~UyIfLhL}J+AI)p@B zII}92A3-b*PD^*2*r$w>+WE3vUh7|N`)Zi+`#Mq)tN zo+|`YDkkg$dW!Vu(*A?-6oZ9xpl8$eyFwlNJXiH4klXM~81v?;vhLx9K`m!CNz+EH z^Vta{o^mKC`R%HxbD@;AE30OecGQYq;8emTR$iGlC zhVYeINlL&yaDWjM_T46#ygYh~qj`0BK(QJP;@5;AX7cq?>m{_@_1Eg6c)I zPX@Ljyt zFK@um+PS320V4rYWn@Hd#q5Aa8u2JSCvo^#Se^5{|1-& zzt9b}2TBqAOz}d`3JW~0*j1&8>l>+`Iv#xCs$v{Sun67{-QjA;rfY;q%9FB;Hvksy zGY|L_!jYSF$rNHN;8WN~X#?F!>Mk&Tsrffb`z@Hz`t1)ahw4r{K{;I6>J>^$jFLt} zwR@ok)mjRfR-jZ^AA1}Wt{V*BHRzq*?DVjK( zf5Kg5^<=o=dQsTnhqd8=BDwAYNoj~gb1prJQ(`RO(+%^UqixB>T%&ZB-{vX2LuZC- zuI4OkR110~Usmm|7^D8Aui>`=zv^#bL`XZAPN9a5=HETbc}_gJOJZdtZO?bbB|Wj8 z;99SI3AYw5MfNir$}ao-sUl{2+V1={H;snZ;f%K++>?5Ag~%oaf)^#&82J+Lgt_yC zH5Y16dCsO&53b&;E*rY^xkvt6OWTkvOsNsSog3r=+kL-&SUS5K8+w_~FXFVkW$rgKwt zx3V-(;S7^Tv5wWIKYM`DZ%wzp_~MAglZ;m}v&J{D40Wg8B?p|*`zUd1b^$hdhid-Y zc5FR#mnC~Yu0n2}Kt<;N;UNI*fEN@~4zSm?wFE#7Fg5I+9~_UTM!pt&THvH2C>-qj z0~%|Gvmu#klKiP*)Sn-%hzcksXDVK`cVtR&9FF?b`Z-v8_8He! z5$7ML32bPF48T=E1q!TnSpd15dk7s<(sz!9}c^8h3O7eu4-wk%}okl!Gfiv_> z7w{4U0S|%QKMO!uGKT={zlslXEf=ueeuQjvZm%(u+%Sr;EHj6sZH_@rN3Lkgu3x9T zGh@7Xi58rF_d(-_qQ$U+`qti?eK^U8~Y`kAabba}Kvazx?6;hLkKMIJemsZ?PLba#s<|G1D3<1M&FKWI&C2Cvg9CQXf5TEEY_gAykTRiN z$k6iZ-Se_57jB2Px^1w0@24yIPP+9?`P_Khq|U8jUUO%?-ACH=)@Ppb+jmvOeRFFs z0+XDWCiEt2HEI8;A%X~Qp#GEsi}o1TH5jr99_8lhIBj$Kss`Vy^bd6{nal}A6!B$# z+REgHVHu;1hV08FF{awCo{@7jpHw>9oCXGkTw`_47S++FSE(>_m(EsZoce`LPo=$V zCGcN}DD>#8KadXhOZD8q=tt^R*_WosCtknF#!Cyq4@KO3}%vb5n*D-V9;YxOf=g>^?E@N-{dK)@`rM7-KYGuwRXflH z%|`K;E&}v)nKIzYxi7x=6Clzn#Ghl2*KijScyAAG7T%EmWykU$+OWa7Sb+Z{or(_R zA@<;%!u1GV`pKsKQe-RJBRff-dOK^W;;4^Hez!x*2yYtKqu0*@<_KnxXbxzra8VQq zUOCmrnDlMiZC{O#>mv9y)>7*8Ym;QM#$PaX?n zU-I3FR8*RZ725N~iz*LPcaCxW8i~ew@TWezKSH{R6D5zy$oLlM>$;t3;i2&_u564k zu9Tu0KaiCC28{X}B&*hsna=@qXMj%4X@f#3g#?D)wz`5-Gy7KBM?Yq{%&9-#T|TJ6 zZzR%i&FzJRpH0d*0hI?|W3*rf zR!sOnQ-2z-oQBC(1yxH;f$p-5YE}t$26H~Q1d!>ziy(mw|L-NvqHu`EpckaUpaRk? z2&N`HA3G`x^sXg=RNVf|+7j%TI7(Xlu%p-L(?Z4dX6^QQkv2E(b2Zg}xHE`Q!52zNv$C%bNFXR z8T@EHy{p(+hDxPdG!B1*@~G7P2k=A90O#ZBtiZ=Z@w=8pXU`(9*DNF9W)*;2YvULH z>W_GDf?X;%Xb{=6l6{eQFQnlF)pkpuKj|h2*7v|#p@4chk;$TP{s{@1YpLZIkUiR? znB(4#5AN1|