From 074c81e0f1a89be6942b45e041e3798ab4956ae2 Mon Sep 17 00:00:00 2001 From: priscilla-pan Date: Thu, 21 Jan 2021 08:34:20 +0000 Subject: [PATCH] resnet50 distillation (#20) #15 --- README.md | 23 ++++- doc/ResNet-50-Knowledge-Distillation.md | 46 ++++++++++ docker/Dockerfile.cpu | 5 +- docker/Dockerfile.gpu | 5 +- examples/resnet_101_imagenet_train.py | 37 ++++++++ examples/resnet_50_imagenet_distill.py | 38 ++++++++ examples/resnet_50_imagenet_train.py | 6 +- imgs/distillation.png | Bin 0 -> 50143 bytes setup.py | 7 +- src/model_optimizer/pruner/dataset/cifar10.py | 3 +- .../pruner/dataset/dataset_base.py | 8 +- .../pruner/dataset/imagenet.py | 14 ++- src/model_optimizer/pruner/dataset/mnist.py | 3 +- .../pruner/distill/__init__.py | 2 + .../pruner/distill/distill_loss.py | 75 ++++++++++++++++ .../pruner/distill/distiller.py | 27 ++++++ .../pruner/learner/__init__.py | 3 + .../pruner/learner/learner_base.py | 85 +++++++++++++----- .../pruner/learner/lenet_mnist.py | 10 ++- .../pruner/learner/mobilenet_v1_imagenet.py | 10 ++- .../pruner/learner/mobilenet_v2_imagenet.py | 10 ++- .../pruner/learner/resnet_101_imagenet.py | 78 ++++++++++++++++ .../pruner/learner/resnet_50_imagenet.py | 20 ++++- .../pruner/learner/vgg_m_16_cifar10.py | 10 ++- src/model_optimizer/pruner/models/__init__.py | 26 ++++-- src/model_optimizer/pruner/models/lenet.py | 7 +- .../pruner/models/mobilenet_v1.py | 10 ++- .../pruner/models/mobilenet_v2.py | 15 ++-- src/model_optimizer/pruner/models/resnet.py | 39 +++++--- src/model_optimizer/pruner/models/vgg.py | 34 ++++--- .../pruner/scheduler/common.py | 2 +- .../distill/resnet_50_imagenet_0.3.yaml | 5 ++ .../quantizer/calib_dataset/cifar10.py | 3 +- .../quantizer/calib_dataset/imagenet.py | 3 +- .../quantizer/calib_dataset/mnist.py | 3 +- .../quantizer/tflite/optimizer.py | 2 +- .../quantizer/tftrt/optimizer.py | 2 +- src/model_optimizer/stat.py | 1 + tests/test_model.py | 36 ++++++++ tests/test_pruner.py | 3 +- tools/common/model_predict.py | 8 +- .../keras_model_predict_resnet_50_imagenet.py | 2 +- 42 files changed, 619 insertions(+), 107 deletions(-) create mode 100644 doc/ResNet-50-Knowledge-Distillation.md create mode 100644 examples/resnet_101_imagenet_train.py create mode 100644 examples/resnet_50_imagenet_distill.py create mode 100644 imgs/distillation.png create mode 100644 src/model_optimizer/pruner/distill/__init__.py create mode 100644 src/model_optimizer/pruner/distill/distill_loss.py create mode 100644 src/model_optimizer/pruner/distill/distiller.py create mode 100644 src/model_optimizer/pruner/learner/resnet_101_imagenet.py create mode 100644 src/model_optimizer/pruner/scheduler/distill/resnet_50_imagenet_0.3.yaml create mode 100644 tests/test_model.py diff --git a/README.md b/README.md index ef91d93..fdfb8e7 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ sparsity pruning depends on special algorithms and hardware to achieve accelerat Adlik pruning focuses on channel pruning and filter pruning, which can really reduce the number of parameters and flops. In terms of quantization, Adlik focuses on 8-bit quantization that is easier to accelerate on specific hardware. After testing, it is found that running a small batch of datasets can obtain a quantitative model with little loss of -accuracy, so Adlik focuses on this method. +accuracy, so Adlik focuses on this method. Knowledge distillation is another way to improve the performance of deep +learning algorithm. It is possible to compress the knowledge in the big model into a smaller model. The proposed framework mainly consists of two categories of algorithm components, i.e. pruner and quantizer. The pruner is mainly composed of five modules:core, scheduler, models, dataset and learner. The core module defines @@ -35,6 +36,14 @@ The following table is the size of the above model files: | LeNet-5 | 1176KB | 499KB(59% pruned) | 120KB | 1154KB (pb) | | ResNet-50 | 99MB | 67MB(31.9% pruned) | 18MB | 138MB(pb) | +Knowledge distillation is an effective way to imporve the performance of model. + +The following table shows the distillation result of ResNet-50 as the student network where ResNet-101 as the teacher network. + +| student model | ResNet-101 distilled | accuracy change | +| ------------- | -------------------- | --------------- | +| ResNet-50 | 77.14% | +0.97% | + ## 1. Pruning and quantization principle ### 1.1 Filter pruning @@ -63,6 +72,16 @@ quantization, only need to have inference model and very little calibration data of quantization is very small, and even some models will rise. Adlik only needs 100 sample images to complete the quantification of ResNet-50 in less than one minute. +### 1.3 Knowledge Distillation + +Knowledge distillation is a compression technique by which the knowledge of a larger model(teacher) is transfered into +a smaller one(student). During distillation, a student model learns from a teacher model to generalize well by raise +the temperature of the final softmax of the teacher model as the soft set of targets. + +![Distillation](imgs/distillation.png) + +Refer to the paper [Distilling the Knowledge in a Neural Network](https://arxiv.org/pdf/1503.02531.pdf) + ## 2. Installation These instructions will help get Adlik optimizer up and running on your local machine. @@ -102,7 +121,7 @@ rm -rf /tmp/openmpi #### 2.2.2 Install python package ```shell -pip install tensorflow-gpu==2.1.0 +pip install tensorflow-gpu==2.3.0 pip install horovod==0.19.1 pip install mpi4py pip install networkx diff --git a/doc/ResNet-50-Knowledge-Distillation.md b/doc/ResNet-50-Knowledge-Distillation.md new file mode 100644 index 0000000..a5f0f7e --- /dev/null +++ b/doc/ResNet-50-Knowledge-Distillation.md @@ -0,0 +1,46 @@ +# ResNet-50 Knowledge Distillation + +The following uses ResNet-101 on the ImageNet data set as teacher model to illustrate how to use the model optimizer to +improve the preformance of ResNet-50 by knowledge distillation. + +## 1 Prepare data + +### 1.1 Generate training and test data sets + +You may follow the data preparation guide [here](https://github.com/tensorflow/models/tree/v1.13.0/research/inception) +to download the full data set and convert it into TFRecord files. By default, when the script finishes, you will find +1024 training files and 128 validation files in the DATA_DIR. The files will match the patterns train-?????-of-01024 +and validation-?????-of-00128, respectively. + +### 2 Train the teacher model + +Enter the examples directory and execute + +```shell +cd examples +horovodrun -np 8 -H localhost:8 python resnet_101_imagenet_train.py +``` + +After execution, the default checkpoint file will be generated in ./models_ckpt/resnet_101_imagenet, and the inference +checkpoint file will be generated in ./models_eval_ckpt/resnet_101_imagenet. You can also modify the checkpoint_path +and checkpoint_eval_path of the resnet_101_imagenet_train.py file to change the generated file path. + +### 3 Distill + +Enter the examples directory and execute + +```shell +horovodrun -np 8 -H localhost:8 python resnet_50_imagenet_distill.py +``` + +After execution, the default checkpoint file will be generated in ./models_ckpt/resnet_50_imagenet_distill, and +the inference checkpoint file will be generated in ./models_eval_ckpt/resnet_50_imagenet_distill. You can also +modify the checkpoint_path and checkpoint_eval_path of the resnet_50_imagenet_distill.py file to change the generated +file path. + +> Note +> +> > i. The model in the checkpoint_path is not the pure ResNet-50 model. It's the hybird of ResNet-50(student) and +> > ResNet-101(teacher) +> > +> > ii. The model in the checkpoint_eval_path is the distilled model, i.e. pure ResNet-50 model diff --git a/docker/Dockerfile.cpu b/docker/Dockerfile.cpu index 8a24000..1698428 100644 --- a/docker/Dockerfile.cpu +++ b/docker/Dockerfile.cpu @@ -3,7 +3,8 @@ FROM ubuntu:18.04 RUN apt-get update && \ apt-get install -y software-properties-common && \ apt-get update -y && \ - apt-get install -y --no-install-recommends build-essential python3.6 python3.6-dev python3-distutils \ + apt-get install -y --no-install-recommends --allow-downgrades --allow-change-held-packages \ + build-essential python3.6 python3.6-dev python3-distutils \ curl git openssh-client openssh-server && \ mkdir -p /var/run/sshd && \ mkdir -p /root/work && \ @@ -26,7 +27,7 @@ RUN mkdir /tmp/openmpi && \ rm -rf /tmp/openmpi # Install Tensorflow and Horovod -RUN pip install --no-cache-dir tensorflow==2.1.0 +RUN pip install --no-cache-dir tensorflow==2.3.0 RUN HOROVOD_WITH_TENSORFLOW=1 pip install --no-cache-dir horovod==0.19.1 diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu index dbc1659..52d2dae 100644 --- a/docker/Dockerfile.gpu +++ b/docker/Dockerfile.gpu @@ -3,7 +3,8 @@ FROM nvidia/cuda:10.1-devel-ubuntu18.04 RUN apt-get update && \ apt-get install -y software-properties-common && \ apt-get update -y && \ - apt-get install -y --no-install-recommends build-essential python3.6 python3.6-dev python3-distutils \ + apt-get install -y --no-install-recommends --allow-downgrades --allow-change-held-packages \ + build-essential python3.6 python3.6-dev python3-distutils \ curl vim git openssh-client openssh-server \ libcudnn7=7.6.5.32-1+cuda10.1 \ libcudnn7-dev=7.6.5.32-1+cuda10.1 \ @@ -38,7 +39,7 @@ RUN mkdir /tmp/openmpi && \ rm -rf /tmp/openmpi # Install Tensorflow and Horovod -RUN pip install --no-cache-dir tensorflow-gpu==2.1.0 +RUN pip install --no-cache-dir tensorflow-gpu==2.3.0 RUN HOROVOD_WITH_TENSORFLOW=1 pip install --no-cache-dir horovod==0.19.1 diff --git a/examples/resnet_101_imagenet_train.py b/examples/resnet_101_imagenet_train.py new file mode 100644 index 0000000..a74832e --- /dev/null +++ b/examples/resnet_101_imagenet_train.py @@ -0,0 +1,37 @@ +# Copyright 2019 ZTE corporation. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Train a ResNet_101 model on the ImageNet dataset +""" +import os +# If you did not execute the setup.py, uncomment the following four lines +# import sys +# from os.path import abspath, join, dirname +# sys.path.insert(0, join(abspath(dirname(__file__)), '../src')) +# print(sys.path) + +from model_optimizer import prune_model # noqa: E402 + + +def _main(): + base_dir = os.path.dirname(__file__) + request = { + "dataset": "imagenet", + "model_name": "resnet_101", + "data_dir": os.path.join(base_dir, "./data/imagenet"), + "batch_size": 128, + "batch_size_val": 100, + "learning_rate": 0.1, + "epochs": 120, + "checkpoint_path": os.path.join(base_dir, "./models_ckpt/resnet_101_imagenet"), + "checkpoint_save_period": 5, # save a checkpoint every 5 epoch + "checkpoint_eval_path": os.path.join(base_dir, "./models_eval_ckpt/resnet_101_imagenet"), + "scheduler": "train", + "classifier_activation": None # None or "softmax", default is softmax + } + prune_model(request) + + +if __name__ == "__main__": + _main() diff --git a/examples/resnet_50_imagenet_distill.py b/examples/resnet_50_imagenet_distill.py new file mode 100644 index 0000000..82b5269 --- /dev/null +++ b/examples/resnet_50_imagenet_distill.py @@ -0,0 +1,38 @@ +# Copyright 2020 ZTE corporation. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Distill a ResNet_50 model from a trained ResNet_101 model on the ImageNet dataset +""" +import os +# If you did not execute the setup.py, uncomment the following four lines +# import sys +# from os.path import abspath, join, dirname +# sys.path.insert(0, join(abspath(dirname(__file__)), '../src')) +# print(sys.path) + +from model_optimizer import prune_model # noqa: E402 + + +def _main(): + base_dir = os.path.dirname(__file__) + request = { + "dataset": "imagenet", + "model_name": "resnet_50", + "data_dir": os.path.join(base_dir, "./data/imagenet"), + "batch_size": 256, + "batch_size_val": 100, + "learning_rate": 0.1, + "epochs": 90, + "checkpoint_path": os.path.join(base_dir, "./models_ckpt/resnet_50_imagenet_distill"), + "checkpoint_save_period": 5, # save a checkpoint every 5 epoch + "checkpoint_eval_path": os.path.join(base_dir, "./models_eval_ckpt/resnet_50_imagenet_distill"), + "scheduler": "distill", + "scheduler_file_name": "resnet_50_imagenet_0.3.yaml", + "classifier_activation": None # None or "softmax", default is softmax + } + prune_model(request) + + +if __name__ == "__main__": + _main() diff --git a/examples/resnet_50_imagenet_train.py b/examples/resnet_50_imagenet_train.py index 0361d60..ec122aa 100644 --- a/examples/resnet_50_imagenet_train.py +++ b/examples/resnet_50_imagenet_train.py @@ -5,12 +5,13 @@ Train a ResNet_50 model on the ImageNet dataset """ import os -# If you did not execute the setup.py, uncomment the following four lines + # import sys # from os.path import abspath, join, dirname # sys.path.insert(0, join(abspath(dirname(__file__)), '../src')) # print(sys.path) + from model_optimizer import prune_model # noqa: E402 @@ -27,7 +28,8 @@ def _main(): "checkpoint_path": os.path.join(base_dir, "./models_ckpt/resnet_50_imagenet"), "checkpoint_save_period": 5, # save a checkpoint every 5 epoch "checkpoint_eval_path": os.path.join(base_dir, "./models_eval_ckpt/resnet_50_imagenet"), - "scheduler": "train" + "scheduler": "train", + "classifier_activation": None # None or "softmax", default is softmax } prune_model(request) diff --git a/imgs/distillation.png b/imgs/distillation.png new file mode 100644 index 0000000000000000000000000000000000000000..7b57ed44d4c817ed8318875cd34437318a59237f GIT binary patch literal 50143 zcmdqJWm8;R7cJZ|a&U*>?(Xi5yC-OHclSVW0>KIHZowUr;1JwBxI=J-_`w z)7@26c6D{{y=1N_V~m+FB?U=DI9xag1cE3n1yzASUNb`=FBboK2|lTm8X^M!z}icF zaDqUPx}N`DB+wz@K_J8sX{d;rd)mR0o6dyh)!V=5ml$uvp~_0a;#GgJ_KHH`9xZ>) zso0cj6rqPt&6i-73f9|dmx^{H(YkyTDV2yQszcL6#lY?iC;f*uN0hkC+s}_&810wB zsr;Wa-iyTNLk%$_T1JX z5X(q|clG~H{UZ??no0P%3ExX4P2VwV$>*tOVc8qQ|GNh7KL@`4zk8G0=?Wfby>@nk zgM-7Q-@b8n6ZG#Tnmsx@JKOc|H91*pHB)goU$+iMD+w%Ul$C_!#V3utvNC!a8X7^t z47ZEZw)06zWL7&+Q`GNhvsDZi|3OWph6H#{k-ROptb za3?w6#`Sh6w;djR3@xo$eFi2!unFO_n@0!az(648y8r)UObp*`9n96>HQ4dsc}bWF zoBbq~^~@#1pwPIE4^u`$>y-Gl1{dpuX7?`&OpHtmXHMBGm${_DL?*r3*&jcIG55it zH5Vi&9|#D1y%!J&g&ocm>M)lc{k+)gy>i6fl~Sv|Je@|;e6_gq^{$60DE0Ab=*yLv zxs_EZFBm>V{&OJoLv*z8^EA{nG^W!OxAYn@nA{8b&NR5NR8XZ6#v<|8i|*1R=`Wwo z_kZ}mnnM7;6FyuughEC?B6J=gt}N6j9-qZ4w|Zc(^+au22ZnM(88za!i+!5dm%{WvH6l(}L-4osjX601ho7Fm z4-5b@%KWtO-9Yc}+FJhSz})6)0@=?z-{9L_pH8~`-3kFA93aKV>30&vkxL=a&o z-*Ax2|6-QwONnOiYTsref)Yf|Ar+p;&>}QYARNyhzgMemZ2!rcAaDhNr4b@Eo3TV+`vU5th)bty z*v%oq2JP=pHjz8-`}A49&Er)OD8%hpVZTMb-frnSL2ql4R5ah&3z=las=S9DlDYuKyWeb|+aw|@INlH49i!Z7HvD~~%UpcHUTIxo4Bp{FE$Kg0(UyZLgj`KUu#uljr%PUEFi`Y!`@99z2P zq`^`;?rK`fiGw5V_qW^eHFt`SB)@$h3bu=aYF>FSpNx(DRCEHiRYdZli9R~=lh&$Z zlYW`j2b6=7xw+8L3XKy{&7;k;Qc;Z>0-4W$Z|+^5v{&Y)O>Y!aj}WISXV=x64d2P{ z$}t(?@b%u|b)oPJVW;e1=!bZ1 zv0rV%s2&@u5>C9dRNm|}o7AB;j=F5b?{jjdINZ;9@J5L@v9d5g=_;=R={8U~D8)XIO1;xt2s=$YaI*#`%QiAy*oVEz_AZFC?f_r|rr~>k2k){o7 zhJ;8IWhi}6o~%G@oPzJT$t_+k4S$P)gD{=r>aQSOp`-r0z8Lnx>lS5NN}lEZq1kF1 zx&?nVItGS|%%}c<%}lsNoWY z^SjtMsfiMlLB1i6lMd>I;o(f$@S=_y2SWsnk2XP@f@OC@rA~$<=x=CV(y(;lA7dGL z$_n+(@t|6&WGTmbQMj=D#xInu>gGQO1_!sV{pip4#jsNJ`}FV#%GV&Ab?v69jopLxoV`@_A$x5M2LtWWs(baYJ$uj;33)=WM}N1Aiqxdv1&T|o~PntkvVvWwujD8GdE*?8fSp%IxY zLU;KVdGL{XeIz{vDhqp%Op{=O;6_oTq_oKUM{ZrJS#7$~GNsJ88KI|`4YVNju2mcaV zcc_)@ubTwbN@7;I$Vu!AI#CURyR&y-IHI0=45XwD7U;+{UzIhcEkO!&Rp)7tvrv{D0u2`4m8i$Q(`DZtGu>IC=h{>G_ZEJm>eNq6}L z`N>?~e8iOaEBpX=D72M0WT1Gs&!;s}OyQI&hL9e!!&b)fc4oIZ^{k6l31o-rY8iG)`y0IC;ag*&MU2RTV zm6HK)!KcqWH8S^}s@uVttFI$}woaArDENN-TX^SJ?scYhVLUh*hQ0Ae!6Ii3x#?AC z=&x2D1KR}(iDKbfBsT{bV)TB7CjY!6`>P7v;g&cFWBtL@Vv%8Lz35ll(cvr1pGpiu z@}^J-;#Dl`KiY^uzu!K?Ms z%pzPoB5sMJ^)L4$7yqslj^yThz0rb{Ui!{}0C%)pv9{1qygv+2OU*WlrCNoj3^U%n zXm*Vx*)PrU$;r#L75quf_KjsGKV|5?vff07uROUxua#+PYBC$?X?r&wbttShyqso{ z>vwx@pOHk7*`*SxGR@~e=)Bt7)m5jBoyhEmf@=;1$7LzU1yzA zHNmk*Tde_*PR=6brF>S^4GU_FsC*j0)xTd*{I(@Q-fR}Ap2I~PvT#|eRM|5^H6XcI z&y?U(o`lSbW_|*~2JTuxvj+ygyzf}TRk3z-?#mhGY{fJa&69}4_LX$B_-S4RvPD!J-hwfqvz z6n1XqQ!M-G3Mg$y=%;D&x|gxVv#+j}=l*KgT11q=xp5Xt5wtidfwviEo|)UGk(>=JicQNjNeReDbys>V`ht zX*qdYC>$z>zMoUg`>oL0`ZtS$*KtG2(eZRVUp6w*q%v1pUY>!0&ZxOg*+Li=2MQG{ z3x%U0PgL2*JKm8a8K5Da8XK=U{(IcgA+HN^(MZGFFo zi_*$=L%w+0OaBwOk1p7y%yvlW#wm`=(#7Qa`=$GQ8x6}PZ@7N&eObJJprJ3@qj2;J zDsJ|F_sJ7@K6FeB3uC@1~4UdzEv$b`O9dMhQB zV!lvs6BhQI)m~m+rgGUa($E<2kwPL@4-ULH2NF1}reDLrMB($WvanR`6>n@9i)l|S z4>P*x4F-yO)2OV?$ib72-AIJJO{f(Rg+eho3pC@z6?_U&EWUeP~U!?m@wrXv~YDJi_}fBzVC z2ETdzI+PRdc(F-!x?d^CA45eBCT6EK(R0Vp^LRMjbeP3rJg?*NmX4CL!0mWhG73ND zP7(bWP4Q3Ge6B-y65SX-SxW3IlH|jOcY6(XXTTh)3Eqa$cI7jPVqe{0QsJ(B3B=wY z<|*0x7N2&h?Wk^B$6h3^Zu6NAFF-vhukhI3POjL)q+(S!S5LRvvd3!Im1~ST8Q0HN zIPx|{qKyf15fYf!)Y{r=yWEO{fx&JzjE{``6_3mII|1sa9%eSSg;tNVNIWi9HZ}qR zf{weRmhF+uGn0Lpcp9n@m9RdIFzjGX4ZMdLCOwX)1w=Qut5VI%3Y{iLU{-{LgdFgj z=ypqGJ9z^_RzZUahkvf}qVuKL!!))zua1_C_~2NIqg0kzeSF(QOKnBU2-QEpF8^(B zo(hXJhne%$L&VXULn-4$3n34SZV7joFynCtS zMQv%9`0Lj%P+)=)E-Wlm>Ua3gRhuv}GQz;X>`nH(o|Jndw&+Yc-|E2zGL4wATuE$f z?6)v5XfgdiR!g+GS%Amr)N-Ml9#m?`Y1JV{d56pCRw$*deYgHqD1+5xAceyU2APoG zY&fm5<8gj*QRn=i&FiWtlX7L4k=IHRtJWpZ+Kc_@O?aQG;qeQ34dY@r5Vh-biN}Kc z-13Ah?*=Dp7T)%-`7eHIemgZt7tbR-<#g^+wl5Su591ZYYM~Izaqvi$tQ10UgoMan zM{#sQhu69@bP~Yn@JjsapwHmxigWHV_NQhs(Q#sgPGd8(m1Y+@3JMAa1_l8E0Wz}n zll5Kz;LOj@V_{+0fWtIVATJ{F0u}akKzuw#y%>y3H*@@BKq#Yr`)^n3y`Yip2wzph ztm^Pf(dJl5D7$bvuRP8!${5(gl+5VRTAKw|@Q}yH$4uYb;?JKu!3lbJcyMuc)@$#h zVv~RT=Pd7&lmJ*4jDkqLOVxU&MbtAmlW$5v2ql&?Q)N7Rg!E=XxL+qv=S!TSoTY!^ z1a^tG2qYR*Ci(~4Z^vMQ7mBXlWKBm$PttO6CH1cBHaSv|kdUyjurM)I0@O)CK|#Rl z^8E7hXu3Q#GEyF-6`(My%+J5r8O#0i=MM-xL4kqvdads@HIokiisT6}^Y$}qY7teVxkfzM1 zG71U$-Z$pw=f5;#prxH}bXboelho1C$;r*VI6n`>6dj+?_=6USLMZQ$(2?ZzUB-E< zB!xk|dBBg8Bl$kIbUWrw?V^fNc`gimsG`98hH5Zbh$e>9G}P~SmC_~GYUqzuj|Rsf zt9~(5(?=m@*FwJ|akS1#bLlj%MiCFMtD@V0ONwuJ??4WECBUpU3^Q^@B`tMNnCjC0xgq`Ak z3{f1$@Pd*Y zp%Y*L_m6kSAP{%}E4n^e-yTZk^1Z(vPUqv=*aXh!A2>K(zsLI-eZRJkKVK50Pv23& zj=sOSxuMgnSZr~#6BhnK0- zs5=6CrOnH&GZ?YPbg1N3xPR3iLH^5iw+Y;beAGr7=y(h1BD{0{-mWMdstH+a{$3CU z8psDxXZl@Ue?aw!B(BuD#NTr-(Si|Ix)=`T9_q{eV@?{Ti#kSL+#heuR5wGCfE{Ky zn2X*t6L`$&Z}Cr@{@%q$*+Xvx-IqK z=;ca8dY*3dUmh-i4Z*_1G zx5xdK2rbfWapm<+@V3c}7RghX`Km@k&%t6faj)p*;o$*~MgNY~xDV{3imEDj55w6- zy+_F9xymM2v*|Xf^i!2rWRU`1>^OCTCHpXIHj1Jnc@96X^2o&*;^vNy&+x#2p7#TV zbmrLUo?;Y~d`y0JAcNE=-_1g0w1!NG zs6FKQI++owQ9sF)Q+2JNwwy^!`j0o; zndN1Z#}f#j=Cy0h$3S*D)nK=hqIj`4B`c(vb&oJG;QtM3E+#L-eqkmzdJhZ=eYhUbsAekX^+O%Xl4`jnWfPEO9G*E-wiU<781 z#i+yWyfgX)4iz|{zP`RgJUI-!$4HQHFg^X%Tum8$37ZisR0mdlWba%*HYyM$!oGaj zaBOPxyeP4e^E1OlOrXqPP}5ODdj*_F0o{zwu0wD z-pZ5JMP@222@I9nK{q4KX*8ChaCFv4O69^SaldZ{);*NmIXmnAD-hP}@}PfUfEPG( z@V?AU{h10~unE3Tk3Qn!aFB8tctt7T=dhXdtc{GEz+M9{U}kI_7#KJ^YksQx5ivp5 zcWlGWe=-mFSN3g!x>lh`tWtmhRT6GQJC;hp(NKPt5}^HJ#XP|1(b=TWMzuR#z5v3- zp5WC*K}(D6-k>Stq&or4xT-e>EeH)c{Y=W6EbLWMb0n1Aw|r?QGx%AX}Wahh6OL z-NCq%+053~*9#TWOHR58SI1OVeuw$>-=(FK-4e1WvJn03+}Du>Fq|3^lIEy|T@ex2 zr_NRTMYp^eA+^XPVp2y(M_MMD+_Yb}A)LvcdP1b6#sg1-PA)Fxe#SMC_&n9*SC|5- zH@O>djZiRj|`$Q$=dSuk&k-*BR!}f~j zP#(&82Iyv$aFoS;KQ|+9hcmH#pqxh

N$RM~FKukULjjzSq#u05I-AU*G!%+og7& zyJnaDdfTOD&B}Pcf6yW9IuSBbaPaV&m3oJN{zRvzKY#*)kk`3LF;md`ZT{<6{b6+i zUz9qhV$wYrbRN+d$qfPPFyoZ|z9N3ub?W0k9JQ}iG!a1+q1jW>-_@{U>!j^5XHBBp zPezJ6`aMzddBB=34>7Thy^%zP)o67dPWO@Pa}Qb8^i%zopBPY;bz~ybX9qesE_m+L zC?uEtN~e5$OT(qzCO3(BRv{-PP5>+}R|6JrvTz!@?X~D@^d(@>#M`#x%dc@usW4wX!MguF!+R-`@r5RJA7AmqVIAm#h6e_{2XE~uAzHPKxkY1|7yqWKoxuQny0 zaxYAW%#{H?E-)d(c<<#Z!89ji$**`5Rk`8SR!AZ1o0~4XQa-F83^*A4thSBUvSv8R@AF|Hl*TZ&hw&mN_@2?&;4Bs*E)uy z2C~iyJE85=q?&KaOBh0TuortjX(>vx4#py2(J70pMa0B9|70mP5G-n*Z=$Hoak!yY zVURpXxn!S4vo8ZG6*w z_=<_Cad~s7-)1;eil%;?FC22Wve)6Oc2W)S7+ zPB!)H9vg=!I?Q*i1G|=lOyE8~vks(yX^0%d(9@13vzrf{#U&8era37tv3#M^v<{`% z=TIl=jit53mZxiX{)WAwWK??)AIU$BS!-XlJ4Yq6QVpPHt{yCN3b)5EAMDB#%M{zsF*u1Cj6bDkv1*hrNbO#=#s%gOAof zBMTN46>)>iE}EqLGFqvk@oj!uX;)sDcDt0f>bE$BpZ$^dN8iajP`WqA!gS_TFBUO! zc;SNJX#yq5352IOrMg@k6Wo=~9}F+Hq)CFhRCFYCCuENPhAv~b=u4Emv4!;V$zA?r zwo~}FaLL}EtLfNF<8GlB{h2SHh59-i$wB8{1Wk{@_)~L!uGn|@Pv7>jaBPGD}&Yt+*5x~>VUf9l}M^;o}Rk6vaHn_)FaoS+HNpAqa zFwV+62sGk=E9GJGYHgq&JEnQADRJyUVBv77C)I8=aP`!bU7yeUrU{N6c5Lsg!TB=~u zu85hCrje0cQM{MesCH^3N|KRqg|j{VZkLv&tVFD+w!{Aa4Wt=LKNYfDPP$`p#PNHRNETHE;uzjOdhdv!KKRG(#{V-ucDFT$B_dw5y~d zrX%8W+PJGSmU3^~48RB-8yw`!L*K+ezkXC+Q*+9@4luLGQOM%he*w*}f@89nB3_gi z;U|rqg9m6ec0NpNmi!vX|6SW{7yOC|O^L1tUAg*MA`@IgB%!twlu~jyE3=;0CbCJe zcKq3QH}Vl8oft{`M0a)zT&eFr=xAzQUFrKxd~H%J_PF>8BcJQTJ{4QpaFRin{dg=iOrQxg?tQ1+u52YaJdi3^7P-m4ODjW2i;kvT=+{JMg@%|o&l)?+ zZQ0$(VgQ9c(pizexI`~#03*Ii>xV-JeD z`By|;0*rWzG3fCM-Vxc8`6LNlIMSPlA(v4;(pxB+w^7KS-Ju)D6J2b_m;Q7QVStfI zb(8In{qwPH(NM;6_fCf?yiQTz+V+goHTN5@$zHh0AYub0O2?NaDPaQ@{df0{caxb^ zhX)6Nv>@YmTJ~y zqd$L+t5Z2zQjL1jRp)$F5?57qQgQ5x`{*4Ku5~179E{LuHH(4T|0mRtv$_@GP5>ZMjJR+h{ zKJ{`t(@(F-v3p`7?$Dk+eh0}#J7t~rBBrh13wJ92un9xijE4_326hewRk^-(f;h*! zXH0ERcsRHuLZB*y+9x=m3!yi{MvaR&!Ikq=MskpAWt=NP*?QQ#GM$sXFcQ1*WE z9o3P)N%m0z;z~6Eh?60mbJ+E6)|Di|B(2TuOOOjU2JI)S#W>S87B<3Cx6Zpp>39BOEC6kMp@)#+C+ zzh8q-V524vDjIflMRj-?r;UHPzGg3Z-1H%3Vv@du?BpA}?kN+S9D; zY?6iS(B0i#aiTA`ma8fc(e5(tzg)RT-9;t#M|CIkRTUmQ49D(+tR*M@0k-~(EP=?n z(H{qh1I_x*7mx&%{XlXdA-{v!Do|pAGODh&)|e09g`!;{@5nNGKp;GFtomYQHFEbx zyZkhRNm-jP^H#7U9Q_+VTsv`0oSJSgqq3h$`d*$aLPP;~(}YI-{b>tv`rZcTdU@Sz zJ;5A|kGhdBqXxX8Id3%LRD)p9)tk7gqL&vJ`S-3q!`#`926*{2Mqg#jR>(5NWA z?;n2xfMhP}4Ma`S2R#;4Qc*Z;1;F(E%`^PzM9d>Ey2F@58d$MmiO zt+({VqTN8JG+e?OPYG+Ksv7SbRK4w**R`KcGPEf78Y^bfxY&Sj_;igcThUS>f>B!? z2bQT%q`HV7M2{ljypo4RcP5J8Ejh}<95n^2v00@ZS2+B5Y7<$f4Rlh;Ec^Y>tcJoBGuR;ocb;8T~VHx28Ba^a6_8L@MHR=B@=} z*Cx(?qmInX3a_qOhg(dIjDV6vp3C)McA`+R5}@k~oQZq_kWT|e7NWnl&)QmBK^^=R zpU3dy#~-fTmN|||xdq7?7ufbrm8!HcA*s%D4Lja`WXBR1#U*)_9*Cj$CzlwzvV`Nv zo{jxsi`{jsUZ_>iL|S2D1C^T=sn(YElSvFivZnbIL#X=Hp8DpVMYbMCPe0sQMTeM% z%RE_1hiB_X$(roK<~gx#;LX{V&-syD?%|BKXd8!^XUF1^(fG}F+udZ7cu0D>?pnT8 z$S<;HeR}rmwU@g6q%am?x2W-|a^XjXW(NAt2tYEDkh zFSGfLMla2XBm{D?^eu&6tl+V`&3bvt?ExWvczF2wTnm(JCHFr+{3+-J{gk!d^Om7E zxfECGAjT{9DajCao9*cgIuMNd%@h#qxn5S@06>KZfY{)~wFA<`y7whyAm0@hwv+kA zix(#+C(ot*Oa%d$M!F{jxr;HuB0NTnB%y7&mypONMp`txI>YWi$^<+}sX&-eIO6^_ z^;+>{T*SC+(3Lxq%J|%`+2l&@X>G29g95KuMMI-Up!;F9kZ2_1K4L}k(mkz zB#lK)Z}V~s?hYAVi>Yaec!P~=&WkVgr09y_JD*Ofk( z;0;s?-d=_d4kw#~Nl*NpkuBF(ovJjI8T?eitL9cQ{+zizRHN+ggQ8z!b}?%;&tvKG zUF_^K|Fr9Qio`iSI4CPGA9A%HxVT)fvzx2#?(N;)n<^FXJpagt4^hIpm?vp+KglUd zDSzXQoukJ`4#m)u&7~!^{`y_G%~i^nOSB|n1=C?tW;IZB6}wd9V79Hs*q@H4&WtZD z*h9nkTzAdB&_Cpgh?#?<+^=WKG<_fFyGS8*VIC_Qcu6MP5!0RPlTOWm;oH zX~Lp^Zw~+}06<$D(bz(YQgJEB0fcDRdeQ^9-&e0*0X5l+Nk$TqFqofH8|n$4;EJS0 zRchwNVm%AU5~A)MjJG6fUJEWII|xV(btH9MOn#h+%omXr)$qyB56YE?M5=0O0W!G{ z?9{7lD*hxJ> zJ0T8Q)Za(41e2$wsv=Fx*%)nfO&t&SgM8>K20-vApAZwXy0%sg;3^u`g6{u>ayUH3UFJMTe%r<5 z-v~ecbApMZMKR8oqxkjp^?Yte_ZxAFfFtR7HZOO}DzVti%oR`#5%9UT0*=PIH-~u^ zh+>PTkixcFK+as7?AwG{8Bxd`T|;q4)mROOkUQ=?uJ_RBHBwtV@mBy|_>2YNY?$x( zTXM2E6#DOQziE+^`}_L?_%bRo(neaSKIT8ItCoky4ZuhQ^)F1q5=2ha-tPTWNp0U2 z_Z-_zE1k=PdBdQ8+mxJ)!)E}gg(fE_Ki8oiryBr*o&})=BkUoNoP(nd(2B6IuzZn`u!l3bMg|6Cc<5e0z%RgI1q==P z)FFfcb`Vw?2u@epeN!Pm?wq_D>(23IL2(Sy>tS<3$!D;5e>9IfI6du076T_empa zGtl^Fz#BkB#KgqV_hgYv0l}}7lv+_<-p21_whkge`?Zb!fTT)9FK6m%Hq=XVS&g2_ zvA{1bc+BE&1U4sB1qIwK?uk0sFOs7Yr&Gx!e}B7NUa;#}svwZPpoNsHC@aTAMh*dV z@n2mfu%36;l;nWyoSvQr3_KyH^*e1AYUKaK-JHw_XMa~a!6DQQecH|(Y?XX6!Pw{+ zXJeV%HH|6Z!9 z69B+_<~Kq9!cFw@M+KYNFjQRp;iQ)coD)}9*Wtmzn(yCH1iG1juXRTN_G8F45HiXR zRwD$7z!LzNCv1d}+3^B-Jk5Wv`d2SL-9BS!fTZFFT+-9S%2V4(58l|=7@&%(c8bX) zqZT{-oT-n5iHfA{@!Ps3E{45<6c0g{u({d4`q{?AL{+$tyI2!n6Gf6HlWYinq{ z1>YSqF*OB4;q(`_-{^}0d~&uj;im$lZ9}MFrk$Z-Y+T$#m67CBWd&~IJJh{^2-Fvl z6n}$v0QMpEJl_FEi@2}~5w2lh>zf7}2rMHW2$kaEQ~}?drXCR<9%>HwxZB&?p=3!- z%?1E&a&mGW9UTGA%3>I(6d?=sH0Q4a+eq2 z;tseHhiB*Kn;07#TU%R$Lk~>K%*@QlD7Un9&{YRgqV$9mrbNvTo^wgmes^`sk!yRH z?0QQmbcyKjW!ke(w8Q?LNj?rrJzaVu%1ZWT? z7{N@x%Y(UV*;c2d$0nBtiLrgrCC=b^0=akS0?+4)Fl6lKS1OmhZu?rTJ)WxmVs-kHCO)bI4$TW;f9c=6m*FuXrgiA>0Qtv&f-xyjF&US5`9{-2hH{G-F7 zI9Qx4q6(OjkJKIC2-|bcz1Ks;1Nanq-D;8rE%p#PuI?`OdtL1l8eiZaGm|zPTAcV_ zUVM>XSv^AOIuq&-^z3hKHL#>7HTz{c0&87dTJ!*F*(0)$71x3Q#G~`X@>{?tPA-Sy ztmK+c^p#?HRD6D$Cghj#Z)1TP&@MRi93b<(chPC}mFnB`XCo=M<8=&N9; zMwQPW%D$9f&q9}XTVuj|kGl6@ZrFQVL5Ra{E!81Xad&wLm{98wsg2EbMQ9hFGimyv zl5ovbN;JNp_jRt~7S@}q%6Ddx^JPn!v)s4yCUFG%BYPaH=?xzIgbQQ9UZ|A~4Rd+} z9mFIWA2OVZog-U5*5P?9#3cZG@kiUSn)swZ;dGa9bS`&PS$DmFse90TG~X6+7@l>@ zhGq*1LH|KP>ebs0wN*fZ#w~FF?paRGRPzt9Uuq9~YxX4RQ*7EqfoU%pc z<>h@K8exQUCXs~4e47V(#f#}TX%;ZD079sB2lB_)bKq*Vf&XrJah?+t)W$W?qvg@sC^4y50oye z(~9t4RIlrZsh;zl>E0Syp92n7JQ7k;(@>W&;z9AB{?>1w-zbI985k7g;`UwWamR8u zaqRbop79q6xpL%?B7Monl58G3w^JXtz1NuynfxBV6v!`($=49!Pbh;z%}aB_TF~J# zQ6>Y@0yjVqxvC*b{JL9kYmQMgS=d`VQ3&WDaMr*=ho78o4u-{fV=>&-?vNFZ2R-zw zeLW2Qq>fMOmRfBm`Hj##cjtR21n7vndwBVft;CEdR%IE)u^>?ZM&J_IMueaG>HJK8jHNFTlnF z`vF!Qhn5?`QVR4#p@Try!RDe8<0XkZmHN4f6Y^pg;;_#ne(9ULim*nJLAm{i}7apH($8q|qh{ZK7{Pp@qf9Kd#y_GZ|S)_~|3+ekI1R6LY zdqD=@n&1VCftN_^0hK4N++W)fkVeMF#>TTWSqlbyd7UB>atOW&5gu6adrG$_t1=wa z1}6u1_}BNv(z{ru#BPTtmGj9p?=HW5Xs<34kslm9*>X1euD^L#sodP$9H=rPzJ3Ks z=fKdAdYq?)fIyo#lJ4KGI4wlSv!+z2ror^c7c;06of~dY_>+|=ofq2GV zF$WRJsa7+=p)I2d~oI>`1G$$C^Rc{lqbDiHprlj(a@83!8hhdX=liBdX@Nc1Yc&6 z3Rw;oFv}h8RvZ47FEmUEpuoa{JgeCqaHr*gc#|Wshr$@tNr2Noe!2ObK&mu<$l&(F zBWCIMX4ktP?!gA{Bp#E?nlsD4YElD@2zq579EQp7Vii8O2mABmi6Hn}3uE06e4ld^ z?<)WKpnk=X4!W`+5G8CbyX8qPF3egE^B(;Fvx|`?2gN{@B|nKz5D>TAQ%+9IZ8KPo1C0Up-1zlk{t_SGZEdgk zGWay$fk&ru7CJ+d(mMj>Gdbmt&Sw8x9giEPz+gBb$_zRMO#Ve&H}+EHvICm&B>vD) zYSP4bM;>nWgcw_K4-W?imIVjl^+h1vJf3rM-1`9%H|8IEj*D;ng^91E51u=oiz2Wq zPy2IrfX!>M-7%gwmVu|5II3W(4@|Xxs(L zoI?J^yfuuOHnmvQ0@9R~L{i<-x3a^T@zl!%u_WkXxmajVdU>biQ{<}FWM|X3wM7B% zc2HvK>gqssQ(RmOLN+L=5)#xwG=2_mvX{w}Ce}CUQl|p(-&5`PO zkT2AQJ)Xc%N@}$}AW1HCt{EPhPj?!N6Z>eNsYU!KZ*y^_l2Qc_f5JDC6F`YrW3T>p zmc09gUUoxb4KJd z3?h-B_w09rf`Wo@JVjnt>3t|iCWCoUPMev(}`|IFdlW5s$SI`0A zypw|tA`K{RiBme1x%m1{9xr$0E}M z5yFeAc4OkD9LVcFlMJJiU$LjuD!CL^544g}rt6d>GUf^7Sm#;3lQtuc)UBNhALP+W z1M;@X>2+$*K?&+3fGdEbf3umSUtLoJ*m7>$MQ%I0V=#@WOIjtci1&`2iDw~9?9(16 zYe9>JTEnTpe~da4@H&q&q0pLE9I&4#=bO~wQ>&nrHg|mB`d4$MnRiSf{hJc>XMITK z+31^_t*S0Gsdv<)(=fu`k$!rxJW$)@bJwRXNrz6&7LTySIECpuw4O>$kxn^ z3uyho>xKLt`T6-<8ylZhRsaWlUt=n7YHFjK`62~LvL2kQ$)Jb`S>9oS4890tNQTy5IF`2+**EgoFSU`trg8A2YN4WRWs; zz+3VEdAIM?)q9iWY)d{B-*#JXGjHoawyn@jaW^!=Z~^8R&0o@ zb>hTe9gmuw)*-b%gpA-^QBtF}S_r&v#$W7taF z!W2&jaE2k}le-*EPPCOPwghVR=B^@hX90l7rZr~e<7)vWImm-~`T6fB6oqtwHW9$C zg>uOOzue4`?q{AmxFre{GID$bk_Ir;=JnKF2ztqEvjeuu#7Mf0QKlM}3`{7No2BXF zo%A9?%NRmKt8a#)x)QeY>li-z@DReVwVGk$Jf33=pkdS~&o0yzr zr>5QkP-H5QW_|n!5cyZ@YkS9)2ugV5|HfxyaNc#xT1Gc};Zn!C6M$KxitA{4*VTUc z``+Pd8P?^vM=EVT)=RH64nWyDt~IX)infQ{gBZv~;_W8W;=DxC&}ogv=Hp>6Mg$+N zPYMUVY3u9z0dxzHZK?%wCl?p#>=r5#62Yuk0a(cYS~4!EQ}lfv^T&4OR6fS3V`9W@ zN($_-r>cL$k`5-ep9Ua;nwdtNm+kr+~YO9q0;!5kUG_{_~;_|96&r8rKh5y z&8RVxOQA#hMU>bx13|PNjmjJU6w~2iwwoi)`xUfYx?Wyfs80r7;IbG(RaC|SAs&oG z&-0Ksiq{ELmP}N(e2^Hm+)GRHKqQIyuS8$XJ5T4Uo}-nx2m36O72eAvTKF zc?Siw+&^U6O>}AksNBCyE-X|f&LownW9$@!v+bzr26Y^uBCbZTSC~ z`s%Q%(rE8PNlJrstAwDGlr&Nzq0(I{-Q7|m(xNnolz@b!AP5o?lG5GXAr0R;&b{~f z_8&eo!+6+d?>E-^)jQ0p`S56^f(8t5R$VwWx^JGD%~pFFQ(T}O6<%_Pc(D&P%PhL} z4FCM|hgVDt1g{sccCg>oG)lkrQs}=A8`AJCl~U2PI~=QcHG?ux2PD}`QgBFH_rT^P$=IL6oHt}u$WPce-1wlaF`um>%JAMe^8c>^~ zdBalfmo8*Qd8FjlWbUca>?)XXKL{^XX;|CAIA9wSel`wJYb9E~I)I#Wb8~b6(}3Do z^Vzc!=-7$=tw~A7ZPAQwb`wSwQt%A-0(eGf*esO$Us)Q&zK zFD)$rJwFVND{@YqW}{*P_J1WxF9rPIS7b9y#A(wSIJ2kqvb}mnBeAreXyyB{JoXzD zznssx`VcDy6+TUER~fwr-RtMNQKx^`MXZ)G3B-;YCMHrg+el|XOjqr`I|s^s zFCgmf=hc|_a4a`N?Ck9)65|V$N_kn(^q_Xn4 zh!Vy);J`4VhhfwAJhZGdMY;d)<-KHtbxgVF1kgf02JRa$VhVz&Z9#ya5caG!4k17POx^KiNk^MTKfP zFFSZ_Ndw1k7ye#`F`)CiJG0VEV_(g2$4e!gz*_ad`}iIY4>Xkj^WhS4_wab_rXDZ> z6noYhLG9$|I1-7K)G{#O3<*IT8ilM#$0oTE=D()Amj%$w&fmYik0#xqIN*U_5YM6c zm6nKaIF#NOJRTHublt8D4fy{iNd`Ay+hVYzfN@O}qEkSIg_O=-UYyw>g+qG1cfi+X z=zFpVTOT_+JB%AG|9^={e64`Pft;GpyyG3nEQN)HfIcw=17f<*U&?U@NvAqs;K4Kn zm<5>9jZpr(H$0GufO+8xNYQlINug=z6I}!`wW7TIjN>Qz&|!dws;hv(U@iP^J%zaMgYek^90p`7g1BA+NZPAcuYe9G=>ttV z(jIj<0KPP^8~xpy>@;xhbw)lF5J({P9#HohHTZ-5v_04J?*`a8noQg?Q2&hs2{d2= zf&0Q$l{=ToQr$_VXqoh+;nrJPUP(aT4 zcV`Dwtlw2xSu){O1Fzx$D<3>e#oT{cQ?TRi{G#{6qrU&UzaNH`gfTefT(_r0ft3LI zSA~IaqRtB(W(UA^K<{8I!l#Py-{4Syc4%kHYrB*dgH;P?A6OnqDJWJJ7Mx*)gV(~P zRgwWC#`+gV{{M^$Sxd_`C=$quaft{C!FPkDjQQrxMZk4IugDI`KJd(YDZpFy zDJm)|I2dC*z<(|=AptpJWt_ostX`mzn4DZjOO%J@ikw{%ybwMEzfQ3-IGB7X)oqIC z;yF}?8?vbRLH`X#X?OQ(kbJhqFpuJ+lfpX(sRxo?gK-Q~-W`N{>#v6I;u=Kl(WIuQ zUtJa^KF@jn{1$YmIj}gagRcSxlCaxWVRrBzQBK1D8D=nilkeO7o0$O;f55t}SMMz( zCMNdx=V(??bLMdH>GRea4V#X!^$(O5*oPGghM{*VJ~J=Nw%c@YE#iuI8;>KioDg4eH%$kwyEYkui!R@nBw(fN{B@#Ry+m+U9&tQrps z$;-I)#>U$;?rCu2S>-BT_o95i425TfKF`#<9jxhnkRB)o}Gh+Md3+BDr#Pt zgQ^mSbBC<}O^*U4T)yyNb3svSwk-9Kp)1VK+QhjuRr z7)t?EIawhX4(X{g>cFf2MTIo_E){(48hgh3MPP^qP*MJ|e7WOrZ1+}!+LufK>I z{RXY&>Hxt5+F;Z%#f&e_Zro<5h!2|YPS@loTDJWjDHre|7IL`48|v5 z)SY2b=$GuRg{9MOqUK~`wCtVnt%U)B(8~1W>u{Y6aDy}V%u?He8;UFL1BiV*NZ5Z`}8W#_d*J@ z+bu17pOawvq%11<+x9{js7A=K(3xs+q_7Z(_gwnIewW_2Z{LPn8GrZTL-C2pmxtm7 z`*-Pw=yztg90}$4`(v+3?ZB)a4uSbyXV0znK++$Cs5T!Tt=PPu^0VnWuR!33t z|8#n$rT^sOQf;FrwNZ^=p>{twQV~U$w%hSnNFbf&BG0drlbY)5b$(aA0tQ5kl>6X1 z&&kaNy-l(+;lcgo?sVQVK5=r|)(cB?w|d%x)81ev{agvMw*t#F6l4#*wjP-Ee$c(| zAZVw3KZ*tQ-xy&K5IBmXd>Z=JzApFVjgvi}ou^-~2C)_DZ0ffewWzbNvK|q;rz5vH zIUg6I5Uzzm?=k#E*KMvD+r`DDUt9GT>^ir+c`UZxJme3z8M$&oMI-AEgH=!=K%gaw zWWjeqQT__W6&5TeTCBS3$Vs%?I=}nZr|id%z|HI=vn_#7$G#d33{}FBvSi0v_OIlw z9(B1mQdaXDiG1k1Nq&uy?brP7^RA@D?QMWg+>yVNiJ6)6RM)WB^ZFESL+5!DW2)RY zr8tECyM2GRhlIEHgH;JEg=|jUO6Z5sF)`!tUkSrx|4V<|`eN@F|Nfc7@_0aI)?bB8 z#M}Gx-u&Lvcc)L3)7s2<6fmV|8{jpveDS?Jd2#zgz-RJ97nG)kXL~tLD+L2KbSaBc z^!FW)1sXATx@n}g7eV9&l*+|7r{#C=nnf;%T~L#|(s-P78fNQKHj7|FIHxH0suzum z-C0xbK-^e{7=eBZRb8!UhM+7JAD@d@vQ7pQ5Ojt=9N9t1$!<(Uh zmC_e`OiJZ3rR0UPva`Dqd6!``Ew8A!nTk=NuBj<7G%Uh2kHTR@L6ILD`NzVU-Rs+Y z84BNxs*ez;;Kc*B2MVue`76R2j{^Jq`@uw1s8wpx`j#@QETb|( zX(?fMK6d1;RsrU#SLKt#IS9K3cN$;;1wlXe9y{@uHCr~TFwYR$9d#;93A`jKU|5HY8Aq52?`5uja7s@lMj%Gw@~rhD!s`_ zxDd|;dB9Za@%*U#Coe4Oh=Tj?)lGAF=7f*;o?T6X7yj<5#P%$dYT}odol&{IzPl`;Ciz?? zLqWcl_d2eXzmrg-TiNSB>yydEMsHX|sSVI=VPiER|L*O2cJR^Pf0K`D)AWY|$I@L_ z5E>cBE5Q4|o$f7_{RBpk8_vVE%a|j3)PM%Rt-{+WA2tTNV@1>|ozOpZCFs?M9*s07 z`~m}R)_evXuN1KZwNyt#2OTP>yqqMKOE@&5j7|E6*Hveg%ih;pW+^nWbn)7&-i)-ar27 z?u&zqAGbm%sVRJG?~c6quF`Y7!DYRttE##KgPogVp>j~#u0-_SjO+#^HA!dHQ4!5W zt?Oz$6WE&HJ+hn3{*!I|*Dh-4(^ervkS9BN_#%H}ZuyVQXPV4X4-CWE1!xJjO9uxB zv&z0d*47@49pvb`)77*5aKC<+(3;{}(_=&d(cA*7(MN12nzbLaaqBsI?GLU{ny!I| zF4VcOXl~aXsz88E0Q!LlforRRd0)zwyXlTE?Ki1z$5M-ikz-Vs*`zzZS?1J{5XSj8 z2zSv@8w)+0cuulut}goxBCz<6wd;Cq69ioxrKQnU@6Zxrq1PU5^RJnnKK;EIqQ6x# zqA)Q&^{G3fqmh+&D?qJ%=(?%cUsrmOnv4v8=Ny88-rjpwLxteW$jf7eCam8E6i9Ylg3aSXdDI`%y)Wy5(Lp&#t(lCP%Lg^u0}RZDa;B zax==>FLo#j_qG+bIZL45Uh>@t!dS)Hqm6HLPPML91c|p{5k~2js+IFH9kM;18=sNR z`cwYXj3@VZk^}FhnA^VU3a3YHZN`1lyczTEi#)f3U}xRip*tmyY;A3`cP2)L#Gf!xufwIf!M>wEz{Z z!^nn$<@@YR(D4u^+6HbQ)`S=do*v5j(3Ja}&CvYz=E`o%O-k0W` zl3WyRM5p4cQYjN{*0|t`gP!y9ofH+}P$d#$$yiKAbL4}={OC)#Hg>$zQW!}v^Zq7f+E273pDS-YsQ3S@xJw#mPoO9ujk%#!3oNXtp~5`eJ_Mp8NNQsVrJ3!i+sls zbk_IK5N)g*)vYgG?lqvQl%V?ay#6Ig9Hjb*Q@83p`>=q!yre)r0pIV0Q8|mzD~;%G z6vz=x;X910$;L-D>`W_Fm6J`(D|M?5X*sWTZ+U`EfBOU^hyXpUt*@U!ECd_dVPQPB zsC@6%>Lg(sH*d2JAO#^R4&fsDIuw zWkQKw`3$l>Oxx)S<_!~yRaP7IXU{-1ODp8a!o%Zu>77z>x9{)Pna07h;dCK@F zM`0~X{x;rX>CU9uEFM)ic`W5`OJK#gK`-WKJCS$D&rB;rb8O%f*Z%~_4ED=M`|jU& zeMt3@tz4@IsHxN6@ipv=XcLR67AyX2uUbW^x|NnW?dCru>ql-ujrxG zCWUs53U^z^o38J!n^dbbU*-33h&e`Cn~9|CD&`AP?@iG>dzlsZ8Y8tMZNzQ$KpwcE_SN}?ZIA_IC+q2@ z2o|4!Rx~16p!B&;z$=!z8w`C!Z`XBI4=t|*&e~gFLplWCOH~<}X27L1iVYrrG&d=pJ#&IC=9yt4vmkc`gtiU#!Cw5Zi_Jxl{X69Niu3wKIryI@L){yb6x+RT8 zt|L_*dHmOi}ihTHcuhHagQZ=r|qZ+A|`+Do1%>)EGun)zl=Y7)?D7W0_`aj>6)sl_A?DAetv$t zdZb~!iyR*ctj%3;>f$Z~c?GPjtN?4A_PIw)xV5epg=~L3YC%}Uq>8Tl|x47&6%<`!F9U!}zDMdI57S8MH(F=JlgCdS|Dy zZ~(ITJ8)1?goE|#g}(lT*;5_B5}&0N^=5F?&ZBwrRx~%u+`W4jtTb4k*>tfO2LeC^ z51Y=OjdiocCs1~O$kZ=dUECB(*uAZ`qF%lkOfTnfqxonGxzJfMh3?c(VnH}-ljc&3o zk2EP!14{5Jau(D81|A+_jS-jWo6*f`xg39o>OJYmri;1(tqL(Q!~tUf#Rhust_1EtNDU6{OlHXG{Kkbvi z3KPZ$MA`5lE6d7s!7u{Yu$dFLsBK6vA#vQVDEiu(8jy>v1G{4_+*N;ufXq z1WbmM;+sV7t?c@V?9d5OpE`Xf?D2>GRK%G7D&8d`X)fj@q*U4J`o@PL8(sea%bPX{x zJGLgtO%R3f|0*S#+_~u3;=~W7emaa%P<;TzYzK=@cZv}7Tg^@u?TnWKEVI)OHrm6Z zeja(b%!3EH z+jV93fB{3c#`(X1H+YV%i;gO(!s=RD@ZBWgU2MG$dkvngB_1<(3~U4|J3>NTJzlU_ z2!0HX2M9U`AHZA4W@KcPOwqiJpKHw+)SL@jCWK%CK%fIJr#<^_c`!e6tX6NVU zuaUI~#?Rnt2RjyA0K-zuGWEbF;D_Wemw#Kwl&x0udskot^7I4nWLRTwL5* z0RI3RbQv@?@21OEn5g2JZo6Hay6n2W1bY{90RvA0z+VureM>2UVF{5Xf8s?fC69BR z?!hq-W-y zfe=A@w81?9pZor&6{$Fs6c`(u1$W_%;yIaJ;U zt!O;4jpE;SpGo70xYLv~XbDPdRWCK;Nu_46;tti@QuofCWAB#SE#6%hsNjB?_@gW8 zhnP+B;^EJR$zk@I{gr13ctQ)fo zK2uj;J$xtJ&n-WP>wb70X52k)Zle8o4Ae%5Pyk+Rdv0!TvuYaLdQ7exdK*oC#G4aQy-$Ovv>xrJ^X=FVq_$i)BrOInpm)yHQ*Kh6k>Gq^X7r%UrZEollAmw`fA$MUMBB?Y zJAJ8gcGIzoYBB$M40jf3mWGx)_SUWkC&wq@z?+kp4JWTNW1BOP1 zZ>5U)2*Yp{6Ps#neGaL45L5`Z8 zyP#j5q8RUIIJKXB0EdjMv@DOziU6G}PtES{*n2p?Y!L{E4Bg6eQB%nT0;C~k6@}Pn z@F^=iqFsIwHcn0?B?5~Q=qo%<_RMDwbP}1yozRJ-Cg1*=hY%TC(8WLi1W-lVz`#jL zN`eRknji>ZtublAfh1EX^Kc4?;E`(PmbUb7l??u6j?F4c`&h^l8}-)uz6dL8+t^sr zuX*5CfOd>%G(z~NMHW4X8$R6sk^Ym%e`qtlZZf$uBuJ^ve`pX3ktdOHTMI>kGU!J5 zt&n)1{R>SW+=R(;eIF3>O?4Y#AV9^pu3K&;o{t*$8-ZZLO{VL!dSKjU%8JU`@k$EYsC1H1}E2LlVqcafwh z#`F1|5?f9Bd}t5bSOd?g{EPewhhJrXkyFNdhSD)Mh{tY(8jYf=g}XB(j_Wg68Rah0 zBt2WAK)!Gr4e`5j!_k_Et#H-ULRlC+78Pu3Zbi3HUb{uthz0damA^m2+ z`BPy0v%t1zgwL>3!K9Mjn(DD=nT+^GnyXeS&k zA2V>QtY#~uG!jX0#$v$$CuPF)2gCs*Z4kw`8~j{oA>qq9AV7h1Mq>3FB&)$sD+#4C zNg*x_Hb)p&n-kx7g@jH(D(9mg0ek>Bt}|d~1U(td*p-rqsMJ*2`I`mfPZc*{C2A{n zbUog|z_h+U4(mi~>n&zx)Arb#w8}yyWo0A-0unI^RriMjReM@Lt5UYl&|kaUox9#^ ze!FQ(Jp$ox@8b1mIF=7RH|42_cOJ(osXWfe`1psV8z`C^BqBFMOc=-ej2mjBNHD-G z{67OL^9eZ5%sbzMX3=lAS3oxdFez9@B%s)Y?_NIo_>qL9C z%UWu}8v?e^ze)kiTuJln8{XPdxL9WWVy=1DS0?tf$MpgaF>^fBlR4Pcybg5_Szj- zos%+oSV`@8u@x8)o>HSS)7V)~Z8Un)aNiODoPw}MRYLCTu4i1%M;Z2=`xPG>*7mwa(k8xl`8i5}Yd7m_ll2|^k2L=Wp=UYfn5InkJB~&Q> zP#E60WiI*n2roA1(Z~=Xr-S?V1tvp!!(F|uKY#vokI7!fy4y|_NEp>#DpTtah+@0Z z9lZ2;8_E$>ulexfaJle9d3EQA0Uc*-tL~+VbDQS*$ZJjpcJT>p`83;`H{<+7t&Sp# zUt>u0TY#9rR;(cj#Q10hiq760k( z#n-*kYnftI=yu|$h$F)UT`gD~!j~;cam=sMibd;M1LW*_)z&U?*Cg|%Yol~#FmZ() zcX{%1wj?W+K|J3okXiH|Q6 zvjbzb3oPb-?dI%P`t-Ub2T?cSp#k(rD7C!|tc!tx0knmNRkr#9`X()Vi|H43*zRkA zuRoBvZt-#xp{l-NKQ=Lq7SwzaNyTpo;DhmUrgrB|$7|GUHMffyDh|$mfG-hEfgUIL zt8Ld9zsP;{YARIqd`)+^_veE&c82s|(e9V5d1^X*IQ)BR#OuOL%L#M8vUaq7ee@_4 zxj=6--oHQ8*~!7mns=5@gxCQoDLRJb6N@8#skwz`5(=B3)M@b^ZE6gqpyan#u`U>o zp-^`sRmt)7T8aMnQReSEX=1yfLhYJ2YX`N#0PeLE2rE9D3aC4iUtSSwnV}!5f${KQ z$8OSL;V>qKlderZq@l1|^en(W2#8^AXv^R!KpqGvg*Jw8+dDgpFTDo^e$aC`^r9qI zb7Q{w;Pio0KPQVNc+g%p1!w1<-yJfd~w5p95S6StC z)IJZ)Twp%`o!mm1PvFeHDmWSL!06O{e)9f3@wcC<=m~@r*P%l3;d9}^^T1Q))L4I* zg;q{?^t$>wfymKD=ua3ss=1`(N16ybd>g(&F z8|;JSw8aS`7Fs52Uk!9GBZ_saY(5}RRfUr&OI*C5kdc3wup!{FJcP)e5j1k0#=j@; zP0D4K@T~uCTAf7p-<{gAyjnHZcX79UiS;^PrQiU)alKY@S7y%*lVE*iB|5U-W^NCm zTI^g_Bt&m``h}U0hAN9xCZE5woJCn=# z74NQG_U}}?otkXcc*I?hp=Hi`kd3nBJzDd4oBq_9PE$@+Pg6aCVwxP3W`2lG2O#Y0f<8h_;t%(eAov67zepf5WB!@4?_h z`R509FQ~(sCz4VXvoHU4vNfKRr+wG87Ye`h(pf2H#Jd(~O^6e`ck$7VU~{B%4^1ZS!NClK{EXHO#*c^4}Gtcp3@U3DmyQodyG#t$VfZmE3q zKh;I5?_V_JZsM5?@;YeJ(nQ=M+qC|I@?OB_#Jo6eRn%$jJpv{MSHjgOU{4Bi7Uk}J zbIDidPo616I?1!iocY7K8 zX;@ZC*)n9APx+Kss|84iV;x#ejFIi>XwusG{gp;GKAxW->B7(N3W`I>{EqG@v6#=P z&p++(TaL!*^qvH>F&$SxyEY4 zsEca9?cr)$bQero+El-v%rE`SrJn58M!m<0w{8h^2>l_nGCM0NtmyU>g#=vH8+lF& z-i!$ULF(u3p0(0>90YAT4;;3yl|nP`tK6G4t}W8G zVG~g9^=$XPdLe>~3sdG8}|%5~gbdc0}d7inIBQ-Rxc5VajXHZ9DbJML3zxoq)-y6!SyY4DL)j$5^}BSV%Ij(&X2E-ulha z>#w4lDS)4d5~tCepy!w@ZdEGkpGr`fXr-O-^LHk7a3I}^)VXG5SCyxb^p_EZcSplw z*J%9l2URGJ1RgSs5&sjIUoETi%cXfyHz09z!aSbs!l0aqhlXW*%-SP~#7X znxOuEzB0Gq*32bZthkDVfq`$|&)~8bT{zg-C$q!!alU7AQ##33Ur>g|j0$nq@F+(s zpD@|Gt2PI1-~=Dwti?Ir^cdSvd2#p3_;Y+b*{Yd*#|W#M(Ji+AMcA{C+!{wpoaJ7=oI&m@uW-Nguq zXUO(&TKdgMn>7l@4Y0Fw4B6@#RyAUH7Q{vFe>C8s^rE-Xt#W9ey9rO3Jg`v<^0hM` zhJ)0Mx#*!@jW7S{vKQII>^((iAzE{kz>ZJtyPrw#D9b2E5)60-(1cU9@bQU%&B zZ_KO`um2%2&IdVUG93E2~%U4j2ZC=Ueeq9! zR&hUY8Q(1lYz#qu?=0fezMx{a%@gm%D|tC06kwG=wskw#73~6xar7RfGcEiItM?`O zbM$M7s}k*T*m_T2VcOj*&5-`d%uY?FiOETYTuIvK)OIJVkILViSNp+J2wBRC!fn(( zYgen{nnn3nX_6x8jI@s}Fc@y6eevd7m0C#`{X`{+#fZ3f?_Rav*%njJ=eJ#MtW{|W zFGcD%aI*bZB;$41cEf=3{-;>#0?h$HzE#$vnMGWE;e7C0 zaikPyNoc-DFaG9`a$8#Xx)fFzafGbM@nide3hG;GWQu2|AFDs5a>*MDu%;+HouDWA zzFV-8T=iHf%ko?J+eW13Yy%=tfB=B}o13%?lkl!gGUbiTr{k&sy z1jn$_Q~ugH`iK_-unyjpKirD5NVfaq>xHVGH7?5iZSO@| z@BN7+3$V#?HjR%b+tURR65oJGoj@<`8H5%ui- z_bV9>M(#$@H{;QZ>D{>@%jA!2==wmU|KW~yW>%JraGTdt4Gl5p86!{2r1ZZMV>bjo zFJnz67A&d$?6+9YZ0*p#DXM&Ya;yUA4Y~!8_)TiYW@Q$?9I^U| z2HzN*e3JO=D6J=@5l!$pG=M5T9@H2h=?$1bcFxZ*fkKKEOS@UEL&- zlFtP9z&7?6ZM7%^nDh=tc{SbiV%D>p=`vcXHZa~N^%med1&&^i)<>aS>`-AJU2O@} z3;nHKy%MJ)*7xU+*HI5s7_zU$Ld*cQ1WMzOkS7&oa&j~@G+b#{1sNGLp`YY_i0p}~ zJp5>zQQ6RfP#S!wmxzVHv0{zFiKqL3t+-BpSMT_w=W6{=Zj^j+0HIWWgmNxqszat$ zI}QFa*atoo8-#s#;8-4)!=rVn|8`9-6yLKWqX4f)F+&PsM06spu@F@54VM0zE~-y( zTPg;K$(er?aaGBTtJ(#PzuvWKeVihwrmLGgu^lRd2E&99a)hAWO%-%_WZFQ2WQ~i=me(@RQeNsmD!EB3y5H3&WA`b_2li$Az7w=A`Zzayo z&of6nog|c!w_b6sfyk@k;=71vnwsLkEI??NLhGOMcL(27au**K2ZgqC^e>RDN}$ly z)(5nZ?VKPo3O_zKd2VUb*E3Wx4QEE^>FdK`DGHZ7@&m{nqY(fCYAP!D_Hx3nU-zvZ zK;jAL*v5&9oEn0?;P2L;y3y&amRh>woJc&aF3uB(a;f&Wekd50IBpoI8P1^yf3gqn?4s6 zJ?BYOx`hl-2bcvGdoZ570g?gM=r?b08q4MJ{@ot1QbIOkd>$usXRjna^}+rE7b2sm zhzqcJh$dp@;N_pU*!q%^c`hXub^AfA;!_09j^N=>J2ppBebYc7f5)`0f2A^B-Pm|w z)WC00IUcLGW4oOhGLwuE0|F>vCs9~d(OiQ)E!VYm_Otk3m z`5SYN-#`N~Rcaa<85vo^@KKc&OgooL$#zgADC9~{BKK%B9S9oC$+_jq17AP}LWx3O z@{Iwxh2r12*#D9bfjpuFrYdkDD#DOKDQHE*m`F}dE%LILp`VRV3ah;cszLCjT!20M z=ipngcPQuLrjNlgUQuxf0J7{`3Q0cZH3LB@xIxg;IXXMDq@p74?i2*D=lC&_E6tMJp zm6aL_3SrJTEE-S@u++kHfq|tP`Ib3|P|B=S>kg>R``MzkpiPP1U}t0N8yvieym>-P zji*loF~SOwBVttDz@X^Mm&d{w$d7#6ZkC^y7p)x7jDvi-nvWkxBqdR;lEODp<@L6U zmHhXMbpCqE!GB?qG&j$#sSyS^wGgI>yemH+A68Y%L*$Pjwjr;msMVI^>ow#D-H7`2 z>B;ARUs5UpzQnP&SY&h?74iY-Ti>Tf@}h*XG;sF_|Npx$xW`l_ct;9waup8zC`I{> z)xTc@V-{e77-d*JvH0P3R|X4iz&%WqARfxQf;@I)Y%EqefQ2@$T}e?9&VgtHw-qFH zK7Rbz#)dP!=XPoVW@g_@sKnq#VE#fS1}PzBm6h+~nsh_uL%^dym&!vKgrOjismhwg z%+P+55KCUpAC6E1-x)j`mS@1Iz)&u}@|t(hqmkBs*r)Y0Bez-Oy_Heh;1Q~rul&|E z$LpvV)k0H4|EmS4l&FtYc~p3VlsoYxTT$^1^zbpoRXrf@*{S4?#ejyrzpsxm6*Y{S zoMrOPK$J5oOU!wbJZKu*+S|E#d6D4edq_ls3XD$JQBdHnipOj1;5eS+sLz9H`8R2m z$ahc#5j6g-S5+S8YZDzDDqpWe%AHQ$EE>$$p;g+_^SvY-jmC1}xFc^y0;2TxJczzo zTeAe~T)$*=*3ABiU&@JT)BDF2d3|OnK3h9G9N1B1|Hc8Hy2Rjl?)8;}a5AW@aRWF_@w{ z7r6{-%udg>wVB^(d&xE!NjEIo5kCrUO?=)pdewIlD|a7l0|;taVPHu#^z;gIbLE6F zhy{?tN)0rEG!NYGt&+Yj0asg14T1Rnj?N~RKtL^9n=vQTbv}xX(fhw!Tzz?S&^Jdp zv1xe-iOK%&;Q(KVN68c9j3d(zvX#wL%&avyyv^yFI<#IZqSt#(F>5w2cYB-Je3fp+ zIa3A6%!9N9UJnd$hbH7g`z|>dq`)&vOR~aPOt*M=`kSn*PX@#`Ry8DA0*F?#a`mcw zMk402jG04+2M1%6164I01XEz7LmVo+kZRklMHn@|*%uFfx6lP9NoBo0J!tB4V zjAcPxUi9(lXeXvwU%ptP&~>*0nf@)f2k0>b%^zL+_t$ISxSpyi0eA#&;br&DKhTal zeZ|}zzCy%G`j@i`|0j*n7$dR`19=uME`icmI<174qBw(IJ*kHTbze;tO*F3uz2U+=uOIWp!0Z zSU48{6_(b1%h>M_&6LFIF9uPc8jXsnt53uEp}ETX#4x~Of*maNQFQ|Uh0EH``MLN1 zzJV0jeUy^#N5C6s00ADH5uh#5b4|1KNGWISyMxa%=P9tAF_Etb5eRn`9Xte~&;9RD z$*{>;c#vJb*&}uJK{yr)gc}N&l5?%g`!_iCr6{fNg%#n!ZT;Nm?`i9TAXAoA1}yA4 z!-spYAAzUJ4U8#a$Tg-COo9;4F+;b>vjQT)M32!!5>9zc$;mLro-~thHgbs2CPrF} zLkjIsU_bzFWtl_sJ+5Pq1p;~&>%Wma>Ii@ooGBIutLb9~LhFh8;Pn?Mia2lswoA4h zW;=eDi!b_xRHLF@Mj}kU3D@Aha8G~3A|NpQ`*$Jk4e9=WOK=;=-amjR4uVDDSvDhG zvOR_ixU$JH_^)cxX|lU>j{=xOF;Gze>#T99qd@WR)?@(#CM=jfr$%TJ+Q@>HX92cG z;QSyI`4%mD*!XU`@{V4F1fAi&d6Puo`EpO`B>CE#*Eb_tUax=rUBE=C5oJ(!^~I?k zMiUeQ;762k;AS!3>B45zj!8=L1Pl~RfN%%R5kNWru>aUN7!8x2i!Z;SWhRREh5WL_ zlkP*+90_|n^1YEA#va+qo_^;o-c~>@wA@q65D~sa1De3BigmC0S*`F>&AIN`K z6vm8oczpP}pn#)coP(H#F~-~mml!7)Rn&qqMMC1E{Gr!9smxN4l7V8~$nS_w;px-3 zMk8=i_yQ#eMC2xc`+?y5N!|P-I<{+9L%3FkJlHp5=<@j<>PNs)to=gK)>egbdIYJ$ z{G0k@3Lw^kZLhErBMjgXZ=0UcS@lTm;!mHXVfPJ!N*9=YD+nma3e2$mn3s3lL?8|> zNpUW&Zt!U5ru)wApKub$`^Msg1dS9tyPSs`H;Tg3mrRXk_lw>v0C^1M#vE=ai^BM;=C(HZ`q)P-Z&t7o?4yFF zbnVpuO2Fjx27L&qWRXLa56rZnCG!OZI%o-irUR1J=VBmyVmae>*Ox&9*x|G&Ux}@< zJPFQz3?9Z?si=q6M2}c)Dn`V(=jN~x#-gZ3rIza^2!E1Vd^dZ{k5F9agbGRDa1j+_5622 zOnQ3H{O(AFONJ6N?e#Ef!x$X{ZIHml8h=?Maz6`Uf%F3~1K;SjjM3h+sTY6(y{e-7 z?cG zyT$)Fzn}oxrnq*98YjD~Dk@Ue3LO|0Ab1L<)N6q-bmepZSnyFr$ddSGJX2MTw#x+3 z@Ohy^_LI-g*n_2v%SM0Fs~{cOgl=9@uARNSv5bT|ezJCMcnAr)o4oxQ2(bVLP&i6QUGIc zY5(LqIlQ1C49?`PC40f2nN>yY>lhH9rU*6P+S=OBE6bNJU&24b1+^qxR#rfIzy*|2 z!ROD$R}=%vrlMcnt-|}~yT~9fIv=!<3Cv9GeQJ_l%|@rt&cf-*8VD^6e0Eio{hT;} zc1Dh;B`{Hg!p_*(I4kgg?+$qZL_Mp=K|=`v8nb?#|6TtaW0qD{C;;0UfG6hobJ0!Y za_!>-f5OteNDgfrsjksI3P;E7$#moaWd^mbU*Hrdw`m^%XkDlxt`&Ng;>XG$i7K)% zAgXlf8iOQ^`7;O4iRcpu-O+duQgnfFA}eU_wSh;}tu#}6&&nQP1uVrxwDNqy8wx=w zrywIrV_~KwzM~W2Ql1|kGzy$kwZ|Rv|4)(Rfm9@M_DJK@Hg;2&|4PRbD`h&Rim%Ue>6CPnSyiqrH4pU6LmDo zPAqmLdxTs_$KlQCOe!@yh z6>*COd-eCOx7ZI5O~V(2d&z4Dfg2oQ3*QYm$3`sXi>Fb1nk)rJa)rWEbKhVH>8-PN3fCLopYl2%x+}1WEv8lFOZf7U?$PPF zW$Wy5x=ISoJtG7Za4~8is)U#qm-V4Ls~z1gH9b=*QCfpeOFOH z`K$q*0WG71e5MN{%UAA89Cdgi4_42sb2Ehh=zIN0PAbL4>pq~F8hQjPBlZjX!|mxG zO=z%2K|*#qKO8H6oq(XHy&X=ei3er{ToEvFj;bLVp-~rp!yh^$6O0&D- zJabVf&vZ@r8TC35V*4;uPo*$2gyaTuq?dr0E`UepCr%?ZT^Zcu5{H3SdjPwWuo(NTt8&6O)Pg~yV4abYGBl(K1E{L`123)nRt=etdz|->-;1@W{0jSmKAAWwI zmhk#_-BHWx(_JG%;HFy?6@1cFX90aaXp2uzM@DpA>q?P8O|7dhf5VjVY=4X6`-kyj ztwvLiHaeL^B5wQNE*`imo~t=<<*>VlWJNKhUMIl316BI3)>hw!u|T>{uvNf$*KBNT zpvB_nhkU8r2Xs9UR|}*GfHgfmJ@6;|E3}%Dk^#;mMV@f91ORN1awIr^UBE<-CiUt< zAdz`U)st{-w3;ES6hmX3Ts^kfCj5(lS6vOQPR|qOf};AO5-Afqp@RH;(DRnFrNPqt zCkalGF^41u2vCIe!5I#_0!JH^z>bbQa1{5V=6wfG-K#>kQY{yik~PsZIOL*Po0DgY z)`|w;Rbha}Absh={#Z*i_*V(p=61PD%oQqG>>%qyAa) zH6`M?em@~8D}ORyUk3o3!m>L-Qc@E5AK@yd2XX0RIJN@TV|n=z!0G^}$jqFB zW2)dhZR9cU09`_Y`m2DLJV5NGF8&r4715EBmc!^q4w8Bw-RkDJU-K@D`48ysva@$g zPNpj7x-C9{#X018NNZDsD87fFkkIU&Aj>=)d?f@&`@t&L(Z=B=EiLWrd{ALE3`TeO z&OscaV4yC8%rc;>R+F3tzkuScN;NDEuN0vL5xjFAnJzehU%htf|4w9cWmw8A0&^TBjVNkO*w&4jcmvJ)Hd7hPXa^ys$S!J2Zucul5%j!ilC7Ba#q zkv&3ol;p^k89r9_-YYA~NcNs(gsjR+*6(%dd*An8zyI#*4|#}lu5-QDYdi)bfIo3w_y}i~9jG0GN(yI+>g@rm%AZZr@tZk6bE0kPvPt#(^TS4DZ(@4_I$eJZ z*ftiY5MP*bTZyKSNAlTPZ*CXh|9RieJ|Ng1;WTCTpX`-vk3ZSgXgMrXiiSJj!kwJ~ z>3bOgs~K?qKOm0-qAb3zmPmSNk@_gc`WsS30L;VbS`1?GxMd_XhP1R8=oQsJ64-Yp! z&z^zFB|uR8sy(QrtVybI^D9ysU%q?ZX9w{jL>Ynj@*qxW=w0*TylMV-siv;~Am#+h z06soGuuDy~wVk284F0ji%VsOzzXJdTSG|{)&p1wch87SL1H?04jdi^92SSu$=p|?P z?9fbL5_Q4=T=eY&mfbWH@ANZt)-!f=&c>-<&NHHrvN0eX- z0F5~4?Hzz+3xuleG&D3%k4_TaXW8Y_ZpE=Hg>s#KvmVkLD>8wu*9Qo}lEhpr0gDc7 z7kCK|!1sZm89b$r?d`-DxC8_=)zo~3BFt)?zC!UCl9>7KYuDf(Fxv;15?2;ghu<*!0l-(L@-txl2&Dp6WXHMjg|1=|XKW3Qq z{8{>-JRPydCnPKkf`flRq6*Acpn-?mc^JihLE{YAaZIqKCULqV!E7BCVQufNMX-(?5x)pPk?i2^y#v4!#1(|+`&i_EW63Qi@GMWg6#YxBa zvYuC}Pf$6~^jj)h1zk|dE=FHGESU)8HU^3>U`}8=JlGvG^Pb(Phq0H3<{A`MYMt4x zUHddMgL6d(8Ob(vd34shchdvy2U!IQZNHn|R?J-}_ zyn?P1ynWp3*FXxuK}g>i6%bBzXOK^l&n@)Rze}rgXsr! zLn(X0M~5?h5>*WG>|&QkE46=0$P6_*tPy|t%0oOAq#Omo5F9Bu;sfLZCUuWFRz)+e zFqT9>EEXJG2mA<2OPoD9gji5dh7`ejQsxP9T3+5|&yA01_X7&tF5ZvCTNPD8|Hf}; zrs^PPY&tAp8Pt9o8W*Q8E8Fr$*cRd`To@>lw6VJCd3tgj7jOl7vcJKqs+4|}9EA57 zB%biD(nPC5TV-)2!aRH$ z>krQpn3MT>1-+>@TN^Z6P=waK#bsr zK%}!%6xKBWk>J)jfg5x8JwNwJ41QfK4EsP@kaSh{_oJUv)^@gs_u&wQg9b8oNh#R2 zXB#_&P4-UD_&ElIwNY4b3F5`GroF(=o@MHf<|@l5l|)aN98|5O%^5dtR}UdApOj{7 z_6&A6xyv{xe>RqU7^A>RL6VquE%b``lF$ddZqac2W|@ze0O#qCkAlXwzqS$i02Vok z*rQQqY`3Wx)E{$mi;*J`k~Kkt@$Fl7!nTULJh9(F;P(1L`qUHag+FBcb7txK3?=MQ z+~dvqMV{4T10lk_kvoguzthv*?+WyBx68(S`v9b74z3Dj~;%e%RN0 ztvKC5xZGzwJ31ca)*M*SaqDRe+pC}(d^!o+ts+c|*L`Q@ZV_dn(XIH=ok0SE4A8%= z13(PUApnAzvUK?#XyL=iMyq7O*Lshi4#Ha8wc6KQ9{mX3}nxQb!=-7IWnI*iay~ypT)+BlPv|Nqlm-wZ!j@~ zzlDvn)SwWeWa7a05C>;xR>;AL18MV(L5=Fj^2qV=aqBy*+f96W(pD7=>^7adYx6nL z(SJnWkBd-I`2tlJwp0*B7o&vC6vVTZmzDqxF%Lc8_^yXQ=19$;2FEF&3x0LtrBOCD zorN7IARqwfaIk@FEDHTbRis5@xkTR`itJVEdS)ncy~S`Vztz2@tY8~qaU+_7AZVyZ z<$deTGfwb~U|r%m^iZ$|bEXoaCb^|jYez?TCn!Mx014u!yKWHuKi-%WMxjtqN_S7si%DYFEagRdjH>ynjmB>DFp4hI{z0T7> z>mk2m^RY-!nQMvD>`&6_1uh!Q){@J=De2efwu+&DyxGtM2zfVB)A4Kz)^V^#5!UNqV|>LtNd zR+XWdyIj0!MT6lHVWXC=v7S2~#S^tdYIw?vu0L4iFBz|U=&FI2b$o95nDmWuamz6A z9oigIJ`zfMCBgsjcSnod?ygqW)PRDzE68U;g-uiPCmJ?-42Fd*3Rs%39l|Y)3=iLH zc%q)217#E_RAy%@WDaDnZ9A!b4lU-OFSchS3qr< zwtdc6K#xoR1qEFje$wNmms5&?7x>?8RcN~Z@{r8VEV~A?Ud(%M9Y6+vX!V7W4h<#E zW-Hf(ZSm4hOV^b!W|@8)1la2)tyVYRF{S;8Wu(bjA3-p!E%;v5X z8a9J+XZr(ZsAU2p7%-Ew&>@H`6|+)L$6QiW!&X-d@SQUmlgH zYVkFuP?gw=_Vb#|*t;Z^_CPHgeq|nI+TqM)o9?&acf!I>9Z>c9>MFs5n+I}zC04fL z?O3niE`yr94sOjYzH)Mqpn&HNJoxbjcY%lw<0>0HNIm=M&hBaC>5eks>Z1h(Lnpq_ zHLSY_&0z>^OU6b=^L2S(Phw_H138t!nQ_k03lIk-?%k;>C@Ax zUu$6&w}d8=RsXp$gS$`Z9+XMIUxb-C8Dau6KiN_ zn1tGQ8)Foo0gjxEC1V5XN>kaQ!qhy=yh~!Z6{1+U{_VvGy&q3|WQMkSd!NKcx&oiQ zlA7%g9Tc~!)+TY*u^?&*QwG)!xYvO04IEGPwY8w50)||;D+ffQR(PN#OTe zS;(WFM^{+~oJgD0TSpF8(B!#02Z=DOCQE8-{c`S(*&DHUiXc^I%_cRoTvisMR|if2 zc-Uz}*&RM0NDC}Ih63^l^zfjEwNrR;z)Hv-a1eZaFkmTus8t~m5u4M|K~uYa7rC6e zxX*QZ~@>$F?)_r9#M)^Lo#fhZvAJ^LYY;CR>wr@`kIIe|i*)LkI`u4)_WunLy*tpsGNlD+FdB2_VP)6^S~m$)?_|iBa;? z-C#gp#8Xr zFjjV`Ut#x~WE;dHV zp`pE|lBH9m4|?iwHb5m18t_aJpfj=x7EM_?6SK3$x;(Ke7D*a4rwb7IKwSo)bTiY_ zm2*K5HiAVS)Nv-sA>cbcI;uRV0Jnvl^rowtt?5pp$)H6bd9e7J+2?oblyOb{i}SV% zFNB`bkf;_a6}VGsGW5gRu+%_EwRj?AC-W_`-ivxr4m{MyUb z4;y6Cq4b4ZOlIgx!o`qr$w!YajA@&9gp%!CNV!DCjBLR4`Z#WV{SSEz$HOK_;$G_t zw`+%FXlQ=}>UVS13$NRz$c06?L`@xdE)>7AxEMt8SocHIz@+JzkXPYWbo+&(!y`W^ z3a&i6OCBG&bIbenxK(jR2~63tGVOFN$To!6pd`u@+l`~1l$J(fL7!#^MYbLyV!Xop zLrF>_uTc+uj;qUQXMOn=`&FvTf`maSY>Zd;aLS6K6H1ZT_5>$@VKO!;4=_CCwa9=%sy+4El1uHG9zcDU)s z`}2TR`?a_C4V00gQ)uZT%D6!^xv0;4_X4gz=mBF{T|PrQP9mLp`u$po^!~cb7Wf|L za7&Cc@xL)NUz3}*e$VyzBZ@)pyK%0;CWnJoa=qkrkNNC~Cpg!Z*6a7w)jvODBWgaf zbic3BRIZ*!=^5Sm+GmFQp3>~C#d)+OxwF`)#~7^;nn$=Qf7m{M^reWQdS+qvpPHep zhM_^F(J!kqmaXr{-&#b>U3YAP4wB%DZ8+$QfZcG(wDSb;d2DVUNn=$&b;w9t`wBGl z9%@8?T7nH4&K+F(w>L&Z6@6=noTQ>tpOq9L0tAei1 zPSSNN;!|J!=)n=p@dNj0x5E+}!;amqCZ3xKlf1APIc+mf;qSNoJbEL<3q1LUj4)tyZI@_W18q6w^i_;m(9v)rS3QlLYjt%dCxI%FI ze6M;i=A0;_29$!GNzV^o`{klHf`F@7{JpX3HoI8NEo2wnY*Jj*}Jm{B@c$L0aJjZVf^O3$vx~cv% zom={QN3Nbx^YVu-N6WYB@7x9z9pBle>*?WSZ-B2?M$P7W`!<+zUe!F=aP(_{VFc-2 zT5#JA=Rb&%OrmlLUSwz6W@o=_h@8Nl6o$xORr3kpQkd48EH*)n*orkQ8SZm7BSu5p z9|QtE0B8%ybEK#&orAB{5vG1FE(QJS2RlVVT@1qVdlB-kpOor3Ztp7nPQ6+(z5Qc8 z?I*SLsncQdgiFt%IK&UwhzOKpDrTsZ*8{7|GXlCqTTtc!rnO0}Q>)qpHcrBNc`l#k z+iFSq^8FuiD0Oh2FjUr6&-0C|pmrN>jI7?w3JPCfBMUJNtzK8=x`Z}|gOxP{dIHLK z9DR3(+-VAOc+8H@t1F1iV>}q^ZQtFnXbtRoGqv@~uaH*!AXs;$?VPI`)V+QKpmrkE8jka~nR`nCOt;KfI5|zAIr~DUWX8 zZS39i+P!L$7oKh+)h9SGddy%;ia>niitTpuGh>$Wo6Qo`ShKC*Ht0WfX)zUZS=l7s zaGY&IZ3X?jatk&#lxoR8mGOQEJ$nWj#);r2ApZam?Uyy%?l<{1Drz~4A%tV%cXbGY z_T72A{H-4m>-0Sd+r3#{AMRE|!)$bgg*Cb}7|XQc=spvERzYK}eCKg=@51=*cg=d8 zln}4aUfj{a2}-SK_N))QZMr#csD`ZX?y@woAQ0YnO--fEHw!iQ{jz3-G%G}7L@%); z2CPs^s(y84T$q95{gZ`QI`z4JYv_kS%q*;Gg+Xb_y%!2jpz&jKbbpf-=eq(WSQ>#XB?4-o~bg?F)ULi+Lt9%G7ADHChCs9@{>D z;Lca;nIyYAea~)(l+b|JNHpCyzpq((MvqI{&uYsLylP*L-hqnz1(~BCI45hAY^mGt zur?NEm5{9ermBjtBS=|?GP+lZ>ego@INNEOnvepM+m|R7HUhZ#o;@HH_}G(O$A?7>vb>(|@yo-W@v$+S2$6h!bp9{e;m`)}9f zpsvn+_UXdMu=L$BD8(8(HeNh@D2w6^BlPYvCJAZJs63mUi71Fk2xpc;lv07oQew@Q zy>GEKgQVZ8<<1(X>qu;IC95<|$ec*N@Qemf0kA|MN%}~Y{%qsD2_V)G9iYKNRUue? zFrf92O^}7L`UdyyY27l9gXq@L*EDo=r;s^DNdCeG-aa!G_ySBfT5iFSH{)iO%sQHy zaV9aGz>X+rre6Hlhb_;fSr1>WzC$ozURYQlz_*(rocU|9;H0G~vmY8dm)+FoGnMNf z;N)!M$v>a6KPLrb14Qb;^z;$9YJmWhp%8zm_FsK`#({xM#eHTCN5hL6yvbhY0+v5H z8dL6R&6k6U#xtQg(4hxwa!}sqMbeG4{uQAo&nYU}SRLuB<~^THX(=^D$Jm{W2=uR5 z+Nk+spQ~Rm2Wc&|mC59^Fd=_s^F@;*Y8*?Jst2YK*=O*KKW4-!UTxf;ZGi097p11D z*$il_R~N6nyvkoH_V1=t!LK2%$Wd5EkSV09MHi3FAnGZz&(3{jbs_rCK;Jj>_)5SZ zFJb5W%>EY66y}Qo7`uR*N0<)o5k8&rwjC+e+yVsY} z6DBID2t)$D%G_?Vgc^dpg2EHi26$G0c1PKbQ(U~*jGPm;+r1^& zzHc2m!&qfTNP9s>1rR+EhW(~=zkj0`sF0~WZN<{m)p6@#G6(Txem~ax$LU|`+ zyfn;k_QfpgmB~n1%9qENj(;6`bn;Hud3qjzN$C-k%TqnSCDzP(eFA|bR)Mp7=eDhI zvjs$mDJjd;r+(#@Mq{yOX~e#?YY@uq5zdUjZjEBZQb)y54ROnz#x&Zf2=U6sW64y7 z|JY;7LwVb5F5^bvy=l_^WLN*(*_8LsH{B$Rca0QCg0bwe>}z{MxV!;tX=K-iM+;ID zA3lCmVUGrD48+7aQW8tpt`Lj_seO;i4BseZDHl}d(_^vE1JBH=tE%KlJ>sK@DMT8q z^@*&g+|lXNdAs>Q;d<29p_}Dgnk6|;84^@f^4_9s)05IfJ)&5jW3lt!wC^?Fp~!9| zPD!!j2sohLc&GJ9)y#~9_BC3AfFxk@QevTb@Jx$KXRD1)FUU?oIv@9)d7b`$HmT0Pj&*WCEd(;)j&xLH`cZ-y48MO$f!EvA*x zPbqp6^NvW2OZ?ky9!|e7RQRPC$;F#53{&u4qqrH~Ksd%UL|DqMCMbl83-LA|`W=aF5@>@jv$4ej{-XjiZ&%`i@9v6Y~wX zzx_B6eL(f{++Sn1Gti$q*CZT6qGn*XSb#iK3ENVYVDTY=7^+)=?ZOjX48yfI5lM>H z6%wX1E5!}RBBiWyzSQ*t4qld}9sU>EwL0>P?i-ZIFY2@S{cg>o^}#E3|F_6L4QU=l zOm%gq%gmv$55VPfx9}(Ht5DtrZ@twdPFB`#UZ$hq%bUo8ihM8CAXpp1gvXqn)`~T( zmuIfT&X8zVN-cN2G;h?8%}{^5ksicU{zcWlQSRTat}k5|-OZ3&+53XOvV!8V(W7;W zw#|;C%R^<_vB{$rm)D81q@D+$>&=B|V(flXr5GyAZM)cdYdY4@zRu2*GI zUty2_?khH${zZu<1cUaX z*0L@hq1XZ(DYXtvjR7xh7Kb2`z#!{`>(L4xqM~VhxBZPeLlUa*3V+b5L7Le}OoBwI zA>Ez-ZYcQ;*ZSV`SemzRbaedm4k7x#gCgTfrmPMS@>R|({2c{8;dG~@eqCOSXs z&xX6C`qfiUoJ%18JO9`oR7}jR&n`S|3V-|wLYW>7T%W<$PT!Kg>gE2#0msh}YG#c} zrPD8%fAc;UYKM$sPMu2ytUUF-uDlq1WJiR7Wx;haF~8kIbR1^6BQ>Gf#R-&Zar@*L zbjtUR9-+1}D_raTTQz9$A>pkyRh3)cd625Qakc4m0H99OjR66-wHpZ1bCF(w8BY+U zoO!1Qefi7LMq}u~#R0Vre0{Y{=SdTx`rL!zz6pp1&Bf_UnMH$e12j5U@3e77Fcol!AIRcHoY1#98`-r(Ff_%YVQrYvOTF{x%^sJO^ zzvUF2Uvr1caha<@0BvY>mBj@8eL%8AfCAnMXPh}G*88sy=&#)9Dc6#sIX!p~NrMs!VJi#3c*OAoYGVJWMFA~*P&OF^gwCHy}EKg~W zo~t%L{LC5n_`B7sUs=W-6cKH%_b}39Nh^h+CR*xr#hM47D(OJalj^p43WQ+;_^4V}ZW|`+`HtR|T&xWX$SPzYHfpNlI;kQ5`Q;3@MkRrJ{SY5Pd|Af>aAS z7k>WlEheBd2Zl#Sd1zjJ#9a!pD($Xl%d`P2=zfE$;tktT-fZ*h7nmZJp>DUiDgAg+ z2`YZsZFnnxefAJ(4Uqjf0oMX5NuNKDKO}7_FWnG;TlIeiZF>Mq*~oxl(0!3hh=!oc zIMXJ7ew!*UaQ+3gYfY(LfE#7v&@rHgcLPb|FNK?4Uz{qZI4mALf+F|m$jEghc|vM@ ze7qJHfFVwxQZQjx`0`RRL>(AlBH*GiAt-+vJJfN2+8Xpb)H(<-HMkikN&(4#S57Kt z^G`2T?`x0BHP@7Ko}j9c_Uvbk=yMc=OH3zD6ouzT?6+l5MMhQKU0uPexpo0)*5J@k zC}2YO<^!-&gI559(*qU3o}P!|LWl-jrqiN*5Bf31-V6D|s3azmjIP-iZN?k_8OhDt zI-K&6xo)4F{LoGo7!as3l7fyTAQ}kHo`u_g3X->I_V`cotOL-6nV!nq+T*`YNyYTMudI>D~GAD%GVsqHq2@*#7<=Z>oTC`n9!z|{J>#NK)^bC<{pnLeG_H;jWI69gt zTdqt2r)3JnFW}Oy0V(IQJrpJdpwD8Fkcsa^f!h(+r>h0rEBk(MXfnay552qRyR4?7 zs%kDyzBUNt9_W4mkpW7R4j`S?D7^jamkQvqds`5Ue;w;2;-CJekUzK%r$gY!yi8~d zymzJ12bD|~kg|XlGRS4Zl!HC$OB!x!jNc4XU~ATqKVecM!q-pNmjx9`HO2cEp}=VB zxfBa<7JlREb>QSex5Yx7710rB9u2o>Xm4LsZzBI6Kklz?EwvVvlsH;jLx*q0F&KWc z$h2{CW=0CofZ)8h>V#lK;v&)YM4dc#*2nMj*dTOren6@1?JY6%Oz<0wS!hIrS@}a{ z9UYygPo>K#5w`yf(ALcW_Y1fKR#w&*x9@|pEOdHPl1nwtw($Jj6cOH>u_{0wnt*;M z;K86PjPrSfGN~F70m9_(X6x?@_D4tN5mK~NRG=pb{KaO#YW8a}cAmkw{re3Q?L1we z?Y-OXm4MM-209DQi4n}IKU;Qmbew{(X?SpO>mV<}I4Jc#0&%Gj@>32ft!~L*Zi|;N zqEOC(Y%Mf26zYp$1o)w&gBj3vu=4<$dbJpkw`ka zeUJwPJfR?x9+B{Ox|f<=HjLrP`QoHP{y;oNdO{_cHgmuV%yGb15iGzf%FBTV4c$6g zUV|Sacg+Cb(@b7AHgA_tvvu8yQhG1gvL7-iv{S0 z??R(lSPeURPL=9b@MxTSnU~>k$2M3k1wY4=Mkd@d; z$jDMaj{;<6+5o}v7i*mmPKPKY|NEx&Vp-k0mKR;wwgTw5-#g_N$z18bK6WK1bEI;x zqg0MNYcIi=g`dUw#F|)E-?q?TiwKy-US-7%`{dvtmqSDK8 z!>7195v_K)iKc^DD>)SSLBPL)E^7RzCBSOH1&16K&P$Gi=1#GdPpcXrodV=76 zz#c?M?TrJ};i_6f%finDa%O}b;_r>}1mIO(u%{T%=l@Mq2y?CgxpKqYJI5yiR;4ZxhqkXr^^RkJCpZS z9`4&tI{T?9?OW(NUx}GC=Z*;`yf`jPqf13ulrVl=zZW*xH~n%95XbZD>pfqc0Mb|o zK76n$hlD68wJk07*BW|E1~dXlL$OLKfxgn4%eNl#dO!gFcOT?cz?BE2 zRBz10LQ1Nbw9k242@M6}rrZXYs(A5!R(J4~E$$+|5r=BxL8(hPg^KKSJ%9B4Q&s|8 zZKs0tOz!4vd5tI@f)NpChllWQceU%CD*9zC+xj?~v^1o5LH$Ea?%gkxxQh3Q#Oio( z?M=i?USKv+^^Lgha0ct-MAgKUN7s@CMG%NH%F5_lx@^C=xgoLie%J56a2DZ&NWjJB zpMIuS8jy@OP~1!!%pe?7gbMzPHDCU;n6pM=Ed%uzvObR<4$GmsQDUW`73>4j*KTb^ z>{Ttg>Y@(6GN7%AX@(Y78K_7j%+~odAnNG&o94m(9>@B+3&1BU1s;#T2_i#~tK$)> zXlp5+$@93K=?AH|?O?xxCbPYjRf`xq0wX1evn&@>n*6noF~8bi8xeP5WuxC6hUsr2 z*TQmOW82>zlP*~MU)0EP=@a;m3$G^O{t3Gv`~MK59OIR&<1I;llOd?;41$yqw=Vr# zjT2?}cO)lf4D?{m?fgA>S|TCKeWRye48uw@3{eE`GXHyS{NKMBzbTTWj(%$?PF_l7 zL+1AME)Y?Pj#TgwrH>#G8|ST`=yqknTr?oFhuIIy03r)SKLJd1f2jp0(PDE2ovMt} z>@SF98zcNPP*TT-?HzZA68rH4E_tPIR=HYuLhJhhZhITb{Rlx^1kuuayx=!l>T)et zuicfUrSjVo`QCPDU0p{KA^h>;6f*CN6Cx0$fR3hSk~+R{|0V~VZ07}o8_XUns7zg!q@q z^i!U@8}H8VqC&a>O7atBI=2osx~>M$;lkCbokXDk4X{{9$Uwd?^3xJTf8Um22L!Sx zV4wk?1TlRCglULB|1udktO6j%#$A|yCc*zde$#VGlv-Ihz~v#8gL?yIc@1>&ZOrrk E11+s4*8l(j literal 0 HcmV?d00001 diff --git a/setup.py b/setup.py index 9d2cc8a..6d11194 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ _REQUIRED_PACKAGES = [ 'requests', - 'tensorflow==2.1.0', + 'tensorflow==2.3.0', 'jsonschema==3.1.1', 'networkx==2.4', 'mpi4py==3.0.3', @@ -43,7 +43,7 @@ def get_dist(pkgname): if get_dist('tensorflow') is None and get_dist('tensorflow-gpu') is not None: - _REQUIRED_PACKAGES.remove('tensorflow==2.1.0') + _REQUIRED_PACKAGES.remove('tensorflow==2.3.0') setup( name="model_optimizer", @@ -60,7 +60,8 @@ def get_dist(pkgname): package_data={ 'model_optimizer': ['**/*.json', 'pruner/scheduler/uniform_auto/*.yaml', - 'pruner/scheduler/uniform_specified_layer/*.yaml'] + 'pruner/scheduler/uniform_specified_layer/*.yaml', + 'pruner/scheduler/distill/*.yaml'] }, ) diff --git a/src/model_optimizer/pruner/dataset/cifar10.py b/src/model_optimizer/pruner/dataset/cifar10.py index 64f0a73..556c48a 100644 --- a/src/model_optimizer/pruner/dataset/cifar10.py +++ b/src/model_optimizer/pruner/dataset/cifar10.py @@ -20,7 +20,7 @@ def __init__(self, config, is_training): :param is_training: whether to construct the training subset :return: """ - super(Cifar10Dataset, self).__init__(config, is_training) + super().__init__(config, is_training) if is_training: self.file_pattern = os.path.join(self.data_dir, 'train.tfrecords') self.batch_size = self.batch_size @@ -32,6 +32,7 @@ def __init__(self, config, is_training): self.num_samples_of_train = 50000 self.num_samples_of_val = 10000 + # pylint: disable=no-value-for-parameter,unexpected-keyword-arg def parse_fn(self, example_serialized): """ Parse features from the serialized data diff --git a/src/model_optimizer/pruner/dataset/dataset_base.py b/src/model_optimizer/pruner/dataset/dataset_base.py index 8cd1c42..a05dff4 100644 --- a/src/model_optimizer/pruner/dataset/dataset_base.py +++ b/src/model_optimizer/pruner/dataset/dataset_base.py @@ -62,9 +62,10 @@ def num_samples(self): else: return self.num_samples_of_val - def build(self): + def build(self, is_distill=False): """ Build dataset + :param is_distill: is distilling or not :return: batch of a dataset """ dataset = tf.data.Dataset.list_files(self.file_pattern, shuffle=True) @@ -73,7 +74,10 @@ def build(self): dataset = dataset.interleave(self.dataset_fn, cycle_length=10, num_parallel_calls=tf.data.experimental.AUTOTUNE) if self.is_training: dataset = dataset.shuffle(buffer_size=self.buffer_size).repeat() - dataset = dataset.map(self.parse_fn, num_parallel_calls=tf.data.experimental.AUTOTUNE) + if is_distill: + dataset = dataset.map(self.parse_fn_distill, num_parallel_calls=tf.data.experimental.AUTOTUNE) + else: + dataset = dataset.map(self.parse_fn, num_parallel_calls=tf.data.experimental.AUTOTUNE) return self.__build_batch(dataset) def __build_batch(self, dataset): diff --git a/src/model_optimizer/pruner/dataset/imagenet.py b/src/model_optimizer/pruner/dataset/imagenet.py index fa3717e..295b596 100644 --- a/src/model_optimizer/pruner/dataset/imagenet.py +++ b/src/model_optimizer/pruner/dataset/imagenet.py @@ -21,7 +21,7 @@ def __init__(self, config, is_training, num_shards=1, shard_index=0): :param is_training: whether to construct the training subset :return: """ - super(ImagenetDataset, self).__init__(config, is_training, num_shards, shard_index) + super().__init__(config, is_training, num_shards, shard_index) if is_training: self.file_pattern = os.path.join(self.data_dir, 'train-*-of-*') self.batch_size = self.batch_size @@ -33,6 +33,7 @@ def __init__(self, config, is_training, num_shards=1, shard_index=0): self.num_samples_of_train = 1281167 self.num_samples_of_val = 50000 + # pylint: disable=no-value-for-parameter,unexpected-keyword-arg def parse_fn(self, example_serialized): """ Parse features from the serialized data @@ -77,3 +78,14 @@ def parse_fn(self, example_serialized): num_channels=3, is_training=self.is_training) return image, label + + def parse_fn_distill(self, example_serialized): + """ + Parse features from the serialized data for distillation + :param example_serialized: serialized data + :return: {image, label},{} + """ + image, label = self.parse_fn(example_serialized) + inputs = {"image": image, "label": label} + targets = {} + return inputs, targets diff --git a/src/model_optimizer/pruner/dataset/mnist.py b/src/model_optimizer/pruner/dataset/mnist.py index 0ddc63d..d251c1f 100644 --- a/src/model_optimizer/pruner/dataset/mnist.py +++ b/src/model_optimizer/pruner/dataset/mnist.py @@ -20,7 +20,7 @@ def __init__(self, config, is_training): :param is_training: whether to construct the training subset :return: """ - super(MnistDataset, self).__init__(config, is_training) + super().__init__(config, is_training) if is_training: self.file_pattern = os.path.join(self.data_dir, 'train.tfrecords') self.batch_size = self.batch_size @@ -33,6 +33,7 @@ def __init__(self, config, is_training): self.num_samples_of_val = 10000 # pylint: disable=R0201 + # pylint: disable=no-value-for-parameter,unexpected-keyword-arg def parse_fn(self, example_serialized): """ Parse features from the serialized data diff --git a/src/model_optimizer/pruner/distill/__init__.py b/src/model_optimizer/pruner/distill/__init__.py new file mode 100644 index 0000000..e18d67c --- /dev/null +++ b/src/model_optimizer/pruner/distill/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2021 ZTE corporation. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/src/model_optimizer/pruner/distill/distill_loss.py b/src/model_optimizer/pruner/distill/distill_loss.py new file mode 100644 index 0000000..a51112d --- /dev/null +++ b/src/model_optimizer/pruner/distill/distill_loss.py @@ -0,0 +1,75 @@ +# Copyright 2021 ZTE corporation. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Distilling Loss Layer +""" +import tensorflow as tf + + +class DistillLossLayer(tf.keras.layers.Layer): + """ + Layer to compute the loss for distillation. + the total loss = the student loss + the distillation loss + + Arguments: + alpha: a float between [0.0, 1.0]. It corresponds to the importance between the student loss and the + distillation loss. + temperature: the temperature of distillation. Defaults to 10. + teacher_path: the model path of teacher. The format of the model is h5. + name: String, name to use for this layer. Defaults to 'DistillLoss'. + + Call arguments: + inputs: inputs of the layer. It corresponds to [input, y_true, y_prediction] + """ + def __init__(self, teacher_path, alpha=1.0, temperature=10, name="DistillLoss", **kwargs): + """ + :param teacher_path: the model path of teacher. The format of the model is h5. + :param alpha: a float between [0.0, 1.0]. It corresponds to the importance between the student loss and the + distillation loss. + :param temperature: the temperature of distillation. Defaults to 10. + :param name: String, name to use for this layer. Defaults to 'DistillLoss'. + """ + super().__init__(name=name, **kwargs) + self.alpha = alpha + self.temperature = temperature + self.teacher_path = teacher_path + self.accuracy_fn = tf.keras.metrics.SparseCategoricalAccuracy(name="accuracy") + self.teacher = tf.keras.models.load_model(self.teacher_path) + + # pylint: disable=unused-argument + def call(self, inputs, **kwargs): + """ + :param inputs: inputs of the layer. It corresponds to [input, y_true, y_prediction] + :return: the total loss of the distiller model + """ + x, y_true, y_pred = inputs + rtn_loss = None + if y_true is not None: + student_loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)(y_true, y_pred) + self.teacher.trainable = False + teacher_predictions = self.teacher(x) + distillation_loss = tf.keras.losses.KLDivergence()( + tf.nn.softmax(y_pred / self.temperature, axis=1), + tf.nn.softmax(teacher_predictions / self.temperature, axis=1) + ) + stu_loss = self.alpha * student_loss + dis_loss = (1 - self.alpha) * self.temperature * self.temperature * distillation_loss + rtn_loss = stu_loss + dis_loss + + self.add_loss(rtn_loss) + self.add_metric(student_loss, aggregation="mean", name="stu_loss") + self.add_metric(dis_loss, aggregation="mean", name="dis_loss") + + self.add_metric(self.accuracy_fn(y_true, y_pred)) + return rtn_loss + + def get_config(self): + """ + Implement get_config to enable serialization. + """ + config = super().get_config() + config.update({"teacher_path": self.teacher_path}) + config.update({"alpha": self.alpha}) + config.update({"temperature": self.temperature}) + return config diff --git a/src/model_optimizer/pruner/distill/distiller.py b/src/model_optimizer/pruner/distill/distiller.py new file mode 100644 index 0000000..6eb3a27 --- /dev/null +++ b/src/model_optimizer/pruner/distill/distiller.py @@ -0,0 +1,27 @@ +# Copyright 2021 ZTE corporation. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +get distiller model +""" +import tensorflow as tf + +from .distill_loss import DistillLossLayer + + +def get_distiller(student_model, scheduler_config): + """ + Get distiller model + :param student_model: student model function + :param scheduler_config: scheduler config object + :return: keras model of distiller + """ + input_img = tf.keras.layers.Input(shape=(224, 224, 3), name='image') + input_lbl = tf.keras.layers.Input((), name="label", dtype='int32') + student = student_model + _, logits = student(input_img) + total_loss = DistillLossLayer(scheduler_config['teacher_path'], scheduler_config['alpha'], + scheduler_config['temperature'], )([input_img, input_lbl, logits]) + distill_model = tf.keras.Model(inputs=[input_img, input_lbl], outputs=[logits, total_loss]) + + return distill_model diff --git a/src/model_optimizer/pruner/learner/__init__.py b/src/model_optimizer/pruner/learner/__init__.py index b689e27..d0f542b 100644 --- a/src/model_optimizer/pruner/learner/__init__.py +++ b/src/model_optimizer/pruner/learner/__init__.py @@ -20,6 +20,9 @@ def get_learner(config): elif model_name == 'resnet_50' and dataset_name == 'imagenet': from .resnet_50_imagenet import Learner return Learner(config) + elif model_name == 'resnet_101' and dataset_name == 'imagenet': + from .resnet_101_imagenet import Learner + return Learner(config) elif model_name == 'mobilenet_v1' and dataset_name == 'imagenet': from .mobilenet_v1_imagenet import Learner return Learner(config) diff --git a/src/model_optimizer/pruner/learner/learner_base.py b/src/model_optimizer/pruner/learner/learner_base.py index 98e5159..5dcafb5 100644 --- a/src/model_optimizer/pruner/learner/learner_base.py +++ b/src/model_optimizer/pruner/learner/learner_base.py @@ -12,6 +12,7 @@ from ..models import get_model from .utils import get_call_backs from ...stat import print_keras_model_summary, print_keras_model_params_flops +from ..distill.distill_loss import DistillLossLayer class LearnerBase(metaclass=abc.ABCMeta): @@ -48,7 +49,8 @@ def __init__(self, config): eval_model = tf.keras.models.clone_model(origin_eval_model) self.models_train.append(train_model) self.models_eval.append(eval_model) - self.train_dataset, self.eval_dataset = self.build_dataset() + self.train_dataset, self.eval_dataset, self.train_dataset_distill, self.eval_dataset_distill = \ + self.build_dataset() self.build_train() self.build_eval() self.load_model() @@ -64,7 +66,7 @@ def resume_epoch(self): return self.resume_from_epoch @abc.abstractmethod - def get_losses(self): + def get_losses(self, is_training=True): """ Model compile losses :return: Return model compile losses @@ -80,7 +82,7 @@ def get_optimizer(self): pass @abc.abstractmethod - def get_metrics(self): + def get_metrics(self, is_training=True): """ Model compile metrics :return: Return model compile metrics @@ -99,7 +101,15 @@ def build_dataset(self): ds_eval = get_dataset(self.config, is_training=False) self.eval_steps_per_epoch = ds_eval.steps_per_epoch eval_dataset = ds_eval.build() - return train_dataset, eval_dataset + train_dataset_distill = None + eval_dataset_distill = None + if self.config.get_attribute("scheduler") == "distill": + ds_train_distill = get_dataset(self.config, is_training=True, num_shards=hvd.size(), shard_index=hvd.rank()) + train_dataset_distill = ds_train_distill.build(True) + ds_eval_distill = get_dataset(self.config, is_training=False) + eval_dataset_distill = ds_eval_distill.build(True) + + return train_dataset, eval_dataset, train_dataset_distill, eval_dataset_distill def build_train(self): """ @@ -120,9 +130,9 @@ def build_eval(self): Model compile for eval model :return: """ - loss = self.get_losses() + loss = self.get_losses(False) optimizer = self.get_optimizer() - metrics = self.get_metrics() + metrics = self.get_metrics(False) eval_model = self.models_eval[-1] eval_model.compile(loss=loss, optimizer=optimizer, @@ -142,21 +152,34 @@ def train(self, initial_epoch=0, epochs=1, lr_schedulers=None): self.callbacks.append(tf.keras.callbacks.ModelCheckpoint(os.path.join(self.checkpoint_path, './checkpoint-{epoch}.h5'), period=self.checkpoint_save_period)) - train_model.fit(self.train_dataset, initial_epoch=initial_epoch, steps_per_epoch=self.train_steps_per_epoch, + if self.config.get_attribute('scheduler') == 'distill': + train_dataset = self.train_dataset_distill + else: + train_dataset = self.train_dataset + train_model.fit(train_dataset, initial_epoch=initial_epoch, steps_per_epoch=self.train_steps_per_epoch, epochs=epochs, verbose=self.verbose, callbacks=self.callbacks) self.cur_epoch += epochs-initial_epoch def eval(self): """ Model eval process, only evaluate on rank 0 + the format of score is like as follows: + {loss: 7.6969 dense1_loss: 5.4490 softmax_1_sparse_categorical_accuracy: 0.0665 + dense1_sparse_categorical_accuracy: 0.0665} :return: """ if hvd.rank() != 0: return eval_model = self.models_eval[-1] score = eval_model.evaluate(self.eval_dataset, steps=self.eval_steps_per_epoch) - print('Test loss:', score[0]) - print('Test accuracy:', score[1]) + loss = score[0] + if self.config.get_attribute("classifier_activation", "softmax") == "softmax": + accuracy = score[2] + else: + accuracy = score[3] + + print('Test loss:', loss) + print('Test accuracy:', accuracy) def get_latest_train_model(self): """ @@ -209,6 +232,14 @@ def load_model(self): Load checkpoint and update cur_epoch resume_from_epoch train_model :return: """ + _custom_objects = { + 'DistillLossLayer': DistillLossLayer + } + if self.config.get_attribute('scheduler') == 'distill': + custom_objects = _custom_objects + else: + custom_objects = None + self.resume_from_epoch = 0 for try_epoch in range(self.epochs, 0, -1): if os.path.exists(os.path.join(self.checkpoint_path, self.checkpoint_format.format(epoch=try_epoch))): @@ -218,7 +249,8 @@ def load_model(self): self.cur_epoch = self.resume_from_epoch model = tf.keras.models.load_model( os.path.join(self.checkpoint_path, - self.checkpoint_format.format(epoch=self.resume_from_epoch))) + self.checkpoint_format.format(epoch=self.resume_from_epoch)), + custom_objects=custom_objects) self.train_models_update(model) def save_eval_model(self): @@ -230,17 +262,28 @@ def save_eval_model(self): return train_model = self.models_train[-1] eval_model = self.models_eval[-1] - clone_model = tf.keras.models.clone_model(eval_model) - for i, layer in enumerate(clone_model.layers): - if 'Conv2D' in str(type(layer)): - clone_model.layers[i].filters = train_model.get_layer(layer.name).filters - elif 'Dense' in str(type(layer)): - clone_model.layers[i].units = train_model.get_layer(layer.name).units - pruned_eval_model = tf.keras.models.model_from_json(clone_model.to_json()) - pruned_eval_model.set_weights(train_model.get_weights()) - save_model_path = os.path.join(self.save_model_path, 'checkpoint-') + str(self.cur_epoch)+'.h5' - pruned_eval_model.save(save_model_path) - self.eval_models_update(pruned_eval_model) + save_model_path = os.path.join(self.save_model_path, 'checkpoint-') + str(self.cur_epoch) + '.h5' + if self.config.get_attribute('scheduler') == 'distill': + model_name = self.config.get_attribute('model_name') + for layer_eval in eval_model.layers: + for layer in train_model.layers: + if layer.name == model_name and layer_eval.name == model_name: + layer_eval.set_weights(layer.get_weights()) + student_eval = layer_eval + break + student_eval.save(save_model_path) + self.eval_models_update(student_eval) + else: + clone_model = tf.keras.models.clone_model(eval_model) + for i, layer in enumerate(clone_model.layers): + if 'Conv2D' in str(type(layer)): + clone_model.layers[i].filters = train_model.get_layer(layer.name).filters + elif 'Dense' in str(type(layer)): + clone_model.layers[i].units = train_model.get_layer(layer.name).units + pruned_eval_model = tf.keras.models.model_from_json(clone_model.to_json()) + pruned_eval_model.set_weights(train_model.get_weights()) + pruned_eval_model.save(save_model_path) + self.eval_models_update(pruned_eval_model) def print_model_summary(self): """ diff --git a/src/model_optimizer/pruner/learner/lenet_mnist.py b/src/model_optimizer/pruner/learner/lenet_mnist.py index 1e973f1..0ee7cd2 100644 --- a/src/model_optimizer/pruner/learner/lenet_mnist.py +++ b/src/model_optimizer/pruner/learner/lenet_mnist.py @@ -15,7 +15,7 @@ class Learner(LearnerBase): Lenet on mnist Learner """ def __init__(self, config): - super(Learner, self).__init__(config) + super().__init__(config) self.callbacks = [ # Horovod: broadcast initial variable states from rank 0 to all other processes. # This is necessary to ensure consistent initialization of all workers when @@ -42,16 +42,20 @@ def get_optimizer(self): opt = hvd.DistributedOptimizer(opt) return opt - def get_losses(self): + def get_losses(self, is_training=True): """ Model compile losses + :param is_training: is training or not :return: Return model compile losses """ return 'sparse_categorical_crossentropy' - def get_metrics(self): + def get_metrics(self, is_training=True): """ Model compile metrics + :param is_training: is training or not :return: Return model compile metrics """ + if self.config.get_attribute('scheduler') == 'distill' and is_training: + return None return ['sparse_categorical_accuracy'] diff --git a/src/model_optimizer/pruner/learner/mobilenet_v1_imagenet.py b/src/model_optimizer/pruner/learner/mobilenet_v1_imagenet.py index 12120a2..464a9b0 100644 --- a/src/model_optimizer/pruner/learner/mobilenet_v1_imagenet.py +++ b/src/model_optimizer/pruner/learner/mobilenet_v1_imagenet.py @@ -15,7 +15,7 @@ class Learner(LearnerBase): Resnet-50 on imagenet Learner """ def __init__(self, config): - super(Learner, self).__init__(config) + super().__init__(config) self.callbacks = [ # Horovod: broadcast initial variable states from rank 0 to all other processes. # This is necessary to ensure consistent initialization of all workers when @@ -52,16 +52,20 @@ def get_optimizer(self): opt = hvd.DistributedOptimizer(opt) return opt - def get_losses(self): + def get_losses(self, is_training=True): """ Model compile losses + :param is_training: is training or not :return: Return model compile losses """ return 'sparse_categorical_crossentropy' - def get_metrics(self): + def get_metrics(self, is_training=True): """ Model compile metrics + :param is_training: is training or not :return: Return model compile metrics """ + if self.config.get_attribute('scheduler') == 'distill' and is_training: + return None return ['sparse_categorical_accuracy'] diff --git a/src/model_optimizer/pruner/learner/mobilenet_v2_imagenet.py b/src/model_optimizer/pruner/learner/mobilenet_v2_imagenet.py index bcc3407..b29cbfc 100644 --- a/src/model_optimizer/pruner/learner/mobilenet_v2_imagenet.py +++ b/src/model_optimizer/pruner/learner/mobilenet_v2_imagenet.py @@ -16,7 +16,7 @@ class Learner(LearnerBase): Resnet-50 on imagenet Learner """ def __init__(self, config): - super(Learner, self).__init__(config) + super().__init__(config) self.callbacks = [ # Horovod: broadcast initial variable states from rank 0 to all other processes. # This is necessary to ensure consistent initialization of all workers when @@ -51,16 +51,20 @@ def get_optimizer(self): opt = hvd.DistributedOptimizer(opt) return opt - def get_losses(self): + def get_losses(self, is_training=True): """ Model compile losses + :param is_training: is training or not :return: Return model compile losses """ return 'sparse_categorical_crossentropy' - def get_metrics(self): + def get_metrics(self, is_training=True): """ Model compile metrics + :param is_training: is training or not :return: Return model compile metrics """ + if self.config.get_attribute('scheduler') == 'distill' and is_training: + return None return ['sparse_categorical_accuracy'] diff --git a/src/model_optimizer/pruner/learner/resnet_101_imagenet.py b/src/model_optimizer/pruner/learner/resnet_101_imagenet.py new file mode 100644 index 0000000..62a3e80 --- /dev/null +++ b/src/model_optimizer/pruner/learner/resnet_101_imagenet.py @@ -0,0 +1,78 @@ +# Copyright 2019 ZTE corporation. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Resnet-101 on imagenet Learner definition +""" +import os +import tensorflow as tf +import horovod.tensorflow.keras as hvd +from .learner_base import LearnerBase + + +class Learner(LearnerBase): + """ + Resnet-101 on imagenet Learner + """ + def __init__(self, config): + super().__init__(config) + self.callbacks = [ + # Horovod: broadcast initial variable states from rank 0 to all other processes. + # This is necessary to ensure consistent initialization of all workers when + # training is started with random weights or restored from a checkpoint. + hvd.callbacks.BroadcastGlobalVariablesCallback(0), + # Horovod: average metrics among workers at the end of every epoch. + # + # Note: This callback must be in the list before the ReduceLROnPlateau, + # TensorBoard or other metrics-based callbacks. + hvd.callbacks.MetricAverageCallback(), + # Horovod: using `lr = 1.0 * hvd.size()` from the very beginning leads to worse final + # accuracy. Scale the learning rate `lr = 1.0` ---> `lr = 1.0 * hvd.size()` during + # the first five epochs. See https://arxiv.org/abs/1706.02677 for details. + hvd.callbacks.LearningRateWarmupCallback(warmup_epochs=5, verbose=0), + # Horovod: after the warmup reduce learning rate by 10 on the 30th, 60th and 90th epochs. + hvd.callbacks.LearningRateScheduleCallback(start_epoch=5, end_epoch=30, multiplier=1.), + hvd.callbacks.LearningRateScheduleCallback(start_epoch=30, end_epoch=60, multiplier=1e-1), + hvd.callbacks.LearningRateScheduleCallback(start_epoch=60, end_epoch=90, multiplier=1e-2), + hvd.callbacks.LearningRateScheduleCallback(start_epoch=90, multiplier=1e-3), + ] + # Horovod: save checkpoints only on worker 0 to prevent other workers from corrupting them. + if hvd.rank() == 0: + self.callbacks.append(tf.keras.callbacks.ModelCheckpoint(os.path.join(self.checkpoint_path, + './checkpoint-{epoch}.h5'), + period=self.checkpoint_save_period)) + + def get_optimizer(self): + """ + Model compile optimizer + :return: Return model compile optimizer + """ + opt = tf.keras.optimizers.SGD(learning_rate=self.learning_rate*hvd.size(), momentum=0.9) + opt = hvd.DistributedOptimizer(opt) + return opt + + def get_losses(self, is_training=True): + """ + Model compile losses + :param: is_training: is training of not + :return: Return model compile losses + """ + softmax_loss = tf.keras.losses.SparseCategoricalCrossentropy() + logits_loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + if self.config.get_attribute('scheduler') == 'distill' and is_training: + return None + else: + if self.config.get_attribute("classifier_activation", "softmax") == "softmax": + return [softmax_loss, None] + else: + return [None, logits_loss] + + def get_metrics(self, is_training=True): + """ + Model compile metrics + :param: is_training: is training of not + :return: Return model compile metrics + """ + if self.config.get_attribute('scheduler') == 'distill' and is_training: + return None + return ['sparse_categorical_accuracy'] diff --git a/src/model_optimizer/pruner/learner/resnet_50_imagenet.py b/src/model_optimizer/pruner/learner/resnet_50_imagenet.py index 0c9651c..b92905c 100644 --- a/src/model_optimizer/pruner/learner/resnet_50_imagenet.py +++ b/src/model_optimizer/pruner/learner/resnet_50_imagenet.py @@ -15,7 +15,7 @@ class Learner(LearnerBase): Resnet-50 on imagenet Learner """ def __init__(self, config): - super(Learner, self).__init__(config) + super().__init__(config) self.callbacks = [ # Horovod: broadcast initial variable states from rank 0 to all other processes. # This is necessary to ensure consistent initialization of all workers when @@ -51,16 +51,28 @@ def get_optimizer(self): opt = hvd.DistributedOptimizer(opt) return opt - def get_losses(self): + def get_losses(self, is_training=True): """ Model compile losses + :param is_training: is training or not :return: Return model compile losses """ - return 'sparse_categorical_crossentropy' + softmax_loss = tf.keras.losses.SparseCategoricalCrossentropy() + logits_loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + if self.config.get_attribute('scheduler') == 'distill' and is_training: + return None + else: + if self.config.get_attribute("classifier_activation", "softmax") == "softmax": + return [softmax_loss, None] + else: + return [None, logits_loss] - def get_metrics(self): + def get_metrics(self, is_training=True): """ Model compile metrics + :param is_training: is training or not :return: Return model compile metrics """ + if self.config.get_attribute('scheduler') == 'distill' and is_training: + return None return ['sparse_categorical_accuracy'] diff --git a/src/model_optimizer/pruner/learner/vgg_m_16_cifar10.py b/src/model_optimizer/pruner/learner/vgg_m_16_cifar10.py index 94e4be1..326e9ee 100644 --- a/src/model_optimizer/pruner/learner/vgg_m_16_cifar10.py +++ b/src/model_optimizer/pruner/learner/vgg_m_16_cifar10.py @@ -15,7 +15,7 @@ class Learner(LearnerBase): VGG_m_16 on cifar10 Learner """ def __init__(self, config): - super(Learner, self).__init__(config) + super().__init__(config) self.callbacks = [ # Horovod: broadcast initial variable states from rank 0 to all other processes. # This is necessary to ensure consistent initialization of all workers when @@ -50,16 +50,20 @@ def get_optimizer(self): opt = hvd.DistributedOptimizer(opt) return opt - def get_losses(self): + def get_losses(self, is_training=True): """ Model compile losses + :param is_training: is training or not :return: Return model compile losses """ return 'sparse_categorical_crossentropy' - def get_metrics(self): + def get_metrics(self, is_training=True): """ Model compile metrics + :param is_training: is training or not :return: Return model compile metrics """ + if self.config.get_attribute('scheduler') == 'distill' and is_training: + return None return ['sparse_categorical_accuracy'] diff --git a/src/model_optimizer/pruner/models/__init__.py b/src/model_optimizer/pruner/models/__init__.py index ae0f003..16d806c 100644 --- a/src/model_optimizer/pruner/models/__init__.py +++ b/src/model_optimizer/pruner/models/__init__.py @@ -4,8 +4,11 @@ """ Get model """ +from ..scheduler.common import get_scheduler +from ..distill.distiller import get_distiller +# pylint: disable=too-many-return-statements def get_model(config, is_training=True): """ Get model @@ -14,26 +17,35 @@ def get_model(config, is_training=True): :return: class of keras Model """ model_name = config.get_attribute('model_name') - if model_name not in ['lenet', 'resnet_18', 'vgg_m_16', 'resnet_50', + scheduler_config = get_scheduler(config) + if model_name not in ['lenet', 'resnet_18', 'vgg_m_16', 'resnet_50', 'resnet_101', 'mobilenet_v1', 'mobilenet_v2']: raise Exception('Not support model %s' % model_name) if model_name == 'lenet': from .lenet import lenet - return lenet(is_training) + return lenet(model_name, is_training) elif model_name == 'vgg_m_16': from .vgg import vgg_m_16 - return vgg_m_16(is_training) + return vgg_m_16(is_training, model_name) elif model_name == 'resnet_18': from .resnet import resnet_18 - return resnet_18(is_training) + return resnet_18(is_training, model_name) elif model_name == 'resnet_50': from .resnet import resnet_50 - return resnet_50(is_training) + student_model = resnet_50(is_training, model_name) + if config.get_attribute('scheduler') == 'distill': + distill_model = get_distiller(student_model, scheduler_config) + return distill_model + else: + return student_model + elif model_name == 'resnet_101': + from .resnet import resnet_101 + return resnet_101(is_training, model_name) elif model_name == 'mobilenet_v1': from .mobilenet_v1 import mobilenet_v1_1 - return mobilenet_v1_1(is_training=is_training) + return mobilenet_v1_1(is_training=is_training, name=model_name) elif model_name == 'mobilenet_v2': from .mobilenet_v2 import mobilenet_v2_1 - return mobilenet_v2_1(is_training=is_training) + return mobilenet_v2_1(is_training=is_training, name=model_name) else: raise Exception('Not support model {}'.format(model_name)) diff --git a/src/model_optimizer/pruner/models/lenet.py b/src/model_optimizer/pruner/models/lenet.py index 9eba187..44767e3 100644 --- a/src/model_optimizer/pruner/models/lenet.py +++ b/src/model_optimizer/pruner/models/lenet.py @@ -7,9 +7,12 @@ import tensorflow as tf -def lenet(is_training=True): +def lenet(name, is_training=True): """ This implements a slightly modified LeNet-5 [LeCun et al., 1998a] + :param name: the model name + :param is_training: if training or not + :return: LeNet model """ input_ = tf.keras.layers.Input(shape=(28, 28, 1), name='input') x = tf.keras.layers.Conv2D(filters=6, @@ -31,5 +34,5 @@ def lenet(is_training=True): x = tf.keras.layers.Dense(120, activation='relu', name='dense_1')(x) x = tf.keras.layers.Dense(84, activation='relu', name='dense_2')(x) output_ = tf.keras.layers.Dense(10, activation='softmax', name='dense_3')(x) - model = tf.keras.Model(input_, output_) + model = tf.keras.Model(input_, output_, name=name) return model diff --git a/src/model_optimizer/pruner/models/mobilenet_v1.py b/src/model_optimizer/pruner/models/mobilenet_v1.py index 8024309..6ace82d 100644 --- a/src/model_optimizer/pruner/models/mobilenet_v1.py +++ b/src/model_optimizer/pruner/models/mobilenet_v1.py @@ -68,28 +68,30 @@ def mobilenet_v1_0_75(num_classes=1001, return _mobilenet_v1(num_classes, dropout_prob, is_training, scale=0.75, depth_multiplier=depth_multiplier) -def mobilenet_v1_1(num_classes=1001, +def mobilenet_v1_1(name, num_classes=1001, dropout_prob=1e-3, is_training=True, depth_multiplier=1): """ Build mobilenet_v1_1.0 model + :param name: the model name :param num_classes: :param dropout_prob: :param is_training: :param depth_multiplier: :return: """ - return _mobilenet_v1(num_classes, dropout_prob, is_training, scale=1.0, depth_multiplier=depth_multiplier) + return _mobilenet_v1(name, num_classes, dropout_prob, is_training, scale=1.0, depth_multiplier=depth_multiplier) -def _mobilenet_v1(num_classes=1000, +def _mobilenet_v1(name, num_classes=1000, dropout_prob=1e-3, is_training=True, scale=1.0, depth_multiplier=1): """ Build mobilenet_v1 model + :param name: the model name :param num_classes: :param dropout_prob: :param is_training: @@ -131,7 +133,7 @@ def _mobilenet_v1(num_classes=1000, name='conv_preds')(x) x = tf.keras.layers.Reshape((num_classes,), name='reshape_2')(x) outputs = tf.keras.layers.Activation('softmax', name='act_softmax')(x) - model = tf.keras.Model(inputs, outputs) + model = tf.keras.Model(inputs, outputs, name=name) return model diff --git a/src/model_optimizer/pruner/models/mobilenet_v2.py b/src/model_optimizer/pruner/models/mobilenet_v2.py index 23eeef0..dab3a9c 100644 --- a/src/model_optimizer/pruner/models/mobilenet_v2.py +++ b/src/model_optimizer/pruner/models/mobilenet_v2.py @@ -61,30 +61,32 @@ def mobilenet_v2_0_75(num_classes=1001, return _mobilenet_v2(num_classes, dropout_prob, is_training, scale=0.75) -def mobilenet_v2_1(num_classes=1001, +def mobilenet_v2_1(name, num_classes=1001, dropout_prob=1e-3, is_training=True): """ Build mobilenet_v2_1.0 model + :param name: the model name :param num_classes: :param dropout_prob: :param is_training: :return: """ - return _mobilenet_v2(num_classes, dropout_prob, is_training, scale=1.0) + return _mobilenet_v2(name, num_classes, dropout_prob, is_training, scale=1.0) -def mobilenet_v2_1_3(num_classes=1001, +def mobilenet_v2_1_3(name, num_classes=1001, dropout_prob=1e-3, is_training=True): """ Build mobilenet_v2_1.3 model + :param name: the model name :param num_classes: :param dropout_prob: :param is_training: :return: """ - return _mobilenet_v2(num_classes, dropout_prob, is_training, scale=1.3) + return _mobilenet_v2(name, num_classes, dropout_prob, is_training, scale=1.3) def mobilenet_v2_1_4(num_classes=1001, @@ -100,12 +102,13 @@ def mobilenet_v2_1_4(num_classes=1001, return _mobilenet_v2(num_classes, dropout_prob, is_training, scale=1.4) -def _mobilenet_v2(num_classes=1001, +def _mobilenet_v2(name, num_classes=1001, dropout_prob=1e-3, is_training=True, scale=1.0): """ Build mobilenet_v2 model + :param name: the model name :param num_classes: :param dropout_prob: :param is_training: @@ -193,7 +196,7 @@ def _mobilenet_v2(num_classes=1001, name='conv_preds')(x) x = tf.keras.layers.Reshape((num_classes,), name='reshape_2')(x) outputs = tf.keras.layers.Activation('softmax', name='act_softmax')(x) - model = tf.keras.Model(inputs, outputs) + model = tf.keras.Model(inputs, outputs, name=name) return model diff --git a/src/model_optimizer/pruner/models/resnet.py b/src/model_optimizer/pruner/models/resnet.py index 6fad6df..93e1590 100644 --- a/src/model_optimizer/pruner/models/resnet.py +++ b/src/model_optimizer/pruner/models/resnet.py @@ -20,10 +20,11 @@ def _gen_l2_regularizer(use_l2_regularizer=True): return tf.keras.regularizers.l2(L2_WEIGHT_DECAY) if use_l2_regularizer else None -def resnet(layer_num, num_classes=1001, use_l2_regularizer=True, is_training=True): +def resnet(layer_num, name, num_classes=1001, use_l2_regularizer=True, is_training=True): """ Build resnet-18 resnet-34 resnet-50 resnet-101 resnet-152 model :param layer_num: 18, 34, 50, 101, 152 + :param name: the model name :param num_classes: classification class :param use_l2_regularizer: if use l2_regularizer :param is_training: if training or not @@ -58,40 +59,52 @@ def resnet(layer_num, num_classes=1001, use_l2_regularizer=True, is_training=Tru num_filters *= 2 x = tf.keras.layers.AveragePooling2D(pool_size=7, name='avg1')(x) x = tf.keras.layers.Flatten(name='flat1')(x) - outputs = tf.keras.layers.Dense(num_classes, activation='softmax', - kernel_initializer=tf.keras.initializers.RandomNormal(stddev=0.01), - kernel_regularizer=_gen_l2_regularizer(use_l2_regularizer), - bias_regularizer=_gen_l2_regularizer(use_l2_regularizer), - name='dense1')(x) - model = tf.keras.Model(inputs, outputs) + logits = tf.keras.layers.Dense(num_classes, kernel_initializer=tf.keras.initializers.RandomNormal(stddev=0.01), + kernel_regularizer=_gen_l2_regularizer(use_l2_regularizer), + bias_regularizer=_gen_l2_regularizer(use_l2_regularizer), name='dense1')(x) + outputs = tf.keras.layers.Softmax()(logits) + model = tf.keras.Model(inputs, [outputs, logits], name=name) return model -def resnet_18(is_training): +def resnet_18(is_training, name): """ Build resnet-18 model :param is_training: if training or not + :param name: the model name :return: resnet-18 model """ - return resnet(18, is_training=is_training) + return resnet(18, is_training=is_training, name=name) -def resnet_34(is_training): +def resnet_34(is_training, name): """ Build resnet-34 model :param is_training: if training or not + :param name: the model name :return: resnet-34 model """ - return resnet(34, is_training=is_training) + return resnet(34, is_training=is_training, name=name) -def resnet_50(is_training): +def resnet_50(is_training, name): """ Build resnet-50 model :param is_training: if training or not + :param name: the model name :return: resnet-50 model """ - return resnet(50, is_training=is_training) + return resnet(50, is_training=is_training, name=name) + + +def resnet_101(is_training, name): + """ + Build resnet-101 model + :param is_training: if training or not + :param name: the model name + :return: resnet-101 model + """ + return resnet(101, is_training=is_training, name=name) def residual_block(stage, block_num, input_data, filters, kernel_size, is_training): diff --git a/src/model_optimizer/pruner/models/vgg.py b/src/model_optimizer/pruner/models/vgg.py index dafd55e..3430f2b 100644 --- a/src/model_optimizer/pruner/models/vgg.py +++ b/src/model_optimizer/pruner/models/vgg.py @@ -11,48 +11,56 @@ BATCH_NORM_EPSILON = 1e-5 -def vgg_16(is_training, num_classes=1001, use_l2_regularizer=True): +def vgg_16(is_training, name, num_classes=1001, use_l2_regularizer=True): """ VGG-16 model :param is_training: if training or not + :param name: the model name :param num_classes: classification class :param use_l2_regularizer: if use l2 regularizer or not :return: """ - return vgg(ver='D', is_training=is_training, num_classes=num_classes, use_l2_regularizer=use_l2_regularizer) + return vgg(ver='D', is_training=is_training, name=name, num_classes=num_classes, + use_l2_regularizer=use_l2_regularizer) -def vgg_19(is_training, num_classes=1001, use_l2_regularizer=True): +def vgg_19(is_training, name, num_classes=1001, use_l2_regularizer=True): """ VGG-19 model :param is_training: if training or not + :param name: the model name :param num_classes: classification class :param use_l2_regularizer: if use l2 regularizer or not :return: """ - return vgg(ver='E', is_training=is_training, num_classes=num_classes, use_l2_regularizer=use_l2_regularizer) + return vgg(ver='E', is_training=is_training, name=name, num_classes=num_classes, + use_l2_regularizer=use_l2_regularizer) -def vgg_m_16(is_training, num_classes=10, use_l2_regularizer=True): +def vgg_m_16(is_training, name, num_classes=10, use_l2_regularizer=True): """ VGG-M-16 model :param is_training: if training or not + :param name: the model name :param num_classes: classification class :param use_l2_regularizer: if use l2 regularizer or not :return: """ - return vgg_m(ver='D', is_training=is_training, num_classes=num_classes, use_l2_regularizer=use_l2_regularizer) + return vgg_m(ver='D', is_training=is_training, name=name, num_classes=num_classes, + use_l2_regularizer=use_l2_regularizer) -def vgg_m_19(is_training, num_classes=10, use_l2_regularizer=True): +def vgg_m_19(is_training, name, num_classes=10, use_l2_regularizer=True): """ VGG-M-19 model :param is_training: if training or not + :param name: the model name :param num_classes: classification class :param use_l2_regularizer: if use l2 regularizer or not :return: """ - return vgg_m(ver='E', is_training=is_training, num_classes=num_classes, use_l2_regularizer=use_l2_regularizer) + return vgg_m(ver='E', is_training=is_training, name=name, num_classes=num_classes, + use_l2_regularizer=use_l2_regularizer) def _gen_l2_regularizer(use_l2_regularizer=True): @@ -73,11 +81,12 @@ def _vgg_blocks(block, conv_num, filters, x, is_training, use_l2_regularizer=Tru return x -def vgg(ver, is_training, num_classes=1001, use_l2_regularizer=True): +def vgg(ver, is_training, name, num_classes=1001, use_l2_regularizer=True): """ VGG models :param ver: 'D' or 'E' :param is_training: if training or not + :param name: the model name :param num_classes: classification class :param use_l2_regularizer: if use l2 regularizer or not :return: @@ -113,15 +122,16 @@ def vgg(ver, is_training, num_classes=1001, use_l2_regularizer=True): kernel_regularizer=_gen_l2_regularizer(use_l2_regularizer), bias_regularizer=_gen_l2_regularizer(use_l2_regularizer), name='fc3')(x) - model = tf.keras.Model(inputs, outputs) + model = tf.keras.Model(inputs, outputs, name=name) return model -def vgg_m(ver, is_training, num_classes=10, use_l2_regularizer=True): +def vgg_m(ver, is_training, name, num_classes=10, use_l2_regularizer=True): """ VGG-M models :param ver: 'D' or 'E' :param is_training: if training or not + :param name: the model name :param num_classes: classification class :param use_l2_regularizer: if use l2 regularizer or not :return: @@ -148,5 +158,5 @@ def vgg_m(ver, is_training, num_classes=10, use_l2_regularizer=True): kernel_regularizer=_gen_l2_regularizer(use_l2_regularizer), bias_regularizer=_gen_l2_regularizer(use_l2_regularizer), name='fc2')(x) - model = tf.keras.Model(inputs, outputs) + model = tf.keras.Model(inputs, outputs, name=name) return model diff --git a/src/model_optimizer/pruner/scheduler/common.py b/src/model_optimizer/pruner/scheduler/common.py index 2614cc1..0dafbad 100644 --- a/src/model_optimizer/pruner/scheduler/common.py +++ b/src/model_optimizer/pruner/scheduler/common.py @@ -26,7 +26,7 @@ def config_get_epochs_to_train(config): :return: """ scheduler_config = get_scheduler(config) - if scheduler_config is None: + if scheduler_config is None or config.get_attribute('scheduler') == 'distill': return 0, [], None return get_epochs_lr_to_train(scheduler_config) diff --git a/src/model_optimizer/pruner/scheduler/distill/resnet_50_imagenet_0.3.yaml b/src/model_optimizer/pruner/scheduler/distill/resnet_50_imagenet_0.3.yaml new file mode 100644 index 0000000..442efed --- /dev/null +++ b/src/model_optimizer/pruner/scheduler/distill/resnet_50_imagenet_0.3.yaml @@ -0,0 +1,5 @@ +version: 1 +alpha: 0.3 +temperature: 10 +student_name: "resnet_50" +teacher_path: "/root/work/examples/models_ckpt/resnet_101_imagenet_120e_logits/checkpoint-120.h5" \ No newline at end of file diff --git a/src/model_optimizer/quantizer/calib_dataset/cifar10.py b/src/model_optimizer/quantizer/calib_dataset/cifar10.py index 98b3900..05bb6ba 100644 --- a/src/model_optimizer/quantizer/calib_dataset/cifar10.py +++ b/src/model_optimizer/quantizer/calib_dataset/cifar10.py @@ -19,10 +19,11 @@ def __init__(self, data_path): :param data_path: tfrecord data path :return: """ - super(Cifar10Dataset, self).__init__(data_path) + super().__init__(data_path) self.dataset_fn = tf.data.TFRecordDataset # pylint: disable=R0201 + # pylint: disable=no-value-for-parameter,unexpected-keyword-arg def parse_fn(self, example_serialized): """ Parse features from the serialized data diff --git a/src/model_optimizer/quantizer/calib_dataset/imagenet.py b/src/model_optimizer/quantizer/calib_dataset/imagenet.py index ced525c..9c91f04 100644 --- a/src/model_optimizer/quantizer/calib_dataset/imagenet.py +++ b/src/model_optimizer/quantizer/calib_dataset/imagenet.py @@ -20,10 +20,11 @@ def __init__(self, data_path): :param data_path: tfrecord data path :return: """ - super(ImagenetDataset, self).__init__(data_path) + super().__init__(data_path) self.dataset_fn = tf.data.TFRecordDataset # pylint: disable=R0201 + # pylint: disable=no-value-for-parameter,unexpected-keyword-arg def parse_fn(self, example_serialized): """ Parse features from the serialized data diff --git a/src/model_optimizer/quantizer/calib_dataset/mnist.py b/src/model_optimizer/quantizer/calib_dataset/mnist.py index 59f6ee0..6fe1f38 100644 --- a/src/model_optimizer/quantizer/calib_dataset/mnist.py +++ b/src/model_optimizer/quantizer/calib_dataset/mnist.py @@ -20,10 +20,11 @@ def __init__(self, data_path): :param data_path: tfrecord data path :return: """ - super(MnistDataset, self).__init__(data_path) + super().__init__(data_path) self.dataset_fn = tf.data.TFRecordDataset # pylint: disable=R0201 + # pylint: disable=no-value-for-parameter,unexpected-keyword-arg def parse_fn(self, example_serialized): """ Parse features from the serialized data diff --git a/src/model_optimizer/quantizer/tflite/optimizer.py b/src/model_optimizer/quantizer/tflite/optimizer.py index 18a769f..06dec1e 100644 --- a/src/model_optimizer/quantizer/tflite/optimizer.py +++ b/src/model_optimizer/quantizer/tflite/optimizer.py @@ -18,7 +18,7 @@ class Quantizer(BaseQuantizer): """ def __init__(self, config, calibration_input_fn): - super(Quantizer, self).__init__(config) + super().__init__(config) self.calibration_input_fn = calibration_input_fn def _do_quantize(self): diff --git a/src/model_optimizer/quantizer/tftrt/optimizer.py b/src/model_optimizer/quantizer/tftrt/optimizer.py index 05211ae..0107094 100644 --- a/src/model_optimizer/quantizer/tftrt/optimizer.py +++ b/src/model_optimizer/quantizer/tftrt/optimizer.py @@ -19,7 +19,7 @@ class Quantizer(BaseQuantizer): """ def __init__(self, config, calibration_input_fn): - super(Quantizer, self).__init__(config) + super().__init__(config) self.calibration_input_fn = calibration_input_fn def _do_quantize(self): diff --git a/src/model_optimizer/stat.py b/src/model_optimizer/stat.py index 67af166..8b5afa9 100644 --- a/src/model_optimizer/stat.py +++ b/src/model_optimizer/stat.py @@ -8,6 +8,7 @@ import numpy as np +# pylint: disable=not-context-manager def get_keras_model_flops(model_h5_path): """ Get keras model FLOPs diff --git a/tests/test_model.py b/tests/test_model.py new file mode 100644 index 0000000..62f7dc8 --- /dev/null +++ b/tests/test_model.py @@ -0,0 +1,36 @@ +""" +Tests for the model_optimizer.models.get_model method. +""" +import os +# If you did not execute the setup.py, uncomment the following four lines +from model_optimizer.pruner.config import create_config_from_obj as prune_conf_from_obj +from model_optimizer.pruner.models import get_model + + +def test_get_model_distill(): + """ + test get_model function for distillation + """ + base_dir = os.path.dirname(__file__) + request = { + "dataset": "imagenet", + "model_name": "resnet_50", + "data_dir": "", + "batch_size": 256, + "batch_size_val": 100, + "learning_rate": 0.1, + "epochs": 90, + "checkpoint_path": os.path.join(base_dir, "./models_ckpt/resnet_50_imagenet_distill"), + "checkpoint_save_period": 5, # save a checkpoint every 5 epoch + "checkpoint_eval_path": os.path.join(base_dir, "./models_eval_ckpt/resnet_50_imagenet_distill"), + "scheduler": "train", + "scheduler_file_name": "resnet_50_imagenet_0.3.yaml", + "classifier_activation": None # None or "softmax", default is softmax + } + + config = prune_conf_from_obj(request) + train_model = get_model(config, is_training=True) + for layer in train_model.layers: + if layer.name == "DistillLoss": + assert False + break diff --git a/tests/test_pruner.py b/tests/test_pruner.py index 7337582..3d2b359 100644 --- a/tests/test_pruner.py +++ b/tests/test_pruner.py @@ -2,11 +2,12 @@ Tests for the model_optimizer package. """ import tensorflow as tf +import numpy as np from model_optimizer.pruner.core import AutoPruner from model_optimizer.pruner.core import SpecifiedLayersPruner -import numpy as np +# noqa: ignore=C901 def test_uniform_auto_prune(): """ Test the AutoPruner prune function. diff --git a/tools/common/model_predict.py b/tools/common/model_predict.py index 11adab0..fca160b 100644 --- a/tools/common/model_predict.py +++ b/tools/common/model_predict.py @@ -38,11 +38,12 @@ def _get_from_saved_model(graph_func, input_data, print_result=False): return output_data -def keras_model_predict(request, file_path): +def keras_model_predict(request, file_path, is_multi_output=False): """ Keras model predict :param request: dict, must match pruner config_schema.json :param file_path: file path + :param is_multi_output: the flag of multiple output of the model :return: """ ds_val = get_dataset(prune_conf_from_obj(request), is_training=False) @@ -53,7 +54,10 @@ def keras_model_predict(request, file_path): cur_steps = 0 start = time.time() for x_test, y_test in val_dataset: - result = keras_model.predict(x_test) + if is_multi_output: + result, _ = keras_model.predict(x_test) + else: + result = keras_model.predict(x_test) output_data = tf.keras.backend.argmax(result) for j in range(y_test.shape[0]): if int(output_data[j]) == int(y_test[j]): diff --git a/tools/keras_model_predict_resnet_50_imagenet.py b/tools/keras_model_predict_resnet_50_imagenet.py index 7892060..ccc9fe8 100644 --- a/tools/keras_model_predict_resnet_50_imagenet.py +++ b/tools/keras_model_predict_resnet_50_imagenet.py @@ -18,4 +18,4 @@ "batch_size_val": 64 } model_path = os.path.join(base_dir, '../examples/models_eval_ckpt/resnet_50_imagenet_pruned/checkpoint-120.h5') - keras_model_predict(request, model_path) + keras_model_predict(request, model_path, True)